feat(filters): JSONPath field filters, operators, AST groups, fragments, and test-match

Add a field filter type reading any payload value by JSONPath with array expansion, 12 comparison operators (eq/ne/contains/startsWith/endsWith/regex/gt/gte/lt/lte/in/exists), a visual AST builder (all/any/not) in the route editor, chip-based multi-value input, a stateless POST /admin/api/test-match dry-run, and named filter fragments stored in D1 (d1_fragments, migration 0010) inlined into route ASTs on insert.
This commit is contained in:
RhenCloud 2026-08-18 12:19:08 +08:00
parent c955db03c3
commit c090281cb2
No known key found for this signature in database
GPG key ID: A574A617378C4E0B
30 changed files with 1793 additions and 422 deletions

View file

@ -1,14 +1,31 @@
import * as v from "valibot";
import type { Group, Route } from "../types";
import type { FilterNode, Group, Route } from "../types";
import { log } from "../lib/log";
import { explainFilterNode } from "../events/filter-ast";
export const CONFIG_SCHEMA_VERSION = 1;
export const filterSchema = v.object({
type: v.picklist(["event", "repo", "actor", "action", "branch", "keyword"]),
match: v.union([v.string(), v.array(v.string())]),
type: v.picklist(["event", "repo", "actor", "action", "branch", "keyword", "field"]),
match: v.optional(v.union([v.string(), v.array(v.string())])),
exclude: v.optional(v.boolean()),
path: v.optional(v.string()),
op: v.optional(
v.picklist([
"eq",
"ne",
"contains",
"startsWith",
"endsWith",
"regex",
"gt",
"gte",
"lt",
"lte",
"in",
"exists",
]),
),
});
export const routeTargetSchema = v.object({
@ -19,7 +36,7 @@ export const routeTargetSchema = v.object({
topicId: v.optional(v.string()),
});
export const filterNodeSchema = v.lazy(() =>
export const filterNodeSchema: v.GenericSchema<FilterNode> = v.lazy(() =>
v.union([
filterSchema,
v.object({ all: v.array(filterNodeSchema) }),

View file

@ -1,4 +1,4 @@
import type { WebhookEvent, Filter, FilterNode } from "../types";
import type { WebhookEvent, Filter, FilterNode, FilterOp } from "../types";
const regexCache = new Map<string, RegExp>();
const keywordBodyCache = new WeakMap<WebhookEvent, string>();
@ -102,39 +102,124 @@ function extractBranch(event: WebhookEvent): string | undefined {
}
}
function toPatterns(filter: Filter): string[] {
if (filter.match === undefined || filter.match === null) return [];
return Array.isArray(filter.match) ? filter.match : [filter.match];
}
function stringify(value: unknown): string {
if (typeof value === "string") return value;
if (value === undefined || value === null) return "";
if (typeof value === "number" || typeof value === "boolean") return String(value);
return JSON.stringify(value);
}
function toNumber(value: unknown): number | null {
if (typeof value === "number") return value;
if (typeof value === "boolean") return value ? 1 : 0;
if (typeof value === "string") {
const n = Number(value.trim());
return Number.isFinite(n) ? n : null;
}
return null;
}
function resolvePath(value: unknown, path: string): unknown[] {
const parts = path.split(".").filter(Boolean);
if (parts.length === 0) return [value];
const out: unknown[] = [];
walkPath(value, parts, 0, out);
return out;
}
function walkPath(node: unknown, parts: string[], index: number, out: unknown[]): void {
if (index >= parts.length) {
out.push(node);
return;
}
if (Array.isArray(node)) {
for (const item of node) walkPath(item, parts, index, out);
return;
}
if (node === null || node === undefined || typeof node !== "object") return;
const next = (node as Record<string, unknown>)[parts[index]!];
walkPath(next, parts, index + 1, out);
}
function valueList(filter: Filter, event: WebhookEvent): unknown[] {
const p = event.payload;
switch (filter.type) {
case "event":
return [event.event];
case "repo":
return [(p.repository as { full_name?: string } | undefined)?.full_name];
case "actor":
return [(p.sender as { login?: string } | undefined)?.login];
case "action":
return [typeof p.action === "string" ? p.action : undefined];
case "branch":
return [extractBranch(event)];
case "field":
return resolvePath(p, filter.path ?? "");
default:
return [];
}
}
function valueMatches(value: unknown, op: FilterOp, patterns: string[]): boolean {
const text = stringify(value);
switch (op) {
case "exists":
return value !== undefined && value !== null;
case "ne":
return !patterns.some((p) => matchField(p, text));
case "contains":
return patterns.some((p) => text.toLowerCase().includes(p.toLowerCase()));
case "startsWith":
return patterns.some((p) => text.toLowerCase().startsWith(p.toLowerCase()));
case "endsWith":
return patterns.some((p) => text.toLowerCase().endsWith(p.toLowerCase()));
case "regex":
return patterns.some((p) => {
const re = compileRegex(p);
return re ? re.test(text) : false;
});
case "gt":
case "gte":
case "lt":
case "lte": {
const n = toNumber(value);
if (n === null) return false;
return patterns.some((p) => {
const pn = toNumber(p);
if (pn === null) return false;
if (op === "gt") return n > pn;
if (op === "gte") return n >= pn;
if (op === "lt") return n < pn;
return n <= pn;
});
}
case "eq":
case "in":
default:
return patterns.some((p) => matchField(p, text));
}
}
function matchFilter(filter: Filter, event: WebhookEvent, keywordBody?: string): boolean {
if (filter.type === "keyword") {
const body = keywordBody ?? getKeywordBody(event);
const patterns = Array.isArray(filter.match) ? filter.match : [filter.match];
const patterns = toPatterns(filter);
const matches = patterns.some((p) => matchKeyword(p, body));
return filter.exclude ? !matches : matches;
}
const value = valueFor(filter, event);
if (!value) return false;
const patterns = Array.isArray(filter.match) ? filter.match : [filter.match];
const matches = patterns.some((p) => matchField(p, value));
const op = filter.op ?? "eq";
const patterns = toPatterns(filter);
const matches = valueList(filter, event).some((v) => valueMatches(v, op, patterns));
return filter.exclude ? !matches : matches;
}
function valueFor(filter: Filter, event: WebhookEvent): string | undefined {
const p = event.payload;
switch (filter.type) {
case "event":
return event.event;
case "repo":
return (p.repository as { full_name?: string } | undefined)?.full_name;
case "actor":
return (p.sender as { login?: string } | undefined)?.login;
case "action":
return typeof p.action === "string" ? p.action : undefined;
case "branch":
return extractBranch(event);
default:
return undefined;
}
}
export function containsKeyword(node: FilterNode): boolean {
if ("all" in node) return node.all.some(containsKeyword);
if ("any" in node) return node.any.some(containsKeyword);
@ -154,10 +239,20 @@ export function evaluateFilterNode(
}
export function explainFilter(filter: Filter): string {
const value = Array.isArray(filter.match)
? filter.match.map((m) => JSON.stringify(m)).join(" or ")
: JSON.stringify(filter.match);
const base = `${filter.type} ${filter.type === "keyword" ? "matches" : "is"} ${value}`;
const patterns = toPatterns(filter);
const value = patterns.map((m) => JSON.stringify(m)).join(" or ");
const label = filter.type === "field" ? filter.path ?? "field" : filter.type;
const op = filter.op ?? "eq";
let base: string;
if (filter.type === "keyword") {
base = `${label} matches ${value}`;
} else if (op === "exists") {
base = `${label} exists`;
} else if (op === "eq") {
base = `${label} is ${value}`;
} else {
base = `${label} ${op} ${value}`;
}
return filter.exclude ? `not (${base})` : base;
}

55
server/lib/fragments.ts Normal file
View file

@ -0,0 +1,55 @@
import type { FilterNode } from "./types";
import { log } from "./lib/log";
export interface NamedFragment {
id: string;
groupId?: string;
name: string;
node: FilterNode;
}
interface D1FragmentRow {
id: string;
group_id: string;
name: string;
node: string;
}
export async function loadFragments(db: D1Database): Promise<NamedFragment[]> {
try {
const stmt = db.prepare(
"SELECT id, group_id, name, node FROM d1_fragments ORDER BY id",
);
if (typeof stmt.all !== "function") return [];
const { results } = await stmt.all<D1FragmentRow>();
if (!results || results.length === 0) return [];
return results.map((r) => ({
id: r.id,
groupId: r.group_id || undefined,
name: r.name,
node: JSON.parse(r.node) as FilterNode,
}));
} catch (err) {
log.warn({ err }, "Failed to load fragments from D1");
return [];
}
}
export async function saveFragments(
db: D1Database,
fragments: NamedFragment[],
): Promise<void> {
const now = Date.now();
const statements: D1PreparedStatement[] = [
db.prepare("DELETE FROM d1_fragments"),
...fragments.map((f) =>
db
.prepare(
`INSERT INTO d1_fragments (id, group_id, name, node, version, created_at, updated_at)
VALUES (?, ?, ?, ?, 1, ?, ?)`,
)
.bind(f.id, f.groupId ?? "", f.name, JSON.stringify(f.node), now, now),
),
];
await db.batch(statements);
}

View file

@ -151,10 +151,43 @@ export interface Group {
logTarget?: RouteTarget;
}
export type FilterType =
| "event"
| "repo"
| "actor"
| "action"
| "branch"
| "keyword"
| "field";
export type FilterOp =
| "eq"
| "ne"
| "contains"
| "startsWith"
| "endsWith"
| "regex"
| "gt"
| "gte"
| "lt"
| "lte"
| "in"
| "exists";
export interface Filter {
type: "event" | "repo" | "actor" | "action" | "branch" | "keyword";
match: string | string[];
type: FilterType;
match?: string | string[];
exclude?: boolean;
/**
* JSONPath (dot notation, arrays expanded so any element matches) into
* `payload` used by `type: "field"` filters. E.g. `pull_request.user.login`.
*/
path?: string;
/**
* Comparison operator. Defaults to `eq`, which keeps the legacy glob/regex/
* case-insensitive-exact semantics. `exists` ignores `match`.
*/
op?: FilterOp;
}
/**

View file

@ -7,8 +7,10 @@ import {
setResponseHeader,
setResponseStatus,
} from "h3";
import type { Route, Group, GroupMember, GroupRole, ForgeSource } from "../types";
import type { Route, Group, GroupMember, GroupRole, ForgeSource, FilterNode } from "../types";
import { loadRoutes, saveRoutes } from "../config";
import { loadFragments, saveFragments, type NamedFragment } from "../fragments";
import { evaluateFilterNode, explainFilterNode } from "../events/filter-ast";
import { getAdminSession, destroyAdminSession, clearAdminCookie } from "./session";
import { saveGroups, loadGroups, identityMatches, normalizeGroupMembers } from "./groups";
import {
@ -35,7 +37,8 @@ import { getTenantSecret, setTenantSecret, deleteTenantSecret } from "./tenants"
import { cfEnv } from "../cf";
import { log } from "../lib/log";
const VALID_FILTER_TYPES = new Set(["event", "repo", "actor", "action", "branch", "keyword"]);
const VALID_FILTER_TYPES = new Set(["event", "repo", "actor", "action", "branch", "keyword", "field"]);
const VALID_OPS = new Set(["eq", "ne", "contains", "startsWith", "endsWith", "regex", "gt", "gte", "lt", "lte", "in", "exists"]);
const ID_RE = /^[a-z0-9][a-z0-9-]*$/;
const HOST_RE =
/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)*$/;
@ -65,6 +68,49 @@ function deepEqual(a: unknown, b: unknown): boolean {
return false;
}
function validateFilterNode(node: unknown, label: string): string | null {
if (!node || typeof node !== "object" || Array.isArray(node)) {
return `${label} must be an object`;
}
const n = node as Record<string, unknown>;
if (Array.isArray(n.all)) {
if (n.all.length === 0) return `${label}.all must not be empty`;
for (let i = 0; i < n.all.length; i++) {
const err = validateFilterNode(n.all[i], `${label}.all[${i}]`);
if (err) return err;
}
return null;
}
if (Array.isArray(n.any)) {
if (n.any.length === 0) return `${label}.any must not be empty`;
for (let i = 0; i < n.any.length; i++) {
const err = validateFilterNode(n.any[i], `${label}.any[${i}]`);
if (err) return err;
}
return null;
}
if (n.not !== undefined) return validateFilterNode(n.not, `${label}.not`);
if (!VALID_FILTER_TYPES.has(n.type as string)) {
return `${label} has unknown type`;
}
if (n.type === "field" && (typeof n.path !== "string" || n.path.trim().length === 0)) {
return `${label}.path is required for field filters`;
}
if (n.path !== undefined && typeof n.path !== "string") {
return `${label}.path must be a string`;
}
if (n.op !== undefined && !VALID_OPS.has(n.op as string)) {
return `${label}.op is invalid`;
}
if ((n.op as string) !== "exists" && !isValidMatch(n.match)) {
return `${label} needs a match value`;
}
if (n.exclude !== undefined && typeof n.exclude !== "boolean") {
return `${label}.exclude must be a boolean`;
}
return null;
}
/**
* Validates the submitted routes. Routes that are byte-for-byte identical to an
* entry in `unchanged` (keyed by id) skip the full content check, so a pre-existing
@ -122,22 +168,16 @@ function validateRoutes(
if (!Array.isArray(r.filters)) {
return { ok: false, error: `route "${r.id}".filters must be an array` };
}
if (r.fallback !== true && r.filters.length === 0) {
if (r.fallback !== true && r.filters.length === 0 && r.ast === undefined) {
return { ok: false, error: `route "${r.id}" needs at least one filter` };
}
for (let j = 0; j < r.filters.length; j++) {
const f = r.filters[j] as Record<string, unknown>;
if (!f || typeof f !== "object")
return { ok: false, error: `route "${r.id}" filter[${j}] invalid` };
if (!VALID_FILTER_TYPES.has(f.type as string)) {
return { ok: false, error: `route "${r.id}" filter[${j}] has unknown type` };
}
if (!isValidMatch(f.match)) {
return { ok: false, error: `route "${r.id}" filter[${j}] needs a match value` };
}
if (f.exclude !== undefined && typeof f.exclude !== "boolean") {
return { ok: false, error: `route "${r.id}" filter[${j}].exclude must be boolean` };
}
const err = validateFilterNode(r.filters[j], `route "${r.id}" filter[${j}]`);
if (err) return { ok: false, error: err };
}
if (r.ast !== undefined) {
const err = validateFilterNode(r.ast, `route "${r.id}".ast`);
if (err) return { ok: false, error: err };
}
const rawTarget = r.target as Record<string, unknown> | undefined;
const rawTargets = r.targets as unknown;
@ -1052,3 +1092,114 @@ export async function adminApiDelivery(
}
return { deliveryId, attempts: rows };
}
function validateFragments(
fragments: unknown,
groupId: string,
): { ok: true; fragments: NamedFragment[] } | { ok: false; error: string } {
if (!Array.isArray(fragments)) return { ok: false, error: "fragments must be an array" };
if (fragments.length > 200) return { ok: false, error: "too many fragments" };
const seen = new Set<string>();
const out: NamedFragment[] = [];
for (let i = 0; i < fragments.length; i++) {
const f = fragments[i] as Record<string, unknown>;
if (!f || typeof f !== "object") return { ok: false, error: `fragment[${i}] is not an object` };
if (typeof f.id !== "string" || !ID_RE.test(f.id)) {
return { ok: false, error: `fragment[${i}].id is invalid` };
}
if (seen.has(f.id)) return { ok: false, error: `duplicate fragment id "${f.id}"` };
seen.add(f.id);
if (typeof f.name !== "string" || f.name.trim().length === 0) {
return { ok: false, error: `fragment "${f.id}" needs a name` };
}
const err = validateFilterNode(f.node, `fragment "${f.id}".node`);
if (err) return { ok: false, error: err };
out.push({ id: f.id, groupId, name: f.name, node: f.node as FilterNode });
}
return { ok: true, fragments: out };
}
/** GET /admin/api/groups/:groupId/fragments */
export async function adminGroupFragmentsGet(
event: H3Event,
groupId: string,
): Promise<Record<string, unknown>> {
await requireAnyAccess(event);
const env = cfEnv(event);
const access = requireGroup(event, groupId);
if (!access.ok) return accessError(event, access);
const all = await loadFragments(env.DB);
return { group: access.group, fragments: all.filter((f) => f.groupId === groupId) };
}
/** PUT /admin/api/groups/:groupId/fragments */
export async function adminGroupFragmentsPut(
event: H3Event,
groupId: string,
): Promise<Record<string, unknown>> {
await requireAnyAccess(event);
const env = cfEnv(event);
const access = requireGroupRole(event, groupId, "admin");
if (!access.ok) return accessError(event, access);
const body = await readJsonBody(event);
if (!body) return respondError(event, 400, "Invalid JSON body");
const submitted = body["fragments"];
const result = validateFragments(submitted, groupId);
if (!result.ok) return respondError(event, 400, result.error);
const existing = await loadFragments(env.DB);
const others = existing.filter((f) => f.groupId !== groupId);
const nextAll = [...others, ...result.fragments];
try {
await saveFragments(env.DB, nextAll);
} catch (err) {
log.error({ err }, "Failed to save fragments");
return respondError(event, 500, "Failed to save fragments");
}
const auth = currentAuth(event);
await recordAudit(env.DB, {
ts: Date.now(),
actorId: auth.session.userId,
actorLogin: auth.session.login,
action: "group.fragments.update",
targetType: "group",
targetId: groupId,
groupId,
detail: { count: result.fragments.length },
ip: clientIp(event),
});
return { ok: true, count: result.fragments.length };
}
/** POST /admin/api/test-match */
export async function adminApiTestMatch(
event: H3Event,
): Promise<Record<string, unknown>> {
await requireAnyAccess(event);
const body = await readJsonBody(event);
if (!body) return respondError(event, 400, "Invalid JSON body");
const rawNode =
body["node"] ?? (Array.isArray(body["filters"]) ? { all: body["filters"] } : undefined);
if (rawNode === undefined) return respondError(event, 400, "Missing filter node");
const err = validateFilterNode(rawNode, "node");
if (err) return respondError(event, 400, err);
const payload = body["payload"];
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
return respondError(event, 400, "Missing payload object");
}
const eventName =
typeof body["event"] === "string" && body["event"].trim() ? body["event"].trim() : "custom";
const webhookEvent = { event: eventName, payload: payload as Record<string, unknown> };
let matched = false;
try {
matched = evaluateFilterNode(rawNode as FilterNode, webhookEvent);
} catch (e) {
return respondError(event, 400, `Failed to evaluate: ${String(e)}`);
}
return { matched, explanation: explainFilterNode(rawNode as FilterNode) };
}

View file

@ -19,6 +19,9 @@ import {
adminAudit,
adminApiMetrics,
adminApiDelivery,
adminApiTestMatch,
adminGroupFragmentsGet,
adminGroupFragmentsPut,
} from "../../../lib/web/admin";
export default defineEventHandler((event) => {
@ -36,24 +39,24 @@ export default defineEventHandler((event) => {
}
if (seg[0] === "logs" && seg.length === 1 && method === "GET") return adminApiLogs(event);
if (seg[0] === "logs" && seg.length === 2 && method === "GET")
return adminApiLogsById(event, Number(seg[1]));
return adminApiLogsById(event, Number(seg[1]!));
if (seg[0] === "groups" && seg[2] === "routes" && seg.length === 3) {
if (method === "GET") return adminGroupRoutesGet(event, seg[1]);
if (method === "PUT") return adminGroupRoutesPut(event, seg[1]);
if (method === "GET") return adminGroupRoutesGet(event, seg[1]!);
if (method === "PUT") return adminGroupRoutesPut(event, seg[1]!);
}
if (seg[0] === "groups" && seg[2] === "invites" && seg.length === 3) {
if (method === "GET") return adminGroupInvitesGet(event, seg[1]);
if (method === "POST") return adminGroupInvitesPost(event, seg[1]);
if (method === "GET") return adminGroupInvitesGet(event, seg[1]!);
if (method === "POST") return adminGroupInvitesPost(event, seg[1]!);
}
if (seg[0] === "invites" && seg.length === 2 && method === "DELETE")
return adminInviteDelete(event, seg[1]);
return adminInviteDelete(event, seg[1]!);
if (
seg[0] === "groups" &&
seg[2] === "rename" &&
seg.length === 3 &&
(method === "POST" || method === "PUT")
)
return adminGroupRename(event, seg[1]);
return adminGroupRename(event, seg[1]!);
if (
seg[0] === "groups" &&
seg[2] === "webhook" &&
@ -61,15 +64,21 @@ export default defineEventHandler((event) => {
seg.length === 4 &&
method === "POST"
)
return adminGroupWebhookRegenerate(event, seg[1]);
return adminGroupWebhookRegenerate(event, seg[1]!);
if (seg[0] === "groups" && seg[2] === "webhook" && seg.length === 3) {
if (method === "GET") return adminGroupWebhookGet(event, seg[1]);
if (method === "DELETE") return adminGroupWebhookDelete(event, seg[1]);
if (method === "GET") return adminGroupWebhookGet(event, seg[1]!);
if (method === "DELETE") return adminGroupWebhookDelete(event, seg[1]!);
}
if (seg[0] === "audit" && seg.length === 1 && method === "GET") return adminAudit(event);
if (seg[0] === "metrics" && seg.length === 1 && method === "GET") return adminApiMetrics(event);
if (seg[0] === "delivery" && seg.length === 2 && method === "GET")
return adminApiDelivery(event, seg[1]);
return adminApiDelivery(event, seg[1]!);
if (seg[0] === "test-match" && seg.length === 1 && method === "POST")
return adminApiTestMatch(event);
if (seg[0] === "groups" && seg[2] === "fragments" && seg.length === 3) {
if (method === "GET") return adminGroupFragmentsGet(event, seg[1]!);
if (method === "PUT") return adminGroupFragmentsPut(event, seg[1]!);
}
setResponseStatus(event, 404);
return { error: "Not found" };