mirror of
https://github.com/Elysola/WebHooker.git
synced 2026-11-21 01:23:40 +00:00
fix(dispatch): isolate and resume tenant sends
Check tenant sources before filters and never dispatch orphan routes. Record each target outcome, isolate update chains by forge and group identity, and keep stale events from overwriting newer messages.
This commit is contained in:
1 parent
a91e8d7a7f
commit
4c03543790
5 files changed
+1236
-282
No files matched your search
+158
-10
@@ -9,7 +9,16 @@ import {
|
||||
kvMessageTracker,
|
||||
messageTracker,
|
||||
} from "../server/lib/lib/message-tracker";
|
||||
import { getDeliveryState, setDeliveryState } from "../server/lib/queue/delivery";
|
||||
import {
|
||||
acquireDelivery,
|
||||
finishLease,
|
||||
getDeliveryState,
|
||||
renewLease,
|
||||
setDeliveryState,
|
||||
} from "../server/lib/queue/delivery";
|
||||
import { createMemoryKV, createSqliteD1 } from "./d1-sqlite";
|
||||
import { chainLock } from "../server/lib/lib/message-tracker";
|
||||
import { recordSendBatch } from "../server/lib/lib/send-log-batch";
|
||||
import { canUseD1 } from "../server/lib/storage/d1";
|
||||
import type { Env } from "../server/lib/types";
|
||||
|
||||
@@ -159,17 +168,20 @@ describe("d1MessageTracker", () => {
|
||||
it("round-trips a message id", async () => {
|
||||
const { db } = createMockD1();
|
||||
const tracker = d1MessageTracker(db);
|
||||
await tracker.set("evt-1", "discord:123", "message-42");
|
||||
await expect(tracker.get("evt-1", "discord:123")).resolves.toBe("message-42");
|
||||
await tracker.set("evt-1", "discord:123", { messageId: "message-42", version: 7 });
|
||||
await expect(tracker.get("evt-1", "discord:123")).resolves.toEqual({
|
||||
messageId: "message-42",
|
||||
version: 7,
|
||||
});
|
||||
await expect(tracker.get("evt-1", "discord:999")).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it("updates on re-set and deletes on delete", async () => {
|
||||
const { db } = createMockD1();
|
||||
const tracker = d1MessageTracker(db);
|
||||
await tracker.set("evt-1", "tg:9", "old");
|
||||
await tracker.set("evt-1", "tg:9", "new");
|
||||
await expect(tracker.get("evt-1", "tg:9")).resolves.toBe("new");
|
||||
await tracker.set("evt-1", "tg:9", { messageId: "old", version: 1 });
|
||||
await tracker.set("evt-1", "tg:9", { messageId: "new", version: 2 });
|
||||
await expect(tracker.get("evt-1", "tg:9")).resolves.toEqual({ messageId: "new", version: 2 });
|
||||
await tracker.delete("evt-1", "tg:9");
|
||||
await expect(tracker.get("evt-1", "tg:9")).resolves.toBeNull();
|
||||
});
|
||||
@@ -227,8 +239,11 @@ describe("factory fallback decision", () => {
|
||||
const db = {} as D1Database;
|
||||
const kv = createMockKV();
|
||||
const tracker = messageTracker(db, kv);
|
||||
await tracker.set("evt-1", "discord:1", "m1");
|
||||
await expect(tracker.get("evt-1", "discord:1")).resolves.toBe("m1");
|
||||
await tracker.set("evt-1", "discord:1", { messageId: "m1", version: 0 });
|
||||
await expect(tracker.get("evt-1", "discord:1")).resolves.toEqual({
|
||||
messageId: "m1",
|
||||
version: 0,
|
||||
});
|
||||
await tracker.delete("evt-1", "discord:1");
|
||||
await expect(tracker.get("evt-1", "discord:1")).resolves.toBeNull();
|
||||
});
|
||||
@@ -240,8 +255,92 @@ describe("factory fallback decision", () => {
|
||||
await expect(idem.has("delivery:github:g1:e1")).resolves.toBe(true);
|
||||
|
||||
const msg = kvMessageTracker(kv);
|
||||
await msg.set("evt-1", "discord:1", "m1");
|
||||
await expect(msg.get("evt-1", "discord:1")).resolves.toBe("m1");
|
||||
await msg.set("evt-1", "discord:1", { messageId: "m1", version: 3 });
|
||||
await expect(msg.get("evt-1", "discord:1")).resolves.toEqual({ messageId: "m1", version: 3 });
|
||||
});
|
||||
|
||||
it("reads message ids stored before versions existed", async () => {
|
||||
const kv = createMockKV();
|
||||
await kv.put("msg:evt-1:discord:1", "legacy-id");
|
||||
await expect(kvMessageTracker(kv).get("evt-1", "discord:1")).resolves.toEqual({
|
||||
messageId: "legacy-id",
|
||||
version: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("D1 failures degrade to KV", () => {
|
||||
it("idempotency claims fall back to KV when the D1 table is missing", async () => {
|
||||
const { db } = createSqliteD1({
|
||||
migrations: (f) => !f.startsWith("0008") && !f.startsWith("0013"),
|
||||
});
|
||||
const { kv } = createMemoryKV();
|
||||
const store = idempotencyStore(db, kv);
|
||||
await expect(store.claim("k", 120)).resolves.toBe(true);
|
||||
await expect(store.claim("k", 120)).resolves.toBe(false);
|
||||
await store.release("k");
|
||||
await expect(store.claim("k", 120)).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it("message tracking falls back to KV when the D1 table is missing", async () => {
|
||||
const { db } = createSqliteD1({
|
||||
migrations: (f) => !f.startsWith("0008") && !f.startsWith("0013"),
|
||||
});
|
||||
const { kv } = createMemoryKV();
|
||||
const tracker = messageTracker(db, kv);
|
||||
await tracker.set("e", "t", { messageId: "m", version: 1 });
|
||||
await expect(tracker.get("e", "t")).resolves.toEqual({ messageId: "m", version: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("acquireDelivery", () => {
|
||||
function env(): Env {
|
||||
return { KV: createMemoryKV().kv, DB: createSqliteD1().db } as Env;
|
||||
}
|
||||
|
||||
it("lets exactly one consumer take a delivery", async () => {
|
||||
const e = env();
|
||||
expect((await acquireDelivery(e, "k")).status).toBe("acquired");
|
||||
expect((await acquireDelivery(e, "k")).status).toBe("busy");
|
||||
});
|
||||
|
||||
it("hands a waiting retry to the next consumer", async () => {
|
||||
const e = env();
|
||||
await acquireDelivery(e, "k");
|
||||
await setDeliveryState(e, "k", "retrying");
|
||||
expect((await acquireDelivery(e, "k")).status).toBe("acquired");
|
||||
});
|
||||
|
||||
it("reports finished deliveries", async () => {
|
||||
const e = env();
|
||||
for (const status of ["delivered", "failed", "dead"] as const) {
|
||||
await setDeliveryState(e, `k-${status}`, status);
|
||||
expect((await acquireDelivery(e, `k-${status}`)).status).toBe("finished");
|
||||
}
|
||||
});
|
||||
|
||||
it("fences a holder whose lease was taken over", async () => {
|
||||
const { db, sqlite } = createSqliteD1();
|
||||
const e = { KV: createMemoryKV().kv, DB: db } as Env;
|
||||
const first = await acquireDelivery(e, "k");
|
||||
sqlite.exec("UPDATE delivery_state SET updated_at = 0");
|
||||
// A takeover happens minutes later in practice; never in the same ms.
|
||||
await Bun.sleep(2);
|
||||
const second = await acquireDelivery(e, "k");
|
||||
expect(second.status).toBe("acquired");
|
||||
// The old holder can neither renew nor overwrite the new holder's outcome.
|
||||
expect(await renewLease(e, "k", first.token)).toBeNull();
|
||||
expect(await finishLease(e, "k", first.token, "delivered")).toBe(false);
|
||||
expect(await finishLease(e, "k", second.token, "delivered")).toBe(true);
|
||||
expect(await getDeliveryState(e, "k")).toBe("delivered");
|
||||
});
|
||||
|
||||
it("takes over a lease whose holder went silent", async () => {
|
||||
const { db, sqlite } = createSqliteD1();
|
||||
const e = { KV: createMemoryKV().kv, DB: db } as Env;
|
||||
await acquireDelivery(e, "k");
|
||||
sqlite.exec("UPDATE delivery_state SET updated_at = 0");
|
||||
expect((await acquireDelivery(e, "k")).status).toBe("acquired");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -259,3 +358,52 @@ describe("delivery state falls back to KV without batch", () => {
|
||||
await expect(getDeliveryState(env, "delivery-state:github:global:nope")).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("KV fallback transition", () => {
|
||||
it("keeps honoring and releasing claims made in KV before the table existed", async () => {
|
||||
const { db: noTable } = createSqliteD1({
|
||||
migrations: (f) => !f.startsWith("0008") && !f.startsWith("0013"),
|
||||
});
|
||||
const { kv } = createMemoryKV();
|
||||
await idempotencyStore(noTable, kv).claim("k", 600);
|
||||
// The migration is applied afterwards (same KV, a database with the table).
|
||||
const { db } = createSqliteD1();
|
||||
const store = idempotencyStore(db, kv);
|
||||
expect(await store.has("k")).toBe(true);
|
||||
expect(await store.claim("k", 600)).toBe(false);
|
||||
await store.release("k");
|
||||
expect(await store.claim("k", 600)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chainLock", () => {
|
||||
it("admits one holder at a time and only the holder can release", async () => {
|
||||
const { db } = createSqliteD1();
|
||||
const lock = chainLock(db, createMemoryKV().kv);
|
||||
const token = await lock.acquire("chain", 60);
|
||||
expect(token).not.toBeNull();
|
||||
expect(await lock.acquire("chain", 60)).toBeNull();
|
||||
await lock.release("chain", "999");
|
||||
expect(await lock.acquire("chain", 60)).toBeNull();
|
||||
await lock.release("chain", token!);
|
||||
expect(await lock.acquire("chain", 60)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("send log group attribution", () => {
|
||||
it("files rows under the id the group instance has at insert time", async () => {
|
||||
const { db, sqlite } = createSqliteD1();
|
||||
sqlite.exec(
|
||||
`INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) VALUES ('renamed', 'R', '{"id":"renamed","name":"R","adminIds":[],"instance":"abc"}', 1, 0, 0)`,
|
||||
);
|
||||
const base = { ts: 1, routeId: "r", event: "push", target: "t", ok: true };
|
||||
await recordSendBatch(db, [
|
||||
{ ...base, groupId: "old-id", groupInstance: "abc" },
|
||||
{ ...base, groupId: "deleted", groupInstance: "gone" },
|
||||
]);
|
||||
expect(sqlite.query("SELECT group_id FROM send_logs ORDER BY id").all()).toEqual([
|
||||
{ group_id: "renamed" },
|
||||
{ group_id: null },
|
||||
]);
|
||||
});
|
||||
});
|
||||
+48
-1
@@ -245,7 +245,8 @@ describe("dispatchEvent fallback routing", () => {
|
||||
route("omega-catch-all-2", "omega", { fallback: true, filters: [] }, "306"),
|
||||
];
|
||||
|
||||
await dispatchEvent({ ...baseConfig, routes }, { event: "push", payload: {} }, env);
|
||||
const groups = ["alpha", "zeta", "omega"].map((id) => ({ id, name: id, adminIds: [] }));
|
||||
await dispatchEvent({ ...baseConfig, routes }, { event: "push", payload: {} }, env, groups);
|
||||
const hits = (channel: string): number => sent.filter((u) => u.includes(`/${channel}/`)).length;
|
||||
// alpha stops after its first match without silencing zeta.
|
||||
expect(hits("301")).toBe(1);
|
||||
@@ -258,6 +259,52 @@ describe("dispatchEvent fallback routing", () => {
|
||||
expect(hits("306")).toBe(0);
|
||||
});
|
||||
|
||||
it("never fires routes whose group no longer exists", async () => {
|
||||
const sent: string[] = [];
|
||||
mockFetch((url) => {
|
||||
sent.push(url);
|
||||
return new Response("{}", { status: 200 });
|
||||
});
|
||||
const env = createEnv({ KV: createMockKV(), DB: createMockDB() });
|
||||
const routes: Route[] = [
|
||||
{
|
||||
id: "orphan",
|
||||
name: "orphan",
|
||||
enabled: true,
|
||||
filters: [{ type: "event", match: "push" }],
|
||||
targets: [{ channelId: "401" }],
|
||||
groupId: "deleted",
|
||||
},
|
||||
];
|
||||
await dispatchEvent({ ...baseConfig, routes }, { event: "push", payload: {} }, env, []);
|
||||
expect(sent).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps isolated groups off the global endpoint but serves their own ingress", async () => {
|
||||
const sent: string[] = [];
|
||||
mockFetch((url) => {
|
||||
sent.push(url);
|
||||
return new Response("{}", { status: 200 });
|
||||
});
|
||||
const env = createEnv({ KV: createMockKV(), DB: createMockDB() });
|
||||
const routes: Route[] = [
|
||||
{
|
||||
id: "mine",
|
||||
name: "mine",
|
||||
enabled: true,
|
||||
filters: [{ type: "event", match: "push" }],
|
||||
targets: [{ channelId: "501" }],
|
||||
groupId: "u-1",
|
||||
},
|
||||
];
|
||||
const groups = [{ id: "u-1", name: "me", adminIds: [], isolated: true }];
|
||||
const event = { event: "push", payload: {} };
|
||||
await dispatchEvent({ ...baseConfig, routes }, event, env, groups);
|
||||
expect(sent).toHaveLength(0);
|
||||
await dispatchEvent({ ...baseConfig, routes }, event, env, groups, { tenantId: "u-1" });
|
||||
expect(sent.filter((u) => u.includes("/501/"))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("prepends role mentions to the Discord message content", async () => {
|
||||
const bodies: string[] = [];
|
||||
mockFetch((url, init) => {
|
||||
|
||||
@@ -21,8 +21,11 @@ describe("kvMessageTracker", () => {
|
||||
it("stores and retrieves a message id by event and target", async () => {
|
||||
const kv = createMockKV();
|
||||
const tracker = kvMessageTracker(kv);
|
||||
await tracker.set("route-1:workflow-9", "channel-1", "msg-42");
|
||||
expect(await tracker.get("route-1:workflow-9", "channel-1")).toBe("msg-42");
|
||||
await tracker.set("route-1:workflow-9", "channel-1", { messageId: "msg-42", version: 5 });
|
||||
expect(await tracker.get("route-1:workflow-9", "channel-1")).toEqual({
|
||||
messageId: "msg-42",
|
||||
version: 5,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null for an unknown event/target", async () => {
|
||||
@@ -31,7 +34,7 @@ describe("kvMessageTracker", () => {
|
||||
expect(await tracker.get("missing", "channel-1")).toBeNull();
|
||||
});
|
||||
|
||||
it("uses the msg:{eventId}:{targetId} key with a 24h TTL", async () => {
|
||||
it("uses the msg:{eventId}:{targetId} key with a 30-day TTL", async () => {
|
||||
const kv = createMockKV();
|
||||
const keys = new Map<string, string>();
|
||||
const ttls = new Map<string, number>();
|
||||
@@ -44,15 +47,15 @@ describe("kvMessageTracker", () => {
|
||||
ttls.set(key, opts?.expirationTtl ?? 0);
|
||||
};
|
||||
const tracker = kvMessageTracker(kv);
|
||||
await tracker.set("event-1", "target-1", "msg-7");
|
||||
await tracker.set("event-1", "target-1", { messageId: "msg-7", version: 0 });
|
||||
expect(keys.has("msg:event-1:target-1")).toBe(true);
|
||||
expect(ttls.get("msg:event-1:target-1")).toBe(86400);
|
||||
expect(ttls.get("msg:event-1:target-1")).toBe(30 * 86400);
|
||||
});
|
||||
|
||||
it("deletes tracked entries", async () => {
|
||||
const kv = createMockKV();
|
||||
const tracker = kvMessageTracker(kv);
|
||||
await tracker.set("event-1", "target-1", "msg-7");
|
||||
await tracker.set("event-1", "target-1", { messageId: "msg-7", version: 0 });
|
||||
await tracker.delete("event-1", "target-1");
|
||||
expect(await tracker.get("event-1", "target-1")).toBeNull();
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user