mirror of
https://github.com/Elysola/WebHooker.git
synced 2026-11-21 01:23:40 +00:00
fix(storage): prune abandoned parked payloads
Oversized payloads can remain after interrupted enqueue attempts. Prune objects beyond their retry lifetime without deleting payloads that a current delivery may still need.
This commit is contained in:
1 parent
88800611e5
commit
6b235547d1
1 file changed
+23
-2
@@ -1,13 +1,33 @@
|
|||||||
import { cfEnv } from "../lib/cf";
|
import { cfEnv } from "../lib/cf";
|
||||||
import { log } from "../lib/lib/log";
|
import { log } from "../lib/lib/log";
|
||||||
|
import { PAYLOAD_TTL_SECONDS } from "../lib/queue/delivery";
|
||||||
|
|
||||||
const DELIVERY_STATE_RETENTION_MS = 7 * 24 * 3600 * 1000;
|
const DELIVERY_STATE_RETENTION_MS = 7 * 24 * 3600 * 1000;
|
||||||
const MESSAGE_TRACKING_RETENTION_MS = 30 * 24 * 3600 * 1000;
|
const MESSAGE_TRACKING_RETENTION_MS = 30 * 24 * 3600 * 1000;
|
||||||
|
/** Parked payloads older than every possible retry are orphans (R2 has no TTL). */
|
||||||
|
const PAYLOAD_RETENTION_MS = PAYLOAD_TTL_SECONDS * 1000 + 24 * 3600 * 1000;
|
||||||
|
const PAYLOAD_PRUNE_BATCH = 500;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete parked queue payloads left behind in R2 (a send that failed after
|
||||||
|
* the upload, a consumer that crashed before discarding). Keys sort by upload
|
||||||
|
* date (`webhooks/YYYY/MM/DD/...`), so each run looks at the oldest objects
|
||||||
|
* first and removes those past the retention window.
|
||||||
|
*/
|
||||||
|
async function prunePayloads(bucket: R2Bucket, now: number): Promise<number> {
|
||||||
|
const listed = await bucket.list({ prefix: "webhooks/", limit: PAYLOAD_PRUNE_BATCH });
|
||||||
|
const stale = listed.objects
|
||||||
|
.filter((o) => o.uploaded.getTime() < now - PAYLOAD_RETENTION_MS)
|
||||||
|
.map((o) => o.key);
|
||||||
|
if (stale.length > 0) await bucket.delete(stale);
|
||||||
|
return stale.length;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Scheduled: purge expired rows from the D1-backed ephemeral stores
|
* Scheduled: purge expired rows from the D1-backed ephemeral stores
|
||||||
* (dedup_keys, delivery_state, message_tracking). These replaced the KV keys
|
* (dedup_keys, delivery_state, message_tracking) and orphaned R2 payloads.
|
||||||
* that relied on per-entry TTL; D1 rows must be removed by this task instead.
|
* These replaced the KV keys that relied on per-entry TTL; D1 rows and R2
|
||||||
|
* objects must be removed by this task instead.
|
||||||
*/
|
*/
|
||||||
export default defineTask({
|
export default defineTask({
|
||||||
meta: { name: "storage:prune", description: "Prune D1 dedup/delivery/message-tracking rows" },
|
meta: { name: "storage:prune", description: "Prune D1 dedup/delivery/message-tracking rows" },
|
||||||
@@ -28,6 +48,7 @@ export default defineTask({
|
|||||||
.bind(now - MESSAGE_TRACKING_RETENTION_MS)
|
.bind(now - MESSAGE_TRACKING_RETENTION_MS)
|
||||||
.run();
|
.run();
|
||||||
removed += messages.meta?.changes ?? 0;
|
removed += messages.meta?.changes ?? 0;
|
||||||
|
if (env.PAYLOAD) removed += await prunePayloads(env.PAYLOAD, now);
|
||||||
if (removed > 0) log.info({ removed }, "Pruned D1 storage rows");
|
if (removed > 0) log.info({ removed }, "Pruned D1 storage rows");
|
||||||
return { result: "ok", removed };
|
return { result: "ok", removed };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
Reference in new issue
Block a user