feat(storage): migrate config, dedup and delivery state to D1

- Move oversized queue payloads from KV to R2 (PAYLOAD binding, webhooks/YYYY/MM/DD/*.json, KV queue:payload:* fallback)
- Persist routes/groups to D1 (d1_routes/d1_groups) with memory -> KV -> D1 three-tier cache, seeded from legacy KV config keys
- Move webhook dedup (dedup_keys), delivery state (delivery_state) and message tracking (message_tracking) to D1 via canUseD1 probe with automatic KV fallback
- Batch send_logs inserts (recordSendBatch) and add group_id/ts index
- Add storage-prune scheduled task for expired dedup/state/tracking rows
- Add TTL to invite:group:{id} index and audit all ephemeral KV keys
- Add D1 indexes for the new tables
- Sync AGENTS.md, README.md/zh and docs/ (en/zh) with the new storage layout
This commit is contained in:
RhenCloud 2026-08-17 15:01:20 +08:00
parent 2e1b0f022e
commit 25ebae4ae5
No known key found for this signature in database
GPG key ID: A574A617378C4E0B
46 changed files with 1450 additions and 117 deletions

View file

@ -0,0 +1,28 @@
CREATE TABLE IF NOT EXISTS d1_groups (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
data TEXT NOT NULL,
version INTEGER NOT NULL DEFAULT 1,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS d1_routes (
id TEXT NOT NULL,
group_id TEXT NOT NULL,
name TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
filters TEXT NOT NULL,
targets TEXT NOT NULL,
stop INTEGER NOT NULL DEFAULT 0,
fallback INTEGER NOT NULL DEFAULT 0,
discord_role_ids TEXT,
ast TEXT,
version INTEGER NOT NULL DEFAULT 1,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
PRIMARY KEY (id, group_id),
FOREIGN KEY (group_id) REFERENCES d1_groups(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_d1_routes_group_id ON d1_routes(group_id);

View file

@ -0,0 +1 @@
CREATE INDEX IF NOT EXISTS idx_send_logs_group_id_ts ON send_logs (group_id, ts DESC);

View file

@ -0,0 +1,25 @@
-- D1 tables replacing ephemeral high-frequency KV keys (dedup, delivery state,
-- message tracking). Keeps KV for things that still benefit from key-based
-- access with short TTL; these tables relieve the KV write quota.
CREATE TABLE IF NOT EXISTS dedup_keys (
key TEXT PRIMARY KEY,
claimed_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_dedup_keys_expires_at ON dedup_keys (expires_at);
CREATE TABLE IF NOT EXISTS delivery_state (
key TEXT PRIMARY KEY,
status TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS message_tracking (
event_id TEXT NOT NULL,
target_id TEXT NOT NULL,
message_id TEXT NOT NULL,
updated_at INTEGER NOT NULL,
PRIMARY KEY (event_id, target_id)
);
CREATE INDEX IF NOT EXISTS idx_message_tracking_updated_at ON message_tracking (updated_at);