diff --git a/CHANGELOG.md b/CHANGELOG.md index bb48975..4eddff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to skillhook, newest first. The format follows [Keep a Chang ## Unreleased +- A webhook that arrives while the disk is full is no longer lost. The server wrote the job, remembered the delivery for de-duplication and only then queued the job, so when saving the de-dupe index failed with `ENOSPC` the request answered 500, the job stayed `queued` without ever running, and the sender's retry was answered as a duplicate of it. The job is now queued before its delivery is remembered, and a failure to save the index is logged as a warning instead of failing the request (the retry is still recognised in memory). - An automatic update no longer bricks the server when the new version rejects the current `skillhook.json`. Before restarting onto a newer version on disk, the server runs that version's `config show --json` against its home. If the new version rejects the config, the server logs `newer version installed but it rejects the current config; keeping this version running` with the error, keeps running, and checks again every minute, so fixing `skillhook.json` lets the restart go ahead. If the check cannot run (it times out, or node is missing), the server restarts as before. On 2026-11-12, 0.16.0 rejected `accounts.*.usage_login` keys that 0.15.2 had accepted. launchd then restarted the daemon 128 times, and it ran no jobs for 17 minutes. ## 0.16.0 (2026-11-10) diff --git a/src/server.test.ts b/src/server.test.ts index bece4fc..a67632b 100644 --- a/src/server.test.ts +++ b/src/server.test.ts @@ -238,6 +238,28 @@ describe("HTTP surface", () => { expect(store.seenDelivery("hello", "ingress:hosted-response-lost-1", Date.now() + 25 * 60 * 60 * 1000)).toBe(first.job_id); }); + it("queues a webhook's job when the de-dupe index cannot be saved, so the retry is a duplicate of a job that runs (RA-021)", async () => { + const enospc = Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" }); + const save = vi.spyOn(store as unknown as { saveDeliveries(): void }, "saveDeliveries").mockImplementationOnce(() => { throw enospc; }); + const post = () => fetch(`${base}/hooks/hello`, { + method: "POST", body: JSON.stringify({ name: "Disk full" }), + headers: { authorization: "Bearer hello-secret", "content-type": "application/json", "x-skillhook-ingress-id": "disk-full-1" }, + }); + try { + const first = await post(); + expect(first.status).toBe(202); + const body = await json(first); + expect(save).toHaveBeenCalledTimes(1); + const job = await waitForJob(String(body.job_id)); + expect(job.status).toBe("succeeded"); + const retry = await post(); + expect(retry.status).toBe(200); + expect(await json(retry)).toMatchObject({ duplicate: true, job_id: body.job_id }); + } finally { + save.mockRestore(); + } + }); + it("honours Prefer: wait=N like ?wait=", async () => { const res = await fetch(`${base}/hooks/hello`, { method: "POST", body: JSON.stringify({ name: "Pref" }), headers: { authorization: "Bearer hello-secret", prefer: "wait=20" } }); expect(res.status).toBe(200); diff --git a/src/server.ts b/src/server.ts index ed48cf3..b39b73a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -578,13 +578,29 @@ export function createServer(deps: ServerDeps): Server { event, rawBody: args.rawBody, }); - if (args.deliveryId) store.rememberDelivery(args.skill.name, args.deliveryId, job.id); - if (args.ingressKey) store.rememberDelivery(args.skill.name, args.ingressKey, job.id, Date.now(), 72 * 60 * 60); logger.info("webhook accepted", { skill: args.skill.name, job: job.id, ip: args.ip, trigger: args.trigger, delivery_id: args.deliveryId, bytes: event.content_length }); + // Queue the job before remembering its delivery. When saving the de-dupe index fails (a full disk), + // the sender's retry must find a job that runs, not one written to disk but never queued (RA-021). queue.enqueue(job); + rememberJobDelivery(args.skill.name, args.deliveryId, job.id); + rememberJobDelivery(args.skill.name, args.ingressKey, job.id, 72 * 60 * 60); return job; } + /** + * Remembers which job handled a delivery. The job is already queued, so a failure to save the index + * (ENOSPC) must not fail the request: the in-memory index is updated before the write, so a retry + * reaching this process is still answered as a duplicate of the queued job. + */ + function rememberJobDelivery(skillName: string, key: string | undefined, jobId: string, retentionSeconds?: number): void { + if (!key) return; + try { + store.rememberDelivery(skillName, key, jobId, Date.now(), retentionSeconds); + } catch (error) { + logger.warn("could not save the delivery de-dupe index; the job is queued", { skill: skillName, job: jobId, error: errorMessage(error) }); + } + } + /** `POST /deliveries//replay` and `POST /jobs//replay`: the original request again, through the skill as it is now, as a new job. */ async function replay(req: IncomingMessage, res: ServerResponse, url: URL, headers: Record, source: "delivery" | "job", id: string): Promise { const rawBody = await readBody(req, config.max_body_bytes);