diff --git a/CHANGELOG.md b/CHANGELOG.md index 5929131..9b77617 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to skillhook, newest first. The format follows [Keep a Chang ## Unreleased +- The `service` health check now says how the server runs and whether systemd keeps it running after logout. Its details carry `run_mode` (`service`, `foreground` or `unknown`) and, on systemd, `linger` (`yes`, `no` or `unknown`, read-only from `loginctl`). A systemd service with lingering off is a `warn` with the `loginctl enable-linger` fix. Both reach Skillhook Cloud in the health report, so a machine that goes offline at logout or when its terminal closes can be told apart from one that was stopped on purpose. `service install` on Linux warns when lingering is off, and skips the tip when it is already on. - A full disk no longer restarts the server. Recording a running job's pid or session id, and writing its stdout/stderr files, happened in child-process handlers outside any `try`, so `ENOSPC` reached the uncaught-exception handler: on 2026-11-11 one machine restarted 18 times in about two hours and every restart interrupted the jobs it was running. Those failures are now logged (`could not record job state`, `could not write job output`) and the job runs on. - The published plugin mirror (`MeterApp/skillhook-plugins`, what Anthropic's plugin directory installs) no longer carries local MCP servers, a launcher script or a lockfile: the directory refuses those from organizations that are not Verified. It now has the skills and the hosted Skillhook Cloud server, which the host signs in to with the browser. This repository's own marketplace (`/plugin marketplace add MeterApp/skillhook`) keeps both local servers, and `skillhook mcp --print-config` adds the local one to any host. - New `runners.claude.auth` (`auto`, `login`, `api_key`) chooses whether Claude Code runs and readiness checks use an API key or the subscription login, instead of whichever the environment happens to hold. `auto` (the default) behaves as before. `login` removes `ANTHROPIC_API_KEY`, `ANTHROPIC_AUTH_TOKEN` and `CLAUDE_CODE_OAUTH_TOKEN` from Claude Code runs and probes, so readiness reflects the login in `CLAUDE_CONFIG_DIR` / `~/.claude`. `api_key` counts the runner as ready only with a key and keeps the key for skills with their own `claude.config_dir`. Named accounts keep their own credential either way. `skillhook health` reports by the same rule. Changing it is answered by a fresh readiness probe at once. diff --git a/docs/operations.md b/docs/operations.md index 12586fe..8424e16 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -311,7 +311,7 @@ skillhook config set defaults.model sonnet | `tailscale` | the configured port is exposed via Funnel or Serve (URL shown) | CLI missing; not running; port not exposed | | | `public url` | `/health` answers | did not answer (certificate still provisioning, or the server is down) | | | `server` | running (version, queue) | not running | | -| `service` | running (pid) | installed but not running | (`skip` when not installed or unsupported platform) | +| `service` | running (pid) | installed but not running; systemd running with lingering off (`loginctl enable-linger $USER`) | (`skip` when not installed or unsupported platform). Details carry `run_mode` (`service`: the server is the service's own process; `foreground`: started by hand, stops with its terminal; `unknown`: no running server) and, on systemd, `linger` (`yes`, `no`, `unknown`, read-only from `loginctl show-user`). Both reach Skillhook Cloud in the health report. | | `cloud link` | connected (URL, machine, mode, last sync) | no running server keeps it; degraded; events dropped | enabled without `SKILLHOOK_CLOUD_TOKEN`; not https; token revoked or machine disabled (`skip` when not connected or `SKILLHOOK_NO_CLOUD` is set) | Every check carries a `group` (`system`, `skillhook`, `runners`, `tools`, `skills`, `exposure`) and, where useful, `data` with the facts behind the line (versions, paths, the last job). diff --git a/src/health.ts b/src/health.ts index 420847f..81d3e1c 100644 --- a/src/health.ts +++ b/src/health.ts @@ -21,7 +21,8 @@ import { checkAccountReadiness, CLAUDE_TOKEN_REFUSED, claudeLoginWorks, claudeMe import { baseRunEnv, CLAUDE_API_KEY_VARS, claudeConfigDir, withClaudeAuth, withClaudeConfigDir } from "./runners/env.js"; import { commandParts } from "./runners/types.js"; import { nextRun } from "./schedule.js"; -import { serviceStatus } from "./service.js"; +import { userInfo } from "node:os"; +import { lingerStatus, runModeOf, serviceStatus } from "./service.js"; import { currentExposures, findTailscale, run, tailscaleStatus, which } from "./tailscale.js"; import { probeClaude, probeCodex, type ClaudeProbe, type CodexProbe } from "./tools.js"; import { autoUpdateEnabled, checkForUpdate, detectInstall, lastInstall, registryUrl, releaseNotesUrl, updateChecksDisabled, updateStatusFromCache } from "./update.js"; @@ -76,6 +77,8 @@ export interface HealthOptions { exposure?: boolean; /** Look at the launchd / systemd service. Default true. */ service?: boolean; + /** Read systemd lingering (tests). Default: `loginctl show-user`. */ + linger?: () => Promise<"yes" | "no" | "unknown">; /** Facts of the server this runs inside, instead of probing it over HTTP. */ live?: () => { started_at: string; queue: { running: number; queued: number } }; /** For the slow probes (`claude mcp list` connects to every server, `codex doctor`). Default 20 s. */ @@ -499,6 +502,7 @@ export async function runHealth(paths: Paths, options: HealthOptions = {}): Prom let server: HealthReport["server"]; let linkStatus: LinkStatusView | null | undefined; let serverRunning = false; + let serverStatePid: number | undefined; if (config && options.live) { const live = options.live(); const baseUrl = localBaseUrl({ host: config.host, port: config.port }); @@ -511,6 +515,7 @@ export async function runHealth(paths: Paths, options: HealthOptions = {}): Prom const baseUrl = running?.baseUrl ?? localBaseUrl({ host: config.host, port: config.port }); server = { base_url: baseUrl, running: Boolean(running), version: running?.health.version }; serverRunning = Boolean(running); + serverStatePid = running?.state.pid; linkStatus = running?.health.cloud; check("skillhook", "server", running ? "ok" : "warn", running ? `running at ${baseUrl} (v${running.health.version}${running.health.queue ? `, ${running.health.queue.running} running / ${running.health.queue.queued} queued` : ""})` : `not running at ${baseUrl}`, running ? undefined : "run: skillhook serve (or: skillhook service install)", { base_url: baseUrl, running: Boolean(running), version: running?.health.version ?? null }); } @@ -532,7 +537,16 @@ export async function runHealth(paths: Paths, options: HealthOptions = {}): Prom if (options.service !== false) { const service = await serviceStatus(paths); if (service.platform === "unsupported") check("skillhook", "service", "skip", "no launchd/systemd on this platform"); - else check("skillhook", "service", service.running ? "ok" : service.installed ? "warn" : "skip", service.running ? `${service.platform} running (pid ${service.pid})` : service.installed ? `${service.platform} installed but not running` : "not installed", service.running ? undefined : "run: skillhook service install", { platform: service.platform, installed: service.installed, running: service.running, pid: service.pid ?? null }); + else { + // run_mode: is the server that answers (this process under `serve`, else the one in server.json) the service's process? + const serverPid = options.live ? process.pid : serverRunning ? serverStatePid : undefined; + const runMode = runModeOf(service, serverPid); + const linger = service.platform === "systemd" && service.installed ? await (options.linger ?? lingerStatus)() : undefined; + const data = { platform: service.platform, installed: service.installed, running: service.running, pid: service.pid ?? null, run_mode: runMode, ...(linger ? { linger } : {}) }; + const modeNote = runMode === "foreground" ? "; this server runs outside it (started by hand: it stops with its terminal)" : ""; + if (service.running && linger === "no") check("skillhook", "service", "warn", `${service.platform} running (pid ${service.pid}), but lingering is off: it stops when you log out${modeNote}`, `run: loginctl enable-linger ${userInfo().username}`, data); + else check("skillhook", "service", service.running ? "ok" : service.installed ? "warn" : "skip", service.running ? `${service.platform} running (pid ${service.pid})${modeNote}` : service.installed ? `${service.platform} installed but not running` : runMode === "foreground" ? "not installed; this server was started by hand and stops with its terminal" : "not installed", service.running ? undefined : "run: skillhook service install", data); + } } const summary = summarize(checks); diff --git a/src/service.test.ts b/src/service.test.ts index f5f8b43..4c50880 100644 --- a/src/service.test.ts +++ b/src/service.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { renderLaunchdPlist, renderSystemdUnit, serviceSpec } from "./service.js"; +import { parseLinger, renderLaunchdPlist, renderSystemdUnit, runModeOf, serviceSpec } from "./service.js"; import { tempHome } from "./test-support/helpers.js"; describe("service files", () => { @@ -35,3 +35,22 @@ describe("stableNodePath", () => { expect(["/opt/homebrew/bin/node", "/usr/local/bin/node", cellar]).toContain(stableNodePath(cellar)); }); }); + +describe("parseLinger", () => { + it("reads loginctl's yes/no and calls anything else unknown", () => { + expect(parseLinger(0, "yes\n")).toBe("yes"); + expect(parseLinger(0, "no\n")).toBe("no"); + expect(parseLinger(0, "")).toBe("unknown"); + expect(parseLinger(1, "no")).toBe("unknown"); + expect(parseLinger(0, "Failed to get user: No such user")).toBe("unknown"); + }); +}); + +describe("runModeOf", () => { + it("is service only when the service's own process is the server", () => { + expect(runModeOf({ running: true, pid: 42 }, 42)).toBe("service"); + expect(runModeOf({ running: true, pid: 42 }, 43)).toBe("foreground"); + expect(runModeOf({ running: false }, 43)).toBe("foreground"); + expect(runModeOf({ running: true, pid: 42 }, undefined)).toBe("unknown"); + }); +}); diff --git a/src/service.ts b/src/service.ts index cd52e29..32541e2 100644 --- a/src/service.ts +++ b/src/service.ts @@ -158,7 +158,9 @@ export async function installService(paths: Paths, label = SERVICE_LABEL): Promi const reload = await run("systemctl", ["--user", "daemon-reload"]); const enable = await run("systemctl", ["--user", "enable", "--now", label]); const output = `${reload.stderr}${enable.stdout}${enable.stderr}`.trim(); - return { ok: enable.code === 0, file, output: enable.code === 0 ? `${output}\nTip: run \`loginctl enable-linger ${spec.user}\` so the service starts without a login session.`.trim() : output }; + const linger = enable.code === 0 ? await lingerStatus(spec.user) : "unknown"; + const tip = linger === "yes" ? "" : `\n${linger === "no" ? "Warning: lingering is off, so this service stops when you log out." : "Tip:"} Run \`loginctl enable-linger ${spec.user}\` so the service keeps running without a login session.`; + return { ok: enable.code === 0, file, output: enable.code === 0 ? `${output}${tip}`.trim() : output }; } return { ok: false, file: "", output: `Unsupported platform ${process.platform}; run \`skillhook serve\` under your own supervisor.` }; } @@ -212,6 +214,33 @@ export async function serviceStatus(paths: Paths, label = SERVICE_LABEL): Promis return { platform: "unsupported", installed: false, running: false, file: "", logFile }; } +export type Linger = "yes" | "no" | "unknown"; + +/** Parse `loginctl show-user --property=Linger --value`: anything but a clean yes/no is "unknown". */ +export function parseLinger(code: number, stdout: string): Linger { + if (code !== 0) return "unknown"; + const value = stdout.trim().toLowerCase(); + return value === "yes" || value === "no" ? value : "unknown"; +} + +/** + * Whether systemd keeps this user's services running without a login session (read-only, never sudo). + * With Linger=no a `systemctl --user` service stops at logout, which the cloud only sees as link.stopped. + */ +export async function lingerStatus(user = userInfo().username): Promise { + if (process.platform !== "linux") return "unknown"; + const result = await run("loginctl", ["show-user", user, "--property=Linger", "--value"]).catch(() => ({ code: 1, stdout: "", stderr: "" })); + return parseLinger(result.code, result.stdout); +} + +/** How this server process runs: under the launchd/systemd service, or started by hand (a terminal, another supervisor). */ +export type RunMode = "service" | "foreground" | "unknown"; + +export function runModeOf(service: { running: boolean; pid?: number }, serverPid: number | undefined): RunMode { + if (!serverPid) return "unknown"; + return service.running && service.pid === serverPid ? "service" : "foreground"; +} + export function readServiceLog(paths: Paths, lines = 100): string { const file = path.join(paths.logsDir, "service.log"); if (!existsSync(file)) return "";