From 5971c2bbddc9530035f5f1b38b43c9eb9db635e2 Mon Sep 17 00:00:00 2001 From: zhangy <40223340+zy1233@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:37:58 +0800 Subject: [PATCH 1/4] feat(cli): send ACP usage_update for models with a declared context_window The agent sent no context use to a client, so the Context row of the VS Code extension had nothing to show. Plan A of docs/specs/2026-11-13-agent-context-usage.md is built. A model that has `context_window` (a whole number of tokens) in `[model]` or in a `[[models]]` entry of agent.toml gets `usage_update(used, size)`: - at each turn_end: the number of the provider plus an estimate for the tool results of the turn; - after a compaction: an estimate of the messages that remain (the kept messages still carry the usage of the context from before); - after session/load and session/resume: the count of the saved history, only when a provider number follows the last compaction; - after a change of model: the last count with the new window. Nothing is sent for a model with no window (the 128,000 of the core is a placeholder), for a provider that gives no usage, or for a turn that ended with error or aborted. There is no `cost`. The key is never inherited: a [[models]] entry has its own value or none. A value that cannot be used is dropped, and the agent says so at the start of the session. Compaction now uses the real window of a model that declares one, and its limits fit small windows (reserve_tokens and keep_recent_tokens are cut to a quarter and a third of the window when that is less). A model with no window does not change. 51 tests in packages/pi-agent-cli/tests/test_context_usage.py. --- docs/TUI-AND-CODE-AGENT.md | 1 + .../2026-11-04-vscode-extension-design.md | 10 +- docs/specs/2026-11-13-agent-context-usage.md | 61 +- packages/pi-agent-cli/AGENTS.md | 11 + packages/pi-agent-cli/agent.example.toml | 7 + packages/pi-agent-cli/pi_agent_cli/agent.py | 101 +++ packages/pi-agent-cli/pi_agent_cli/config.py | 59 +- .../pi_agent_cli/context_usage.py | 106 +++ packages/pi-agent-cli/pi_agent_cli/factory.py | 33 +- .../pi-agent-cli/tests/test_context_usage.py | 715 ++++++++++++++++++ 10 files changed, 1070 insertions(+), 34 deletions(-) create mode 100644 packages/pi-agent-cli/pi_agent_cli/context_usage.py create mode 100644 packages/pi-agent-cli/tests/test_context_usage.py diff --git a/docs/TUI-AND-CODE-AGENT.md b/docs/TUI-AND-CODE-AGENT.md index 2ceb270b..b3351535 100644 --- a/docs/TUI-AND-CODE-AGENT.md +++ b/docs/TUI-AND-CODE-AGENT.md @@ -117,6 +117,7 @@ provider = "deepseek" # or "openai", "anthropic" id = "deepseek-ai/DeepSeek-V4-Flash" api_key_env = "REAL_LLM_API_KEY" # env var holding the key for `provider` only base_url = "https://api.siliconflow.cn/v1" # for OpenAI-compatible gateways +context_window = 128000 # tokens; the agent then tells a client how full the context is (ACP `usage_update`) [skills] # ".pi/skills" is relative to the project, so it loads only for a trusted project (see [extensions]). diff --git a/docs/specs/2026-11-04-vscode-extension-design.md b/docs/specs/2026-11-04-vscode-extension-design.md index bdb3089d..e0c96846 100644 --- a/docs/specs/2026-11-04-vscode-extension-design.md +++ b/docs/specs/2026-11-04-vscode-extension-design.md @@ -17,7 +17,7 @@ - **Two kinds of trust.** VS Code Workspace Trust decides if the extension starts at all. The agent's own project trust decides if the project may load its extensions. The text of the two prompts says so. - **Publishing.** Open VSX first, then the Marketplace. The first upload to each (0.1.0, a pre-release) was a hand upload of a VSIX that the packaging script made. The publish workflow (`.github/workflows/vscode-extension.yml`, section 11) runs from `main` by `workflow_dispatch`, with Trusted Publishing and the environment `vscode-publish`; it publishes the stable releases, 1.0.0 first. - **Added in 1.0.0.** A first-run guide, `always-approve` in the mode menu (with a warning, never restored), a Past chats view (list, open, delete), and a Thinking menu. All four use what the bundled agent already has (0.1.0 bundled 0.9.0; 1.0.0 bundles **0.9.1**): no wire item and no agent change. The Thinking menu edits one line of `agent.toml` and restarts the agent, because 0.9.0 has no live setting (section 7c). -- **Added after 1.0.0.** Markdown in the chat (section 2, rule 5, and D34), and a richer status row (section 7d): the context window and the cost when the agent sends the standard `usage_update`, the time of the turn, and the folder. Agent 0.9.1 sends no `usage_update`, so the Context row does not show with the bundled agent; `docs/specs/2026-11-13-agent-context-usage.md` records what an agent change needs. The agent was not changed. +- **Added after 1.0.0.** Markdown in the chat (section 2, rule 5, and D34), and a richer status row (section 7d): the context window and the cost when the agent sends the standard `usage_update`, the time of the turn, and the folder. Agent 0.9.1 sends no `usage_update`, so the Context row does not show with the bundled agent. The agent change that sends it was built on 2026-11-16 (`docs/specs/2026-11-13-agent-context-usage.md`); it is not released, so the row shows after the next agent release and a new pin. - **Not yet.** Model choice, images, `@file`, MCP servers, fallback package, and everything of report section 3.3 from Phase 2 on (section 12). ## 1. Facts the design rests on @@ -350,7 +350,7 @@ Nothing else is needed, so there is **no wire item to add and no agent change**. | Time of the turn | the clock of the host, from `send` to the end of the turn | in the header after the state: `Working · 1 min 12 s`, then `Ready · last turn 8.4 s` until the next turn; a line of the tooltip | While a turn runs, and after it. | | Folder | `cwd` of the session | in the line of the agent and the model: `folder pi-python` (the full path is its tooltip); a line of the tooltip of the status bar item | When the chat has a folder. | -**Agent 0.9.1 sends no `usage_update`, so the Context row does not show with the bundled agent.** What was found, and what an agent change needs, is in `docs/specs/2026-11-13-agent-context-usage.md`. That record is the only work on the agent side: nothing in the agent was changed, and no pin was moved. +**Agent 0.9.1 sends no `usage_update`, so the Context row does not show with the bundled agent.** What was found is in `docs/specs/2026-11-13-agent-context-usage.md`. The agent change that sends `usage_update` (for a model that has `context_window` in `agent.toml`) was built later, on 2026-11-16. It is not released, and no pin was moved. **Design notes.** @@ -372,7 +372,7 @@ Nothing else is needed, so there is **no wire item to add and no agent change**. | The git branch | VS Code shows it in its own status bar. The agent does not send it. | | The thinking level and the mode | Already in the menus of the header. | -**Tested.** `test/status-row.test.ts`: the reader of `usage_update` and all the texts. `test/usage.test.ts`: the controller against the fake agent with a fake clock (an update; a newer one; a size of 0; an update that the SDK drops; a new chat, a restart and a stop; the clock while a turn runs, after it, in the next turn, and when the clock goes back). `test/webview.test.ts`, group "status row", in headless Chrome: the row with its numbers, the levels, the row with no size, the row that goes, text and not HTML, the running clock, a tick that leaves a menu alone, "last turn". **Not verified:** an agent that sends `usage_update` (none exists yet), and a real VS Code theme (the screenshots used the colours of the test page). +**Tested.** `test/status-row.test.ts`: the reader of `usage_update` and all the texts. `test/usage.test.ts`: the controller against the fake agent with a fake clock (an update; a newer one; a size of 0; an update that the SDK drops; a new chat, a restart and a stop; the clock while a turn runs, after it, in the next turn, and when the clock goes back). `test/webview.test.ts`, group "status row", in headless Chrome: the row with its numbers, the levels, the row with no size, the row that goes, text and not HTML, the running clock, a tick that leaves a menu alone, "last turn". **Not verified:** the extension against an agent that sends `usage_update`, in a real VS Code (the changed agent exists since 2026-11-16; a probe read its message on the wire, and the extension did not), and a real VS Code theme (the screenshots used the colours of the test page). ## 8. The diff view @@ -499,7 +499,7 @@ What this means for the package: From the report, section 3.3 (Phase 2 and later), and from the limits above: -- The Context row with the bundled agent. It needs an agent change: a window size for each model, and a `usage_update` after each turn. Section 7d and `docs/specs/2026-11-13-agent-context-usage.md` (record only: the agent was not changed). +- The Context row with the bundled agent. It needs the agent change that was built on 2026-11-16 (a `context_window` for each model, and a `usage_update` after each turn), an agent release, and a new pin. Section 7d and `docs/specs/2026-11-13-agent-context-usage.md`. - A live thinking level (no restart, a level for each chat, a list of levels for each model). It needs an agent change: a `thinking_level` config option on the session (the standard Session Config Options, like `model` and `mode`) that calls the harness `set_thinking_level` and is saved in the chat, and a way to say that a model has no thinking. Decision D28 (option C). - Past chats beyond the list of section 7b: search, rename, grouping by date, a list of the chats of other folders, `session/resume` (the extension uses `session/load`). - Model choice (`session/set_config_option` with `model`). Phase 1 shows the model name only. @@ -577,7 +577,7 @@ From the report, section 3.3 (Phase 2 and later), and from the limits above: | D33 (adopted) | **Recommended: as built.** The change is possible only while no answer runs. After the restart the same chat is opened with `session/load` (history replayed); a chat with no message gets a new chat; Always approve is off again, and a notice says so. | A restart ends a running turn. `session/load` keeps the history of the chat (D21 to D24). Always approve is never restored (D19). | | D34 (adopted) | **Option A, chosen by the user.** The chat shows Markdown with `markdown-it` 15.0.3 and `markdown-it-cjk-friendly` 3.0.0, both bundled into `dist/webview.js`. The parser only reads. Its HTML option is off, and the webview makes DOM nodes from plain data, never from an HTML string (section 2, rule 5). Options not taken: **B**, a longer hand-written reader (no dependency, but each new mark is more code, and the rule for `**` next to Chinese, Japanese and Korean characters is hard to write); **C**, the `markdown.api.render` command of VS Code (it returns an HTML string, so the webview would need `innerHTML` and rule 5 would break). | The reader of 1.0.0 made paragraphs, lists, code and links only. In 47 session files of the author (131 answer texts, a small sample), 41 answers (31 %) had a heading, a table, bold text, a quote, a rule or a nested list, and the chat showed the marks as text. `markdown-it` is the reference parser of CommonMark and has its own test suite. Plain CommonMark does not close `**重要:**文本` (3 of 10 patterns that we tried stayed raw), and the plugin fixes it. Cost: the webview bundle grows from 11 KB to 118 KB, and `THIRD-PARTY-NOTICES` lists 8 more packages. D3 stays true: the webview is a plain TypeScript bundle with no UI framework, and the new dependency is a parser. | | D35 (built) | The webview draws the text of a streamed answer at most once each 80 ms. The first piece is drawn at once. The pieces that come in a pause are joined into one draw at the end of the pause, counted from the end of the last draw. The last piece is always drawn. A reset of the chat cancels a draw that is planned. | A draw makes the DOM of the whole answer again, so its cost grows with the length of the answer. We measured it in headless Chrome, with the text already N characters long (the median of 25 pieces of 4 characters, two runs): 7 ms at 3,000 characters, 16 ms at 8,000 and 37 ms at 20,000. The reader of 1.0.0 took 5, 12 and 26 ms, so the new reader is about 1.4 times slower for each draw. The host sends each piece at once. If a model sends 50 pieces each second (an assumption: we did not measure a real model), the time between two pieces is 20 ms, and a draw of 37 ms does not fit in it. With the pause there are at most 12 draws each second, so a draw of 37 ms uses less than half of the time of the page. A change that draws only the last block again is a later option, if 80 ms is not enough. | -| D36 (built) | **Recommended: as built.** The context and the cost in the status row come only from the standard `usage_update` of the agent. The extension counts no tokens, has no setting for the size of the window, and shows nothing when the agent sends nothing. Options not taken: **B**, an estimate that the extension makes (for example characters ÷ 4); **C**, a setting `zycode.contextWindow`; **D**, change the agent now (the user said: write down the investigation only). | Agent 0.9.1 does not know the real window (`Model.context_window` is 128,000 for every model, and no key sets it) and has no price (`cost.total` is 0 in all 21 messages that we read), so a number from the extension would be a guess that looks like a fact. The pager made the same decision on 2026-10-13: it shows no context and no cost, and no number is invented. The row shows by itself when an agent sends `usage_update`, with no change in the extension. The findings and the agent change that is needed are in `docs/specs/2026-11-13-agent-context-usage.md`. | +| D36 (built) | **Recommended: as built.** The context and the cost in the status row come only from the standard `usage_update` of the agent. The extension counts no tokens, has no setting for the size of the window, and shows nothing when the agent sends nothing. Options not taken: **B**, an estimate that the extension makes (for example characters ÷ 4); **C**, a setting `zycode.contextWindow`; **D**, change the agent now (the user said: write down the investigation only). | Agent 0.9.1 does not know the real window (`Model.context_window` is 128,000 for every model, and no key sets it) and has no price (`cost.total` is 0 in all 21 messages that we read), so a number from the extension would be a guess that looks like a fact. The pager made the same decision on 2026-10-13: it shows no context and no cost, and no number is invented. The row shows by itself when an agent sends `usage_update`, with no change in the extension. The findings are in `docs/specs/2026-11-13-agent-context-usage.md`. On 2026-11-16 the user asked for the agent change, and it was built (Plan A of that record). The extension still counts no tokens. | | D37 (built) | **Recommended: as built.** The time of the turn is measured by the clock of the host, and the folder comes from the session. Both show in the header (and in the tooltip of the status bar item). The page counts the running time with its own timer, and changes one element each 500 ms. Options not taken: a clock in the status bar item (it needs a timer in the extension host each second); the tokens of the last turn (the agent does not send them). | The agent sends no times, and the pager draws clocks of its own for the same reason (2026-10-13). A tick that changes one element leaves a menu that is open alone; a new `state` message for each second would close it. The host sends a fresh `elapsedMs` in each `state` message, so a clock that has drifted is set right at the next one. | | D38 (built) | **Recommended: as built.** The extension lives in `vscode/`, at the root of the repository, and not in `packages/pi-vscode/`. The workflow file `vscode-extension.yml` and the environment `vscode-publish` keep their names. Options not taken: **B**, keep `packages/pi-vscode/`; **C**, a longer name (`zycode-vscode/`, `extensions/vscode/`). | `packages/` holds the six Python packages. Each has a `pyproject.toml`, and the uv workspace and the pytest paths in the root `pyproject.toml` name them one by one. The extension is not Python and is in neither list. `web/` and `tui/` are at the root for the same reason, and each has its own CI, as the extension has (`vscode-extension.yml`). **The move itself does not change which CI runs for a change.** Before D39, `ci.yml` (Ruff and pytest) had no `paths` filter, so it ran for every pull request, also for one that only touches `web/`; D39 adds the filter, in a separate commit. What the move changed: `REPO_ROOT` in `scripts/lib.mjs`, the three `../../..` in the opt-in tests, and `parents[4]` in `test/support/fake_model.py` (the first commit of the move missed it; the review found it, because `npm run test:agent` failed 7 tests with `fake model exited with 1` until it was `parents[3]`; those tests are off in CI); the test that reads the workflow; `repository.directory`; and the workflow (8 `working-directory`, the artifact path, the trigger path, and three `../../vsix/` that have no `pi-vscode` in the text, so a search by name misses them; two of them are in the publish job, which does not run in a pull request). A test now checks all of these against the real layout (`points every path of the workflow at the real layout`). The two registries name the repository, the workflow file and the environment, and not a directory, so the trusted publishers stay valid. Old links to `packages/pi-vscode/...` on GitHub do not work any more. The README of 1.0.0 names the old folder as text only, with no link (the next release says `vscode`), and its CHANGELOG and walkthrough have no link into it. | | D39 (built) | **Recommended: as built.** `ci.yml` (Ruff and pytest) does not run for a change that touches only `vscode/`, `docs/` and `vscode-extension.yml`, with no `.py` file. It is a `paths` filter with `!` patterns, on `push` and on `pull_request`. Options not taken: a `paths-ignore` filter with `vscode/**` (it would also skip the three Python helpers in `vscode/`, which Ruff reads); also leaving out `web/**` and `tui/**` (the same idea, but not asked for, and not checked against their tests). | A change of the extension started the whole Python CI (Ruff, and pytest on three Python versions, about 15 minutes of runner time), because `ci.yml` had no filter. For a pull request that only changes the extension and its documents, no Python result can change. No test reads `vscode/` or `docs/`, `docs` is in `extend-exclude` of Ruff, and `docs/` has Markdown only. Ruff does read the three Python files in `vscode/` (`scripts/python_licenses.py`, `test/support/fake_model.py`, `test/support/pty_answers.py`), so the last pattern, `**/*.py`, puts every Python file back (GitHub reads the patterns in order, and a positive pattern after a `!` pattern puts a file back). The workflow still runs when any other file is in the change (for example `AGENTS.md`, `README.md`, or `ci.yml` itself). No check is required on `main` (the ruleset only forbids a deletion and a force push), so a skipped run blocks no merge. Not verified: the effect on GitHub. The pull request that adds the filter changes `ci.yml`, so it runs the workflow itself. The first pull request that touches only the extension shows the effect. | diff --git a/docs/specs/2026-11-13-agent-context-usage.md b/docs/specs/2026-11-13-agent-context-usage.md index 7ccb03e8..6101a123 100644 --- a/docs/specs/2026-11-13-agent-context-usage.md +++ b/docs/specs/2026-11-13-agent-context-usage.md @@ -1,8 +1,8 @@ # Context use and cost from the agent: investigation record -> Date: 2026-11-13. -> Status: **record only.** No agent file was changed. This record does not describe a feature that is built in the agent. It describes what was found, and what an agent change must do. -> Reason: the user asked for the context use in the status row of the VS Code extension. The user also said: if the agent must change, write down the result of the investigation, and do not make the change. +> Date: 2026-11-13 (investigation); 2026-11-16 (built). +> Status: **built (Plan A), not released.** Sections 1 to 3 are the investigation of 2026-11-13. They describe agent 0.9.1, and they are not changed. Section 4 says what the agent does now. +> Reason: the user asked for the context use in the status row of the VS Code extension. On 2026-11-13 the user said: if the agent must change, write down the result of the investigation, and do not make the change. On 2026-11-16 the user asked for the change, with Plan A (option A of section 4, item 1, in the first version of this record). > Related: `docs/specs/2026-11-04-vscode-extension-design.md` section 7d (the extension side, which is built); `docs/specs/2026-11-05-acp-extension-registry.md` section 5; the entry of 2026-10-13 in `docs/PLAN/archive/PLAN-RUST-AGENT-RUNTIME-REMOVAL-EXECUTION-LOG.md` (the pager had the same question). ## 1. Question @@ -11,7 +11,7 @@ Can the status row of the extension show how much of the context window is in us ## 2. Answer -**No.** Agent 0.9.1 sends no token count, no window size and no cost to a client. Three parts are missing: +**No, for agent 0.9.1.** It sends no token count, no window size and no cost to a client. Three parts were missing (section 4 says what the agent does now): 1. **The message.** The agent never sends the ACP message `usage_update` (section 3, rows 1 and 2). 2. **The window size.** The agent does not know it. No file and no setting gives it (row 4). @@ -33,35 +33,46 @@ The extension reads `usage_update` when an agent sends it (design section 7d). U | 6 | Where is the window used today? | Only for compaction. `should_compact` is true when the count is above `context_window - reserve_tokens` (128,000 - 16,384 = 111,616 today). `auto_compact` is off by default. `factory.py` turns it on when `max_turns >= 30`. | | 7 | Do other clients read `usage_update`? | The pager `zypi` does not: no Rust file under `tui/` names it. The pager reads `_meta.totalTokens`, and the agent does not send it (registry section 5). The decision of 2026-10-13 (PR #19) says the same: the pager shows no context and no cost, and no number is invented. | -## 4. What an agent change must do (a proposal: not built, not tested) - -1. **A source for the window size.** Add the key `context_window` to `[model]` and `[[models]]` (`ModelChoice`), and set `Model.context_window` from it in `model_for_choice`. For a model that has no value, there are four options: - - **A (recommended).** Send no `usage_update` for it. No number is a guess. - - **B.** A table of known models in the agent. Not recommended: it goes out of date, and it is wrong for gateways. - - **C.** Keep 128,000 and send it. Not recommended: the number is invented. - - **D.** Send `size: 0`. The extension reads it as "size not known" and shows the count only. It is not standard, and another client can divide by zero. -2. **The message.** At each `turn_end`, send `UsageUpdate(used, size)`. Build `acp.schema.UsageUpdate` directly. `project_event` (`events.py`) already turns harness events into ACP updates. A `turn_end` case can take `used` from `event.message.usage` (`calculate_context_tokens`) and add `estimate_message_tokens` for each of `event.tool_results`. No call to `build_context()` is needed. `project_event` needs the window size as an argument. -3. **After a compaction, and after `session/load`.** The count drops after a compaction, and `_maybe_auto_compact` runs after `turn_end`. Send a second update when the compaction ends. Without it, the row shows the old count until the next turn ends. After `session/load`, send one update when the replay ends. -4. **No `cost`.** Leave the field out until the agent has a price source. Do not send 0. -5. **Tests.** A test of the projection with a fake harness event. A test over ACP (a Python SDK client) that one prompt gives one `usage_update`. A test that a model with no `context_window` gives none. +## 4. What the agent does now (built 2026-11-16, Plan A) + +The change follows the proposal of the first version of this record. The differences are marked. The numbers are found in `packages/pi-agent-cli/pi_agent_cli/context_usage.py`. The sending is in `agent.py`. + +1. **The window size.** `[model]` and each `[[models]]` entry can have `context_window`: a whole number of tokens, from 1 to 1,000,000,000. This is option A. A model with no value gets no `usage_update`. The 128,000 of the core is never sent. + - A value that cannot be used (text, 0, a negative number, a boolean, a fraction, a number that is too large) is dropped. The agent says so at the start of a session, in the notes about settings that were read and not used. + - The value is never inherited. A `[[models]]` entry has its own value, or none. This is also true when it has the same provider as `[model]`. A window belongs to one model. + - `model_for_choice` sets `Model.context_window` only when the choice has a value. +2. **When the agent sends it.** At each `turn_end`, `PiAcpAgent._report_context_use` sends `UsageUpdate(used, size)`. `used` is the number of the provider for the assistant message (`calculate_context_tokens`), plus `estimate_message_tokens` for each tool result of the turn. The agent sends nothing when the provider gave no number: `usage` is 0 (a gateway), or the turn ended with `error` or `aborted`. + - *Difference.* The proposal put this in `project_event`. That function is pure and has no state. The new code keeps the last count of each session. So it is a method of the agent, and `project_event` is not changed. +3. **Other times.** + - *After a compaction.* The agent sends an estimate for the messages that remain (4 characters for 1 token). The kept messages still carry the `usage` of the context from before, so that `usage` is not used. The agent sends this only when it has sent a count for the session before. Otherwise there is no old number to correct. + - *After `session/load` and `session/resume`.* The agent sends the count of the saved history after the response has gone out (the setup that follows the response; see zed#60199), and before the question about the project. It sends only a number that is still true: an assistant message with a number of the provider must come after the last compaction. A history that has none gets no update, and the next turn gives one. + - *After a change of model* (`session/set_config_option`). The agent sends the last count again, with the window of the new model. When the new model has no window, it sends nothing. ACP has no message that clears the numbers, so the client keeps what it shows. +4. **No `cost`.** The field is left out. The agent has no price table. +5. **Compaction by the real window** (decision 3 in section 7: yes, in the same change). A model with a `context_window` has the real value in `Model.context_window`, and `should_compact` uses it. The limits of compaction fit the window (`factory.compaction_settings`): the defaults (16,384 tokens kept free, the last 20,000 tokens kept) are cut to a quarter and a third of the window when that is less. A window of 65,536 tokens or more keeps the defaults. Without this, a small window (8,192 or 32,000 tokens) would compact again after every turn. A change of model sets the limits again. A model with no window keeps the placeholder and the defaults, so nothing changes for it. +6. **Tests.** `packages/pi-agent-cli/tests/test_context_usage.py`. It tests the numbers (pure functions, with a number from before a compaction). It tests the configuration (reading, the key at the top of the file, no inheritance, bad values and their notes, `model_for_choice`, `compaction_settings`). It tests the agent in one process with a scripted model: one update for one turn, a tool turn, no window, no usage, a failed turn, a client that refuses the update, a change of model, `session/close`, `session/load`, `session/resume`, a compaction, and compaction by a declared window against the placeholder. Seven deliberate faults in the code (for example, a send of a window that is not declared) each made a test fail. +7. **Checked on the wire** (Windows, 2026-11-16). The probe started the real agent process (`python -m pi_agent_cli`) on stdio, with a fake OpenAI-format model server (`vscode/test/support/fake_model.py`), and read raw ACP JSON lines. One prompt gave `{"used": 5, "size": 128000, "sessionUpdate": "usage_update"}` once, after the answer, with no `cost`. `session/load` in a new process gave the same message after the replay. After a change to a model with no window, a prompt gave none. A second check ran the real agent with the real chat controller of the extension (`ChatController`). The test was not kept. With `context_window = 128000`, the state of the controller had `used` 5 and `size` 128000 after one prompt. With no `context_window`, it had no usage. ## 5. Risks and side effects -- **Compaction changes.** A real window changes `should_compact` for each session that has `auto_compact` on. A model with a window under 128,000 compacts earlier (this is correct). A model with a larger window compacts later. This is a change of behaviour. It needs its own decision and its own test. -- **The count is partly an estimate.** A tool result that the model has not read yet is an estimate. The count is exact after the next model call. -- **Gateways.** A gateway that sends no usage gives a count that is only an estimate. -- **One client.** Only the extension reads `usage_update` today. To show it in `zypi`, the pager needs a Rust change. +- **Compaction changes (on purpose).** For a model with a `context_window`, `should_compact` uses the real window, in each session that has `auto_compact` on (`max_turns >= 30`; `agent.example.toml` sets 50). A model with a window under 128,000 compacts earlier (this is correct). A model with a larger window compacts later. A model with no `context_window` does not change. Test: `test_compaction_uses_the_window_that_the_user_declared`. +- **The count is partly an estimate.** A tool result that the model has not read yet is an estimate. The count after a compaction is an estimate for the messages only: it has no system prompt and no tool descriptions, and 4 characters for 1 token is too low for Chinese, Japanese and Korean text. The next model call gives an exact count. +- **Gateways.** A gateway that sends no usage gives no update. The agent does not show an estimate that has no number of a provider behind it. +- **A window that is too small.** A user who writes a window under the real one can see more than 100 %. The agent sends the number as it is. +- **A change to a model with no window.** The client keeps the old numbers (section 4, item 3). +- **Clients.** Only the extension shows `usage_update`. `zypi` reads the message and ignores it. Its crate `agent-client-protocol` 0.10.4 parses `usage_update` only with the feature `unstable`, and `tui/Cargo.toml` turns that feature on. A check of 2026-11-16 used a small Rust program with the same crate versions (from `tui/Cargo.lock`) and the message of the probe: the message parses, and the same program without the feature fails with `unknown variant usage_update`. After the parse, the pager has a catch-all arm for the variants that it does not use (`acp/tracker.rs`), so it does nothing with the update. Do not remove that feature without a new check. To show the update in `zypi`, the pager needs a Rust change. A client that cannot read `usage_update` gets it only when the user wrote a `context_window`. ## 6. Release path An agent change is not enough. The extension bundles one fixed agent binary. After the change: release the agent, then change `zycodeAgentVersion` and the size and the sha256 in `vscode/agent-assets.json`, then release the extension. A user who sets `zycode.agentPath` gets the change as soon as that program is new. -## 7. Open decisions (for the owner) +At the extension release, change the sentences that say that the bundled agent sends no `usage_update`: `vscode/README.md` (the Context row), `vscode/CHANGELOG.md` and design section 7d. They are true for agent 0.9.1 only. + +## 7. Decisions -1. Change the agent? When? -2. The source of the window size (section 4, item 1: A to D). -3. Compaction by the real window: yes or no, in the same change? -4. Teach the pager to read `usage_update`? +1. Change the agent? **Yes** (the user, 2026-11-16). It is built, and it is not released. +2. The source of the window size: **A** (section 4, item 1). +3. Compaction by the real window, in the same change? **Yes.** It follows from setting `Model.context_window`. It has its own test, and a guard for small windows (section 4, item 5). +4. Teach the pager to read `usage_update`? **Open.** Not decided, not built. ## 8. What the extension does today @@ -71,4 +82,4 @@ It does what needs no agent change (design section 7d): - It measures the time of the turn with its own clock. - It shows the folder of the chat. -With agent 0.9.1 the Context row does not show. +With agent 0.9.1, which the extension bundles now, the Context row does not show. With an agent that has the change in section 4, it shows for a model that has a `context_window`. The extension reads the message in tests with a fake agent. Its chat controller ran once against the changed agent (section 4, item 7). The extension has not been run against the changed agent in a real VS Code. diff --git a/packages/pi-agent-cli/AGENTS.md b/packages/pi-agent-cli/AGENTS.md index 66f3c873..f80a7859 100644 --- a/packages/pi-agent-cli/AGENTS.md +++ b/packages/pi-agent-cli/AGENTS.md @@ -36,6 +36,16 @@ A home with no `agent.toml` has no model and no key. The configuration then fall Standard ACP **Session Config Options** — `session/set_model` is gone upstream (SDK ≥ 0.11). `new_session` / `load_session` / `resume_session` return `configOptions` with one select (`id="model"`, `category="model"`, `currentValue`, `options`); the client switches with `session/set_config_option` (`PiAcpAgent.set_config_option` → `harness.set_model`, response carries the full option set). Choices = `[model]` + optional `[[models]]` tables in `agent.toml` (`ModelChoice`, `CliConfig.model_choices()`). The model is persisted in the session (`model_change`) and restored on load/resume if still configured. Legacy `_meta` hints (`pi/currentModelId`, `pi/currentModelDisplayName`, `pi/provider`) are still emitted for older clients. Each choice carries its own `api_key_env`: `get_api_key` answers only for that choice's provider (`factory.api_key_for_choice`), and `Model.reasoning` follows `CliConfig.model_reasoning` whichever choice is active. +## Context use (`usage_update`) + +The agent tells a client how full the context window is, with the standard ACP session update `usage_update` (`used` and `size`; no `cost`). The numbers are in `context_usage.py`. The sending is in `PiAcpAgent._report_context_use` and the methods near it. Record: `docs/specs/2026-11-13-agent-context-usage.md`. + +- **The window comes from the user.** `[model]` and each `[[models]]` entry can have `context_window` (a whole number of tokens). It is never inherited: a window belongs to one model. A model with no value gets no `usage_update`. The 128,000 of the core (`Model.context_window`) is a placeholder, and it is never sent. A value that cannot be used is dropped, and a note says so (`auto.notes`). +- **When it is sent.** At each `turn_end` (the number of the provider for the assistant message, plus an estimate for the tool results of the turn). After a compaction (an estimate for the messages that remain, only for a session that has a count already). After `session/load` and `session/resume` (the saved history, only a number from after the last compaction). After a change of model (the same count with the new window; no window, nothing). A provider that gives no usage, and a turn that ends with `error` or `aborted`, give no update. +- **A send never raises.** The harness turns the error of a subscriber into an error of the turn, so every send is guarded. +- **Compaction uses the same window.** A model with a `context_window` compacts by it. `factory.compaction_settings` cuts `reserve_tokens` and `keep_recent_tokens` for a small window (a quarter and a third of it), so that a small window does not compact after every turn. +- **Clients.** The VS Code extension shows the update (a Context row). `zypi` reads it and ignores it. It can read it only because its ACP crate has the feature `unstable` (`tui/Cargo.toml`): keep that feature. + ## Project trust A project's own content is used only if the project is trusted: `/.pi-python/extensions` (imported — i.e. executed — when a session opens), `/.pi-python/workflows` (each `*.py` becomes a slash command with a description the repository wrote, and a command that has the model run the script), `/.pi/SYSTEM.md`, `/.pi/APPEND_SYSTEM.md`, and `[skills].paths` entries relative to the project (`.pi/skills`) — `trust_fingerprint.gated_project_resources` lists them. One more thing is gated that is not a file of the project: the `` block (`prompt_options._git_status_block`). `git status` runs the program that `core.fsmonitor` in the repository's own `.git/config` names, so a project that ships its `.git` could run code at session start. The block is taken only when the session's `ProjectTrust` says yes. The trust reason `nothing` (the project ships nothing gated) counts as untrusted, so **by default there is no block**; the user gets one with `--trust-project-extensions`, `default_project_trust = "always"`, `[extensions] trusted_projects` or a saved answer. It is not in `gated_project_resources` (nothing to fingerprint or to ask about), so no question is asked for it and the "left out" notice does not name it. The model can still run `git status` through `bash` (a tool call, so permissions apply). One tool call is gated on trust as well: `workflow` with `isolation=true` (`pi-dynamic-workflows`, `worktree.py`) runs `git worktree add`, `git add`, `git diff` and `git apply` in the project, which run the `post-checkout` hook, the filters, `core.fsmonitor` and `diff.external` that the repository names. In a project that is not trusted the call is refused (no run, no silent fallback to no isolation); the extension reads `pi.project_trusted`. Never read the decision from a file inside the project, or a repository could trust itself. Not gated: entry-point packages, `~/.pi-python/extensions`, `/workflows` (the user's own), `AGENTS.md` / `CLAUDE.md` (upstream loads them whatever the trust), prompts set in `agent.toml`, and absolute or `~` skills paths. The workflows are read by the dynamic-workflows extension, not by the CLI: it asks `pi.project_trusted` (the harness's `trust_project_extensions`, final by the time extensions activate), so an extension that reads anything under `/.pi-python/` must do the same. Design: `docs/specs/2026-09-22-phase7-extension-api-design.md` §4.4. @@ -197,6 +207,7 @@ The agent must reap the tools it is running when its client goes away: the TUI c | `git_context.py` | Read-only branch and `git status` snapshot for the system prompt | | `headless.py` | `-p` one-shot mode, prompt override application | | `events.py` | Internal events → ACP `session_update` projection | +| `context_usage.py` | The numbers of the ACP `usage_update`: the message, the count at the end of a turn, after a compaction and for a saved session (see Context use) | | `permissions.py` | Tool permission gating (ask / auto / always-approve): `needs_permission` (a tool that declares itself harmless is never asked about, in `ask` and in `auto`), the title of a question (it holds the reason when Auto asks), the allowed outcome (see Tool permissions) | | `auto_rules.py` | The Auto rules: `decide` / `decide_rules_only` over a `ToolCall` and an `AutoContext` (`build_context`); the floor, the allow rules, the `bash` parser, the barred forms (D-07), `DEFAULT_PROTECTED_GLOBS`. No model, no session state (see Auto mode) | | `auto_classifier.py` | The model step of Auto: the policy text, the answer parser, the data block, `ModelClassifier` (timeout, cancel, circuit breaker, counters) and `UserMessageLog`. It calls no tool and has no session state beyond the breaker | diff --git a/packages/pi-agent-cli/agent.example.toml b/packages/pi-agent-cli/agent.example.toml index 97f7db74..d1eb882a 100644 --- a/packages/pi-agent-cli/agent.example.toml +++ b/packages/pi-agent-cli/agent.example.toml @@ -35,13 +35,20 @@ api_key_env = "REAL_LLM_API_KEY" # key for `provider` only; other providers use # Can the model reason? A thinking_level other than "off" is sent to the provider only if so. # Unset: any level but "off" says it can. `false` keeps a model that rejects reasoning parameters out of them. # reasoning = true +# The context window of the model, in tokens (a whole number). With it the agent tells a client how +# full the context is (ACP `usage_update`; the VS Code extension shows it as a Context row), and +# compaction (`max_turns` of 30 or more) uses it. Unset: the agent sends no context size and does not +# guess one. Use the real window of your model: a wrong number gives a wrong percent. +# context_window = 128000 # Extra models for the TUI `/model` picker (ACP session config option "model"). # `[model]` above is always offered first; unset fields inherit from it, but # base_url / api_key_env / supports_images only when `provider` is unset or the same. +# `context_window` is never inherited: a window belongs to one model, so set it in each entry. # [[models]] # id = "deepseek-ai/DeepSeek-V4-Pro" # name = "DeepSeek V4 Pro" +# context_window = 128000 # # [[models]] # provider = "openai" diff --git a/packages/pi-agent-cli/pi_agent_cli/agent.py b/packages/pi-agent-cli/pi_agent_cli/agent.py index 5d6e0eb4..ac4c38af 100644 --- a/packages/pi-agent-cli/pi_agent_cli/agent.py +++ b/packages/pi-agent-cli/pi_agent_cli/agent.py @@ -81,6 +81,12 @@ load_local_env, pi_home, ) +from pi_agent_cli.context_usage import ( + estimated_context_tokens, + stored_context_tokens, + turn_context_tokens, + usage_update, +) from pi_agent_cli.events import project_event, project_message_replay from pi_agent_cli.extension_notices import failed_extensions_notice from pi_agent_cli.extension_trust import ( @@ -95,6 +101,7 @@ ) from pi_agent_cli.factory import ( api_key_for_choice, + compaction_settings, create_session_harness, default_stream_fn, load_session_resources, @@ -221,6 +228,10 @@ def __init__( self._abort_tasks: set[asyncio.Task[Any]] = set() self._session_cwds: dict[str, str] = {} self._session_models: dict[str, ModelChoice] = {} + # The tokens in the context of each open session, at the last point where the agent + # knew the number (``_report_context_use``). It is kept so that a change of model can + # send it again with the window of the new model. + self._context_used: dict[str, int] = {} self._client_capabilities: ClientCapabilities | None = None self._client_info: Implementation | None = None self._extensions: list[Any] = extensions or [] @@ -444,6 +455,7 @@ async def resume_session( def _release_session(self, session_id: str) -> AgentHarness | None: """Forget the in-memory state of *session_id* and return its harness, if any.""" self._session_models.pop(session_id, None) + self._context_used.pop(session_id, None) self._session_cwds.pop(session_id, None) self._reserved_ids.discard(session_id) harness = self._harnesses.pop(session_id, None) @@ -563,8 +575,15 @@ async def set_config_option( {"configId": config_id, "value": value, "reason": "unknown model"} ) await harness.set_model(model_for_choice(choice, reasoning=self._config.model_reasoning)) + # The limits of compaction depend on the window of the model (``compaction_settings``). + harness.compaction = compaction_settings( + choice, auto_compact=harness.compaction.auto_compact + ) harness.get_api_key = api_key_for_choice(choice) self._session_models[session_id] = choice + # The count stays true, but it is now a part of another window. When the new model has + # no window, nothing is sent (the client keeps what it shows). + await self._send_context_use(session_id) return SetSessionConfigOptionResponse(config_options=self._config_options(session_id)) async def set_session_mode( @@ -958,6 +977,7 @@ async def on_tool_call(event: Any) -> dict[str, Any] | None: async def on_event(event: Any, signal: Any | None = None) -> None: await self._emit_updates(session_id, event) + await self._report_context_use(session_id, event) harness.subscribe(on_event) self._drop_trust(session_id) # binding again: a question still open is for the old one @@ -972,6 +992,7 @@ async def on_event(event: Any, signal: Any | None = None) -> None: self._session_modes[session_id] = mode self._harnesses[session_id] = harness self._session_cwds[session_id] = cwd + self._context_used.pop(session_id, None) # the number of an earlier harness is gone self._trust[session_id] = state if replaced is not None and replaced is not harness: await self._end_replaced_harness(session_id, replaced) @@ -1024,6 +1045,8 @@ async def _deferred_session_setup(self, session_id: str) -> None: to the client is no different. """ await asyncio.sleep(0) + # Before the question about the project: that one can wait for the user for a long time. + await self._report_stored_context_use(session_id) await self._settle_project_trust(session_id) await self._advertise_commands(session_id) # The client has taken an update for the session, so it knows the session: the lines @@ -1351,6 +1374,84 @@ async def _emit_updates(self, session_id: str, event: Any) -> None: for update in project_event(event, cwd=cwd): await self._conn.session_update(session_id=session_id, update=update) + async def _report_context_use(self, session_id: str, event: Any) -> None: + """Tell the client how full the context is, when an event changes the number. + + The number changes at the end of a turn (the provider counted the context) and after a + compaction (the count falls). This never raises: the harness turns the error of a + subscriber into an error of the turn, and the count is only an aid for the user. + See ``context_usage`` for what is counted and when nothing is sent. + """ + kind = getattr(event, "type", None) + if self._conn is None or kind not in ("turn_end", "session_compact"): + return + try: + if kind == "turn_end": + used = turn_context_tokens(event.message, event.tool_results) + else: + used = await self._context_after_compaction(session_id) + if used is None: + return + self._context_used[session_id] = used + await self._send_context_use(session_id) + except Exception: + logger.warning( + "Could not report the context use of session %s", session_id, exc_info=True + ) + + async def _context_after_compaction(self, session_id: str) -> int | None: + """The count after a compaction: an estimate for the messages that remain. + + Only a session that has a count has something to correct. A session with none shows + no number, and an estimate alone is not sent (``context_usage``). + """ + harness = self._harnesses.get(session_id) + if harness is None or session_id not in self._context_used: + return None + context = await harness.session.build_context() + return estimated_context_tokens(context.messages) + + async def _report_stored_context_use(self, session_id: str) -> None: + """After ``session/load`` and ``session/resume``: the count of the saved history. + + Nothing is sent for a model with no window, and nothing for a history with no number + that is still true (``context_usage.stored_context_tokens``). Never raises. + """ + harness = self._harnesses.get(session_id) + if ( + self._conn is None + or harness is None + or self._current_choice(session_id).context_window is None + ): + return + try: + used = stored_context_tokens(await harness.session.get_branch()) + except Exception: + logger.warning( + "Could not read the context use of session %s", session_id, exc_info=True + ) + return + if used is not None: + self._context_used[session_id] = used + await self._send_context_use(session_id) + + async def _send_context_use(self, session_id: str) -> None: + """Send ``usage_update`` for a session, when it has a count and its model has a window. + + A model with no ``context_window`` in ``agent.toml`` gets none: the default of the core + (128,000) is a placeholder, and the agent does not guess a size. Never raises. + """ + used = self._context_used.get(session_id) + size = self._current_choice(session_id).context_window + if self._conn is None or used is None or size is None: + return + try: + await self._conn.session_update(session_id=session_id, update=usage_update(used, size)) + except Exception: + logger.warning( + "Could not send the context use of session %s", session_id, exc_info=True + ) + async def _handle_tool_call(self, session_id: str, event: Any) -> dict[str, Any] | None: name = event.toolName mode = self._session_mode(session_id) diff --git a/packages/pi-agent-cli/pi_agent_cli/config.py b/packages/pi-agent-cli/pi_agent_cli/config.py index 33d64148..6e8d3c86 100644 --- a/packages/pi-agent-cli/pi_agent_cli/config.py +++ b/packages/pi-agent-cli/pi_agent_cli/config.py @@ -131,6 +131,8 @@ class ModelChoice: In ``[[models]]`` entries unset fields inherit from the default ``[model]`` table, but ``base_url`` / ``api_key_env`` / ``supports_images`` are only inherited when ``provider`` is unset or equals the default provider (credentials never leak across providers). + ``context_window`` is never inherited: the size of a context window belongs to one model, + and another model of the same provider can have another size. """ id: str @@ -139,8 +141,16 @@ class ModelChoice: base_url: str | None = None api_key_env: str | None = None supports_images: bool | None = None + # The context window of the model, in tokens, as the user wrote it. ``None``: not written. + # Only a model with a value gets a ``usage_update`` (``context_usage``), and only its value + # reaches ``Model.context_window`` (``factory.model_for_choice``). + context_window: int | None = None +# The largest ``context_window`` that is accepted. It is far above any model that exists. A +# value above it is a typing error, and a client could not read it as a size. +MAX_CONTEXT_WINDOW = 1_000_000_000 + AutoClassifier = Literal["rules", "model"] # What a ``bash`` call that Auto let through is held to (``[auto] bash_timeout_seconds`` and @@ -193,6 +203,9 @@ class CliConfig: max_turns: int | None = None api_key_env: str | None = None supports_images: bool | None = None + # ``[model] context_window``: the window of the default model, in tokens. ``None``: not set + # (see ``ModelChoice.context_window``). + context_window: int | None = None # Whether the model can reason (``Model.reasoning``). ``None``: not set, and then asking # for a thinking level is taken as saying so (``model_reasoning``). reasoning: bool | None = None @@ -238,6 +251,7 @@ def default_choice(self) -> ModelChoice: base_url=self.base_url, api_key_env=self.api_key_env, supports_images=self.supports_images, + context_window=self.context_window, ) def model_choices(self) -> tuple[ModelChoice, ...]: @@ -263,6 +277,8 @@ def model_choices(self) -> tuple[ModelChoice, ...]: supports_images=item.supports_images if item.supports_images is not None or not same_provider else self.supports_images, + # Not inherited, not even from the same provider: see ``ModelChoice``. + context_window=item.context_window, ) ) return tuple(out) @@ -375,6 +391,20 @@ def _from_toml(data: dict[str, Any]) -> CliConfig: supports_images_raw = data.get("supports_images") supports_images = bool(supports_images_raw) if supports_images_raw is not None else None + # A ``context_window`` that cannot be used is dropped, and the user is told (``auto.notes``). + window_notes: list[str] = [] + + def _window(value: object, where: str) -> int | None: + window, note = _context_window(value, where) + if note is not None: + window_notes.append(note) + return window + + context_window_raw = model.get("context_window") + if context_window_raw is None: + context_window_raw = data.get("context_window") + context_window = _window(context_window_raw, "[model]") + reasoning_raw = model.get("reasoning") if reasoning_raw is None: reasoning_raw = data.get("reasoning") @@ -413,12 +443,16 @@ def _optional_str(value: object) -> str | None: base_url=_optional_str(item.get("base_url")), api_key_env=_optional_str(item.get("api_key_env")), supports_images=bool(images) if images is not None else None, + context_window=_window( + item.get("context_window"), f'the "{model_id}" entry of [[models]]' + ), ) ) auto = _auto_config(data.get("auto")) - if permission_note is not None: - auto = replace(auto, notes=(permission_note, *auto.notes)) + leading_notes = [note for note in (permission_note, *window_notes) if note is not None] + if leading_notes: + auto = replace(auto, notes=(*leading_notes, *auto.notes)) config = CliConfig( permission=permission, # type: ignore[arg-type] @@ -429,6 +463,7 @@ def _optional_str(value: object) -> str | None: max_turns=max_turns, api_key_env=model.get("api_key_env") or data.get("api_key_env"), supports_images=supports_images, + context_window=context_window, reasoning=reasoning, skills_dirs=skills_dirs, agent_command=agent_command, @@ -600,6 +635,26 @@ def _default_project_trust(value: object) -> DefaultProjectTrust | None: return text if text in _VALID_DEFAULT_TRUST else None # type: ignore[return-value] +def _context_window(value: object, where: str) -> tuple[int | None, str | None]: + """The ``context_window`` of one model: ``(tokens, None)``, or ``(None, note)``. + + The value is a whole number from 1 to ``MAX_CONTEXT_WINDOW``. A key that is not there gives + ``(None, None)``. Any other value cannot be used: it is dropped, and the note says so. The + agent then sends no context size for that model, and it does not guess one. A TOML float + with no fraction (``1.28e5``) is a whole number; a boolean is not. + """ + if value is None: + return None, None + if isinstance(value, float) and value.is_integer(): + value = int(value) + if isinstance(value, int) and not isinstance(value, bool) and 0 < value <= MAX_CONTEXT_WINDOW: + return value, None + return None, ( + f"context_window of {where} must be a whole number of tokens from 1 to " + f"{MAX_CONTEXT_WINDOW}. The agent ignores it and sends no context size for that model." + ) + + def _as_bool(value: object, default: bool) -> bool: if value is None: return default diff --git a/packages/pi-agent-cli/pi_agent_cli/context_usage.py b/packages/pi-agent-cli/pi_agent_cli/context_usage.py new file mode 100644 index 00000000..a4e933b7 --- /dev/null +++ b/packages/pi-agent-cli/pi_agent_cli/context_usage.py @@ -0,0 +1,106 @@ +"""The context use that the agent tells a client: the ACP ``usage_update``. + +A client can show how full the context window of the model is (the VS Code extension has a +Context row for it). ACP has one session update for this, ``usage_update``, with two numbers: +``used`` (the tokens in the context now) and ``size`` (the tokens that the window holds). This +module has the pure part: how the two numbers are found. ``agent.PiAcpAgent`` has the part that +sends them. The record is ``docs/specs/2026-11-13-agent-context-usage.md``. + +Two rules decide what is never sent: + +* **No window, no update.** ``size`` is the ``context_window`` that the user wrote for the model + (``config.ModelChoice``). The core has a default of 128,000 tokens. That default is a + placeholder, not a fact about the model, so it is never sent. +* **No provider number, no update.** ``used`` starts from the ``usage`` that the provider gave + with an assistant message. A gateway can give none, and a failed or cancelled turn has none + that can be trusted. Then the agent says nothing. A count that is only an estimate can be far + from the truth (the estimate is 4 characters for 1 token, which is too low for Chinese, + Japanese and Korean text). + +One case has no provider number and still gets an update: a compaction. The count falls, and the +old number would stay on the screen until the next turn ends. After a compaction the number is an +estimate of the messages that remain (``estimated_context_tokens``). The next turn replaces it +with a number of the provider. + +``cost`` is never sent: the agent has no price table. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from acp.schema import UsageUpdate + +from pi_agent_core.messages import AssistantMessage +from pi_agent_harness.compaction import ( + calculate_context_tokens, + estimate_context_tokens, + estimate_message_tokens, +) +from pi_agent_harness.session.session import build_session_context +from pi_agent_harness.types import CompactionEntry, SessionTreeEntry + +# A turn that ended with one of these has no usage that can be trusted (the harness ignores +# such a message when it counts the context for compaction, too). +_NO_NUMBER = ("error", "aborted") + + +def usage_update(used: int, size: int) -> UsageUpdate: + """The ACP ``usage_update`` for ``used`` tokens in a window of ``size`` tokens. + + ``used`` can be more than ``size``: the user wrote a window that is too small, or the + context is over the limit. The number is sent as it is. + """ + return UsageUpdate(session_update="usage_update", used=used, size=size) + + +def _provider_tokens(message: object) -> int: + """The tokens that a provider reported with an assistant message, or 0 when there are none.""" + if not isinstance(message, AssistantMessage) or message.stopReason in _NO_NUMBER: + return 0 + return calculate_context_tokens(message.usage) + + +def turn_context_tokens(message: object, tool_results: Sequence[Any] = ()) -> int | None: + """The tokens in the context at the end of one turn, or ``None`` when there is no number. + + ``message`` is the assistant message of the turn. The provider counted everything that it + was sent, and the answer too. The tool results of the turn are not in that count yet: the + next request carries them. They are added as an estimate. + """ + reported = _provider_tokens(message) + if reported <= 0: + return None + return reported + sum(estimate_message_tokens(result) for result in tool_results) + + +def estimated_context_tokens(messages: Sequence[Any]) -> int: + """An estimate of the tokens in ``messages``, with no number of a provider. + + It is for the context after a compaction. The kept messages still carry the usage of the + time before the compaction, and that usage counts a context that no longer exists. + """ + return sum(estimate_message_tokens(message) for message in messages) + + +def stored_context_tokens(entries: Sequence[SessionTreeEntry]) -> int | None: + """The tokens in the context of a saved session, or ``None`` when there is no number. + + ``entries`` are the entries of the branch, from the root to the leaf. This is for + ``session/load`` and ``session/resume``. + + The number of a provider is true for the context that existed when the provider gave it. A + compaction ends that context. So a number counts only when an assistant message that carries + one comes after the last compaction. When there is none, there is nothing to say yet, and + the next turn gives the number. + """ + entries = list(entries) + start = 0 + for index, entry in enumerate(entries): + if isinstance(entry, CompactionEntry): + start = index + 1 + after_last_compaction = build_session_context(entries[start:]).messages + if not any(_provider_tokens(message) > 0 for message in after_last_compaction): + return None + return estimate_context_tokens(build_session_context(entries).messages) diff --git a/packages/pi-agent-cli/pi_agent_cli/factory.py b/packages/pi-agent-cli/pi_agent_cli/factory.py index d93284c5..53b3df9f 100644 --- a/packages/pi-agent-cli/pi_agent_cli/factory.py +++ b/packages/pi-agent-cli/pi_agent_cli/factory.py @@ -57,15 +57,44 @@ def model_for_choice(choice: ModelChoice, *, reasoning: bool = False) -> Model: The choice is the ``[model]`` default or a ``[[models]]`` entry. ``reasoning`` (whether the model can think, ``CliConfig.model_reasoning``) is a property of the whole session configuration, so every choice gets the same value. + + ``Model.context_window`` is set only when the choice has a ``context_window``. A choice + with none keeps the default of the core (128,000), which is a placeholder: compaction uses + it, and ``context_usage`` never sends it to a client. """ provider = choice.provider or "mock" - return Model( + model = Model( provider=provider, model_id=choice.id, base_url=choice.base_url, supports_images=_detect_vlm_support(provider, choice.id, choice.supports_images), reasoning=reasoning, ) + if choice.context_window is not None: + model.context_window = choice.context_window + return model + + +def compaction_settings(choice: ModelChoice, *, auto_compact: bool) -> CompactionSettings: + """Compaction limits for ``choice``. + + The defaults of the harness keep 16,384 tokens free (``reserve_tokens``) and keep the last + 20,000 tokens as they are (``keep_recent_tokens``). They fit a window of about 64,000 + tokens or more. In a smaller window the limit for compaction falls below the part that is + kept, and compaction would run again after every turn. So for a choice with a declared + ``context_window`` the two numbers are cut to a share of the window (a quarter and a + third); a large window keeps the defaults. A choice with no window keeps the defaults: + its window is the placeholder of the core (``model_for_choice``). + """ + window = choice.context_window + if window is None: + return CompactionSettings(auto_compact=auto_compact) + defaults = CompactionSettings() + return CompactionSettings( + auto_compact=auto_compact, + reserve_tokens=max(1, min(defaults.reserve_tokens, window // 4)), + keep_recent_tokens=max(1, min(defaults.keep_recent_tokens, window // 3)), + ) def api_key_for_choice(choice: ModelChoice) -> Callable[[str], str | None] | None: @@ -279,7 +308,7 @@ async def system_prompt_callback(ctx: dict[str, Any]) -> str: system_prompt=system_prompt_callback, thinking_level=config.thinking_level, max_turns=config.max_turns, - compaction=CompactionSettings(auto_compact=auto_compact), + compaction=compaction_settings(choice, auto_compact=auto_compact), auto_discover_extensions=True, trust_project_extensions=( trust.trusted if trust is not None else project_extensions_trusted(config, cwd_s) diff --git a/packages/pi-agent-cli/tests/test_context_usage.py b/packages/pi-agent-cli/tests/test_context_usage.py new file mode 100644 index 00000000..622dd9f2 --- /dev/null +++ b/packages/pi-agent-cli/tests/test_context_usage.py @@ -0,0 +1,715 @@ +"""The context use that the agent tells a client: the ACP ``usage_update``. + +Record: ``docs/specs/2026-11-13-agent-context-usage.md``. The rules under test: + +* A model with no ``context_window`` in ``agent.toml`` gets no update (the 128,000 of the core is + a placeholder, and it is never sent). +* ``used`` is the number that the provider gave, plus an estimate for the tool results of the turn. + A provider that gives none, and a failed turn, give no update. +* After a compaction the count is an estimate of what remains. After ``session/load`` and + ``session/resume`` the count is read from the saved history, but only a number that is still + true: a number from before a compaction is not. +* A window that is declared reaches compaction, and the limits of compaction fit small windows. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest +from acp import text_block +from acp.schema import UsageUpdate +from pydantic import BaseModel + +from pi_agent_cli.agent import PiAcpAgent +from pi_agent_cli.config import MAX_CONTEXT_WINDOW, CliConfig, ModelChoice, load_config +from pi_agent_cli.context_usage import ( + estimated_context_tokens, + stored_context_tokens, + turn_context_tokens, + usage_update, +) +from pi_agent_cli.extension_trust import TRUST_ENV +from pi_agent_cli.factory import compaction_settings, model_for_choice +from pi_agent_core.event_stream import AssistantMessageEventStream +from pi_agent_core.extensions import ExtensionLoader +from pi_agent_core.messages import AssistantMessage, ToolResultMessage, Usage, UserMessage +from pi_agent_core.tests.mock_stream import _base_partial, mock_text_stream +from pi_agent_core.types import DoneEvent, ErrorEvent, Model, StartEvent, TextDeltaEvent +from pi_agent_harness.compaction import estimate_message_tokens +from pi_agent_harness.compaction.compaction import COMPACTION_SYSTEM_PROMPT +from pi_agent_harness.types import CompactionEntry, MessageEntry + + +@pytest.fixture(autouse=True) +def _isolated(monkeypatch: pytest.MonkeyPatch) -> None: + # What is installed or set on this machine must not add a tool or a notice of its own. + monkeypatch.delenv(TRUST_ENV, raising=False) + monkeypatch.setattr(ExtensionLoader, "discover_entry_points", lambda self: []) + + +# --- The pure part ----------------------------------------------------------------------- + + +def _assistant(total: int, *, stop: str = "stop", text: str = "answer") -> AssistantMessage: + return AssistantMessage( + content=[{"type": "text", "text": text}], + usage=Usage(input=total, totalTokens=total), + stopReason=stop, # type: ignore[arg-type] + timestamp=1, + ) + + +def _tool_result(text: str) -> ToolResultMessage: + return ToolResultMessage( + toolCallId="call_1", + toolName="read", + content=[{"type": "text", "text": text}], + timestamp=1, + ) + + +def test_the_update_has_the_two_numbers_and_no_cost(): + update = usage_update(1_200, 200_000) + + assert isinstance(update, UsageUpdate) + assert update.model_dump(by_alias=True, exclude_none=True) == { + "sessionUpdate": "usage_update", + "used": 1_200, + "size": 200_000, + } + + +def test_the_count_of_a_turn_is_the_number_of_the_provider(): + assert turn_context_tokens(_assistant(12_000)) == 12_000 + + +def test_the_tool_results_of_a_turn_are_added_as_an_estimate(): + result = _tool_result("x" * 400) # 4 characters for 1 token: 100 tokens + + assert estimate_message_tokens(result) == 100 + assert turn_context_tokens(_assistant(1_000, stop="toolUse"), [result]) == 1_100 + + +@pytest.mark.parametrize( + "message", + [ + _assistant(0), # a gateway that sends no usage + _assistant(5_000, stop="error"), # a failed turn + _assistant(5_000, stop="aborted"), # a cancelled turn + UserMessage(content="hello", timestamp=1), + None, + ], + ids=["no-usage", "error", "aborted", "not-an-assistant-message", "no-message"], +) +def test_a_turn_without_a_trusted_number_has_no_count(message): + assert turn_context_tokens(message, [_tool_result("x" * 400)]) is None + + +def test_the_estimate_after_a_compaction_ignores_the_usage_of_the_messages(): + messages = [ + _assistant(90_000, text="x" * 40), # carries the usage of a context that is gone + _tool_result("y" * 80), + ] + + assert estimated_context_tokens(messages) == 10 + 20 + assert estimated_context_tokens([]) == 0 + + +_STAMP = "2026-11-16T00:00:00Z" + + +def _entry(entry_id: str, parent: str | None, message: BaseModel) -> MessageEntry: + return MessageEntry( + id=entry_id, parentId=parent, timestamp=_STAMP, message=message.model_dump(mode="json") + ) + + +def _user(text: str = "hello") -> UserMessage: + return UserMessage(content=text, timestamp=1) + + +def test_the_count_of_a_saved_session_is_the_last_number_plus_what_follows(): + entries = [ + _entry("e1", None, _user()), + _entry("e2", "e1", _assistant(10_000)), + _entry("e3", "e2", _user("y" * 40)), # 10 tokens, and the provider has not seen them + ] + + assert stored_context_tokens(entries) == 10_010 + + +@pytest.mark.parametrize( + "assistant", + [_assistant(0), _assistant(5_000, stop="error")], + ids=["no-usage", "error"], +) +def test_a_saved_session_with_no_trusted_number_has_no_count(assistant): + entries = [_entry("e1", None, _user()), _entry("e2", "e1", assistant)] + + assert stored_context_tokens(entries) is None + assert stored_context_tokens([]) is None + + +def _compaction(entry_id: str, parent: str, first_kept: str) -> CompactionEntry: + return CompactionEntry( + id=entry_id, + parentId=parent, + timestamp=_STAMP, + summary="SUMMARY", + firstKeptEntryId=first_kept, + tokensBefore=50_000, + ) + + +def test_a_number_from_before_a_compaction_is_not_true_any_more(): + """The kept assistant message still says 50,000. The context that it counted is gone.""" + entries = [ + _entry("e1", None, _user()), + _entry("e2", "e1", _assistant(50_000)), + _entry("e3", "e2", _user()), + _entry("e4", "e3", _assistant(50_000)), + _compaction("c1", "e4", first_kept="e3"), + _entry("e5", "c1", _user()), + ] + + assert stored_context_tokens(entries) is None + + +def test_a_number_from_after_a_compaction_is_true(): + entries = [ + _entry("e1", None, _user()), + _entry("e2", "e1", _assistant(50_000)), + _entry("e3", "e2", _user()), + _entry("e4", "e3", _assistant(50_000)), + _compaction("c1", "e4", first_kept="e3"), + _entry("e5", "c1", _user()), + _entry("e6", "e5", _assistant(7_000)), + ] + + assert stored_context_tokens(entries) == 7_000 + + +# --- Configuration ----------------------------------------------------------------------- + + +def _load(tmp_path: Path, text: str) -> CliConfig: + (tmp_path / "agent.toml").write_text(text, encoding="utf-8") + return load_config(tmp_path) + + +def test_the_window_of_a_model_is_read_from_the_model_table(tmp_path): + config = _load(tmp_path, '[model]\nprovider = "openai"\nid = "m1"\ncontext_window = 200_000\n') + + assert config.context_window == 200_000 + assert config.default_choice().context_window == 200_000 + assert config.auto.notes == () + + +def test_a_model_with_no_window_has_none(): + assert CliConfig().context_window is None + assert CliConfig().default_choice().context_window is None + + +def test_a_listed_model_never_takes_the_window_of_the_default(tmp_path): + """The window belongs to one model. Another model of the same provider has its own.""" + config = _load( + tmp_path, + """ +[model] +provider = "openai" +id = "big" +context_window = 1000000 + +[[models]] +id = "same-provider-no-window" + +[[models]] +id = "same-provider-own-window" +context_window = 64000 + +[[models]] +provider = "anthropic" +id = "other-provider" +context_window = 200000 +""", + ) + + windows = {choice.id: choice.context_window for choice in config.model_choices()} + assert windows == { + "big": 1_000_000, + "same-provider-no-window": None, + "same-provider-own-window": 64_000, + "other-provider": 200_000, + } + + +@pytest.mark.parametrize("raw", ["128_000", "1.28e5"], ids=["integer", "float-without-fraction"]) +def test_a_whole_number_is_accepted(tmp_path, raw): + config = _load(tmp_path, f'[model]\nprovider = "openai"\nid = "m1"\ncontext_window = {raw}\n') + + assert config.context_window == 128_000 + assert config.auto.notes == () + + +def test_a_key_at_the_top_of_the_file_is_read_like_the_other_model_keys(tmp_path): + """The flat form is an old way to write ``[model]`` keys. It must stay in the same place.""" + config = _load(tmp_path, 'context_window = 64000\n[model]\nprovider = "openai"\nid = "m1"\n') + + assert config.context_window == 64_000 + assert config.default_choice().context_window == 64_000 + assert config.auto.notes == () + + +def test_the_model_table_wins_over_a_key_at_the_top_of_the_file(tmp_path): + config = _load( + tmp_path, + """ +context_window = 64000 + +[model] +provider = "openai" +id = "m1" +context_window = 200000 +""", + ) + + assert config.context_window == 200_000 + + +@pytest.mark.parametrize( + "raw", + ['"big"', "0", "-5", "true", "1.5", str(MAX_CONTEXT_WINDOW + 1)], + ids=["text", "zero", "negative", "boolean", "fraction", "too-large"], +) +def test_a_window_that_cannot_be_used_is_dropped_and_the_user_is_told(tmp_path, raw): + config = _load(tmp_path, f'[model]\nprovider = "openai"\nid = "m1"\ncontext_window = {raw}\n') + + assert config.context_window is None + (note,) = config.auto.notes + assert "context_window of [model]" in note + assert "sends no context size" in note + + +def test_the_note_for_a_listed_model_names_the_model(tmp_path): + config = _load(tmp_path, '[[models]]\nid = "m2"\ncontext_window = "wide"\n') + + (choice,) = (c for c in config.model_choices() if c.id == "m2") + assert choice.context_window is None + (note,) = config.auto.notes + assert '"m2"' in note + + +def test_the_window_reaches_the_model_only_when_it_is_declared(): + declared = model_for_choice(ModelChoice(id="m1", provider="mock", context_window=64_000)) + undeclared = model_for_choice(ModelChoice(id="m1", provider="mock")) + + assert declared.context_window == 64_000 + assert undeclared.context_window == Model(provider="mock", model_id="m1").context_window + + +def test_compaction_keeps_its_defaults_when_there_is_no_window_or_the_window_is_large(): + defaults = compaction_settings(ModelChoice(id="m"), auto_compact=False) + + assert (defaults.reserve_tokens, defaults.keep_recent_tokens) == (16_384, 20_000) + assert defaults.auto_compact is False + for window in (65_536, 200_000, 1_000_000): + choice = ModelChoice(id="m", context_window=window) + settings = compaction_settings(choice, auto_compact=True) + assert (settings.reserve_tokens, settings.keep_recent_tokens) == (16_384, 20_000) + assert settings.auto_compact is True + + +@pytest.mark.parametrize( + ("window", "reserve", "keep"), + [(32_000, 8_000, 10_666), (8_192, 2_048, 2_730), (1_000, 250, 333)], +) +def test_compaction_limits_fit_a_small_window(window, reserve, keep): + settings = compaction_settings(ModelChoice(id="m", context_window=window), auto_compact=True) + + assert (settings.reserve_tokens, settings.keep_recent_tokens) == (reserve, keep) + # Room is left between the point where compaction starts and the part that is kept. + assert settings.keep_recent_tokens < window - settings.reserve_tokens + + +def test_compaction_limits_never_reach_zero(): + settings = compaction_settings(ModelChoice(id="m", context_window=1), auto_compact=True) + + assert (settings.reserve_tokens, settings.keep_recent_tokens) == (1, 1) + + +# --- The agent --------------------------------------------------------------------------- + + +@dataclass +class Step: + """What the model answers to one request.""" + + total: int = 0 # ``usage.totalTokens``; 0 is a provider that gives no usage + text: str = "answer" + tool: tuple[str, dict[str, Any]] | None = None # a tool call, instead of text + fail: bool = False + + +class Script: + """A model that answers request N as ``steps[N]`` says. + + A request to summarize (compaction) is answered with a summary, and it takes no step. + ``calls`` counts the requests that took a step. + """ + + def __init__(self, *steps: Step) -> None: + self._steps = list(steps) + self.calls = 0 + + async def __call__( + self, model: Model, context: Any, options: Any = None + ) -> AssistantMessageEventStream: + if context.system_prompt == COMPACTION_SYSTEM_PROMPT: + step = Step(text="SUMMARY") + else: + step = self._steps[self.calls] + self.calls += 1 + if step.tool is not None: + name, arguments = step.tool + call = {"type": "toolCall", "id": f"call_{self.calls}", "name": name} + content: list[Any] = [{**call, "arguments": arguments}] + else: + content = [{"type": "text", "text": step.text}] + partial = _base_partial(model, content) + partial.usage = Usage(input=step.total, totalTokens=step.total) + partial.stopReason = "toolUse" if step.tool else "error" if step.fail else "stop" + stream = AssistantMessageEventStream() + stream.push(StartEvent(partial=partial.model_copy(deep=True))) + if step.tool is None and not step.fail: + stream.push(TextDeltaEvent(partial=partial.model_copy(deep=True), delta=step.text)) + if step.fail: + partial.errorMessage = "scripted error" + stream.push( + ErrorEvent( + partial=partial.model_copy(deep=True), + reason="error", + error_message="scripted error", + ) + ) + else: + reason: Any = partial.stopReason + stream.push(DoneEvent(partial=partial.model_copy(deep=True), reason=reason)) + stream.set_final_message(partial) + stream.end() + return stream + + +class Recorder: + """A client that records what the agent sends and is never asked anything.""" + + def __init__(self, *, refuse_usage: bool = False) -> None: + self.updates: list[tuple[str, Any]] = [] + self.refuse_usage = refuse_usage + + async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None: + if self.refuse_usage and isinstance(update, UsageUpdate): + raise ConnectionError("the client went away") + self.updates.append((session_id, update)) + + def usage(self) -> list[UsageUpdate]: + return [update for _, update in self.updates if isinstance(update, UsageUpdate)] + + def kinds(self) -> list[str]: + return [getattr(update, "session_update", "?") for _, update in self.updates] + + +def make_config(window: int | None = 200_000, **extra: Any) -> CliConfig: + return CliConfig( + permission="ask", provider="mock", model_id="mock", context_window=window, **extra + ) + + +def make_agent( + home: Path, stream_fn: Any, config: CliConfig | None = None, **client: Any +) -> tuple[PiAcpAgent, Recorder]: + agent = PiAcpAgent(stream_fn=stream_fn, home=home, config=config or make_config()) + recorder = Recorder(**client) + agent.on_connect(recorder) # type: ignore[arg-type] + return agent, recorder + + +async def settle(agent: PiAcpAgent) -> None: + # The setup of a session runs after the response has flushed. + await asyncio.gather(*agent._background_tasks) + + +async def say(agent: PiAcpAgent, session_id: str, text: str = "hello") -> None: + await agent.prompt(session_id=session_id, prompt=[text_block(text)]) + + +@pytest.mark.asyncio +async def test_one_turn_sends_one_update_with_the_count_and_the_window(tmp_path): + agent, client = make_agent(tmp_path, Script(Step(total=12_000))) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await settle(agent) + assert client.usage() == [] # a new session has nothing to count + + await say(agent, session_id) + + (update,) = client.usage() + assert (update.used, update.size) == (12_000, 200_000) + assert update.cost is None + assert client.updates[-1] == (session_id, update) # it follows the answer + assert "agent_message_chunk" in client.kinds() + + +@pytest.mark.asyncio +async def test_a_model_with_no_window_gets_no_update_even_when_the_provider_counts(tmp_path): + script = Script(Step(total=12_000)) + agent, client = make_agent(tmp_path, script, make_config(window=None)) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await say(agent, session_id) + + assert script.calls == 1 # the turn ran + assert "agent_message_chunk" in client.kinds() + assert client.usage() == [] + + +@pytest.mark.asyncio +async def test_a_provider_that_gives_no_usage_gets_no_update(tmp_path): + agent, client = make_agent(tmp_path, mock_text_stream) # usage is 0 + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await say(agent, session_id) + + assert client.usage() == [] + + +@pytest.mark.asyncio +async def test_a_failed_turn_gets_no_update(tmp_path): + agent, client = make_agent(tmp_path, Script(Step(total=9_000, fail=True))) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await say(agent, session_id) + + assert client.usage() == [] + + +@pytest.mark.asyncio +async def test_each_turn_of_a_tool_loop_reports_the_count_of_its_end(tmp_path): + (tmp_path / "notes.txt").write_text("x" * 400, encoding="utf-8") + script = Script( + Step(total=1_000, tool=("read", {"path": "notes.txt"})), + Step(total=1_500, text="done"), + ) + agent, client = make_agent(tmp_path, script) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await say(agent, session_id) + + first, second = client.usage() + # The first count has the tool result: the model has not seen it yet, so it is an estimate. + assert first.used > 1_000 + 100 - 1 + assert second.used == 1_500 + assert script.calls == 2 + + +@pytest.mark.asyncio +async def test_a_client_that_refuses_the_update_does_not_break_the_turn(tmp_path, caplog): + """The harness turns the error of a subscriber into an error of the turn.""" + agent, client = make_agent(tmp_path, Script(Step(total=12_000)), refuse_usage=True) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + + response = await agent.prompt(session_id=session_id, prompt=[text_block("hello")]) + + assert response.stop_reason == "end_turn" + assert "Could not send the context use" in caplog.text + assert client.usage() == [] + + +@pytest.mark.asyncio +async def test_a_change_of_model_sends_the_count_again_with_the_new_window(tmp_path): + config = make_config( + window=200_000, + models=( + ModelChoice(id="small", provider="mock", context_window=32_000), + ModelChoice(id="unknown", provider="mock"), + ), + ) + agent, client = make_agent(tmp_path, Script(Step(total=12_000)), config) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await say(agent, session_id) + harness = agent._harnesses[session_id] + assert [(u.used, u.size) for u in client.usage()] == [(12_000, 200_000)] + assert harness.compaction.reserve_tokens == 16_384 + + await agent.set_config_option(config_id="model", session_id=session_id, value="small") + assert [(u.used, u.size) for u in client.usage()][-1] == (12_000, 32_000) + assert (harness.compaction.reserve_tokens, harness.compaction.keep_recent_tokens) == ( + 8_000, + 10_666, + ) + + sent = len(client.usage()) + await agent.set_config_option(config_id="model", session_id=session_id, value="unknown") + assert len(client.usage()) == sent # no window: nothing is sent, and nothing is guessed + assert harness.compaction.reserve_tokens == 16_384 + + await agent.set_config_option(config_id="model", session_id=session_id, value="mock") + assert [(u.used, u.size) for u in client.usage()][-1] == (12_000, 200_000) + + +@pytest.mark.asyncio +async def test_closing_a_session_forgets_its_count(tmp_path): + agent, _ = make_agent(tmp_path, Script(Step(total=12_000))) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await say(agent, session_id) + assert agent._context_used == {session_id: 12_000} + + await agent.close_session(session_id) + + assert agent._context_used == {} + + +@pytest.mark.asyncio +async def test_a_new_session_reads_no_history_and_logs_nothing(tmp_path, caplog): + agent, client = make_agent(tmp_path, Script()) + await agent.new_session(cwd=str(tmp_path)) + await settle(agent) + + assert client.usage() == [] + assert "context use" not in caplog.text + + +async def _saved_session(home: Path, *steps: Step, config: CliConfig | None = None) -> str: + """Run one prompt per step in a session of its own agent, so that the session has a file.""" + agent, _ = make_agent(home, Script(*steps), config) + session_id = (await agent.new_session(cwd=str(home))).session_id + for index, _ in enumerate(steps): + await say(agent, session_id, f"request {index}") + return session_id + + +@pytest.mark.asyncio +async def test_load_sends_the_count_of_the_saved_session_after_the_replay(tmp_path): + session_id = await _saved_session(tmp_path, Step(total=12_000)) + + agent, client = make_agent(tmp_path, Script()) + await agent.load_session(cwd=str(tmp_path), session_id=session_id) + await settle(agent) + + (update,) = client.usage() + assert (update.used, update.size) == (12_000, 200_000) + assert "agent_message_chunk" in client.kinds() # the replay + assert client.kinds().index("usage_update") > client.kinds().index("agent_message_chunk") + assert [sid for sid, u in client.updates if u is update] == [session_id] + + +@pytest.mark.asyncio +async def test_resume_sends_the_count_without_a_replay(tmp_path): + session_id = await _saved_session(tmp_path, Step(total=12_000)) + + agent, client = make_agent(tmp_path, Script()) + await agent.resume_session(cwd=str(tmp_path), session_id=session_id) + await settle(agent) + + (update,) = client.usage() + assert (update.used, update.size) == (12_000, 200_000) + assert "agent_message_chunk" not in client.kinds() + + +@pytest.mark.asyncio +async def test_load_sends_nothing_for_a_model_with_no_window(tmp_path): + session_id = await _saved_session(tmp_path, Step(total=12_000), config=make_config(None)) + + agent, client = make_agent(tmp_path, Script(), make_config(None)) + await agent.load_session(cwd=str(tmp_path), session_id=session_id) + await settle(agent) + + assert client.usage() == [] + + +@pytest.mark.asyncio +async def test_load_sends_nothing_for_a_session_that_gave_no_number(tmp_path): + session_id = await _saved_session(tmp_path, Step(total=0)) + + agent, client = make_agent(tmp_path, Script()) + await agent.load_session(cwd=str(tmp_path), session_id=session_id) + await settle(agent) + + assert client.usage() == [] + + +async def _compacted_session(home: Path) -> tuple[str, Recorder]: + """Three turns, then a compaction. Returns the id and the client that was told.""" + config = make_config(window=40) # a window so small that three short turns need a summary + agent, client = make_agent( + home, Script(Step(total=1_000), Step(total=2_000), Step(total=3_000)), config + ) + session_id = (await agent.new_session(cwd=str(home))).session_id + for index in range(3): + await say(agent, session_id, f"request number {index}") + await agent._harnesses[session_id].compact() + return session_id, client + + +@pytest.mark.asyncio +async def test_a_compaction_sends_an_estimate_of_what_remains(tmp_path): + session_id, client = await _compacted_session(tmp_path) + + told = client.usage() + assert [update.used for update in told[:3]] == [1_000, 2_000, 3_000] + assert len(told) == 4 + after = told[3] + assert after.size == 40 + assert 0 < after.used < 3_000 # the count fell, and it is an estimate: no usage of a provider + assert {sid for sid, update in client.updates if isinstance(update, UsageUpdate)} == { + session_id + } + + +@pytest.mark.asyncio +async def test_load_after_a_compaction_does_not_send_the_number_from_before_it(tmp_path): + session_id, _ = await _compacted_session(tmp_path) # the session is saved: it has a file + + agent, client = make_agent(tmp_path, Script(), make_config(window=40)) + await agent.load_session(cwd=str(tmp_path), session_id=session_id) + await settle(agent) + + assert client.usage() == [] # the kept messages still carry 3,000; the context does not + + +@pytest.mark.asyncio +async def test_a_compaction_with_no_count_on_the_client_sends_nothing(tmp_path): + """No number was sent before, so there is no old number to correct.""" + config = make_config(window=40) + agent, client = make_agent( + tmp_path, Script(Step(total=0), Step(total=0), Step(total=0)), config + ) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + for index in range(3): + await say(agent, session_id, f"request number {index}") + await agent._harnesses[session_id].compact() + + assert any(isinstance(entry, CompactionEntry) for entry in await _entries(agent, session_id)) + assert client.usage() == [] + + +async def _entries(agent: PiAcpAgent, session_id: str) -> list[Any]: + return await agent._harnesses[session_id].session.get_entries() + + +@pytest.mark.asyncio +async def test_compaction_uses_the_window_that_the_user_declared(tmp_path): + """The same context compacts for a model with a small declared window, and not for one with + no window (compaction then uses the placeholder of 128,000 tokens).""" + + async def run(window: int | None) -> bool: + home = tmp_path / ("declared" if window else "undeclared") + home.mkdir() + config = make_config(window=window, max_turns=30) # 30 turns or more: auto-compaction + script = Script(*(Step(total=900) for _ in range(3))) + agent, _ = make_agent(home, script, config) + session_id = (await agent.new_session(cwd=str(home))).session_id + for index in range(3): + await say(agent, session_id, f"request {index} " + "a" * 4_000) + return any(isinstance(e, CompactionEntry) for e in await _entries(agent, session_id)) + + assert await run(window=1_000) is True # 900 tokens is above 1,000 - 250 + assert await run(window=None) is False # 900 tokens is far below 128,000 - 16,384 From b6887015edcf263cc5e5230d8f4147f221e8e05a Mon Sep 17 00:00:00 2001 From: zhangy <40223340+zy1233@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:04:28 +0800 Subject: [PATCH 2/4] feat(core,cli): add the `max` thinking level, and tell the user about an unknown level Providers now name their highest effort `max` (DeepSeek V4, OpenAI GPT-5.6 and later, Anthropic adaptive models, Kimi K3, GLM 5.x), and the reference `pi` has a level of its own for it since v0.80.6. The agent had six levels. With `provider = "openai"` (the way to reach a gateway) the level `xhigh` is sent as "high", so a gateway that serves a DeepSeek V4 model could never be asked for "max". `ThinkingLevel` has a seventh value, `max`: - OpenAI format (gateways too): `reasoning_effort = "max"`, as it is. A model that does not take it answers with an error, and nothing is lowered here. - DeepSeek's own API: "max", the same as `xhigh`. - Anthropic: the budget of `xhigh` (40000). A budget has no step above. The three mapping tables must name every level: a level that a table lacks sends no parameter, and the model then silently does not think. A test checks it. `thinking_level` in agent.toml that is not one of the levels was silently `off`, so a typo looked like "off". It is `off` still, and the user is now told at the start of the session (the same note channel as `context_window`). A list as the value used to stop the loading with a TypeError. The valid levels now come from the type, so a new level of the core is valid in the CLI at once. Not done: a table of levels per model, as `pi` has (`thinkingLevelMap`), so `max` is open for every model. Not changed: Anthropic's budget form, which the documentation of Anthropic says that Claude 4.7 and later refuse (not verified against the live API; noted in docs/DESIGN.md). The VS Code extension has six levels in its menu. It needs an agent that knows `max`, so it follows after the agent release. --- docs/DESIGN.md | 4 +- packages/pi-agent-cli/agent.example.toml | 2 +- packages/pi-agent-cli/pi_agent_cli/config.py | 19 ++++++-- packages/pi-agent-cli/tests/test_config.py | 44 ++++++++++++++++++- .../tests/test_model_reasoning.py | 1 + pi_agent_core/adapters/langchain_stream.py | 10 ++++- pi_agent_core/tests/test_deepseek_replay.py | 2 +- pi_agent_core/tests/test_deepseek_thinking.py | 6 ++- pi_agent_core/tests/test_thinking.py | 38 +++++++++++++++- pi_agent_core/types.py | 2 +- 10 files changed, 114 insertions(+), 14 deletions(-) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 9210d938..35bca20b 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -146,7 +146,9 @@ agent_start → turn_start → message_start(user) → message_end(user) ### 5.2 Thinking / Reasoning -`ThinkingLevel`(off/minimal/low/medium/high/xhigh)映射为 provider 参数:Anthropic `thinking.budget_tokens` + 联动 `max_tokens`;OpenAI `reasoning_effort`;DeepSeek 官方 API 经 `extra_body` 发 `thinking: enabled|disabled` 与 `reasoning_effort`(它默认开启思考,所以闸门关闭时显式发 `disabled`;网关不发。见 Phase 6 规格 §5.4)。Anthropic thinking 块流式捕获为 `ThinkingContent`(含 `signature` 用于多轮工具回放),以 `thinking_delta` 事件实时发射。DeepSeek 风格网关流式返回的 `reasoning_content`(或 `reasoning`)同样捕获:`provider="deepseek"` 由 `ChatDeepSeek` 保留,`provider="openai"` 由 `adapters/openai_reasoning.py` 的 `ChatOpenAI` 子类保留(普通 `ChatOpenAI` 会丢掉它;冻结版只有 openai provider,靠这个才能显示思考;该子类只保留流,不回传)。DeepSeek 官方 API 开启思考时,之前 assistant 消息的这部分在后续请求里按 `reasoning_content` 回传(不回传会 400;`adapters/deepseek_replay.py`,Phase 6 规格 §5.5),网关不回传。 +`ThinkingLevel`(off/minimal/low/medium/high/xhigh/max)映射为 provider 参数:Anthropic `thinking.budget_tokens` + 联动 `max_tokens`;OpenAI `reasoning_effort`;DeepSeek 官方 API 经 `extra_body` 发 `thinking: enabled|disabled` 与 `reasoning_effort`(它默认开启思考,所以闸门关闭时显式发 `disabled`;网关不发。见 Phase 6 规格 §5.4)。Anthropic thinking 块流式捕获为 `ThinkingContent`(含 `signature` 用于多轮工具回放),以 `thinking_delta` 事件实时发射。DeepSeek 风格网关流式返回的 `reasoning_content`(或 `reasoning`)同样捕获:`provider="deepseek"` 由 `ChatDeepSeek` 保留,`provider="openai"` 由 `adapters/openai_reasoning.py` 的 `ChatOpenAI` 子类保留(普通 `ChatOpenAI` 会丢掉它;冻结版只有 openai provider,靠这个才能显示思考;该子类只保留流,不回传)。DeepSeek 官方 API 开启思考时,之前 assistant 消息的这部分在后续请求里按 `reasoning_content` 回传(不回传会 400;`adapters/deepseek_replay.py`,Phase 6 规格 §5.5),网关不回传。 + +`max` 是 `xhigh` 之上的等级(上游 pi 在 v0.80.6 加入;pi 靠每个模型的 `thinkingLevelMap` 决定哪些等级可用,本适配器没有这张表)。它在 OpenAI 格式(含网关)和 DeepSeek 官方 API 上原样发 `"max"`;Anthropic 的预算形式没有比 `xhigh` 更高的一档,所以 `max` 用 `xhigh` 的预算(40000)。适配器不按模型降级:模型不接受 `max` 时,服务端返回错误,错误会显示出来。三张映射表(`_ANTHROPIC_BUDGET`、`_OPENAI_EFFORT`、`_DEEPSEEK_EFFORT`)必须列出每一个等级(测试检查):缺一个键,请求里就没有推理参数,模型会静默不思考。`agent.toml` 里写了不认识的等级时,agent 用 `off`,并在会话开始时告诉用户。Anthropic 的文档说,较新的模型(Claude 4.7 及以后)会拒绝 `budget_tokens`,要用自适应思考加 `effort`;本适配器只发预算形式(未对真实 API 验证;这是另一项工作)。 ### 5.3 跨 Provider 消息回放 (`transform.py`) diff --git a/packages/pi-agent-cli/agent.example.toml b/packages/pi-agent-cli/agent.example.toml index d1eb882a..472a3bdf 100644 --- a/packages/pi-agent-cli/agent.example.toml +++ b/packages/pi-agent-cli/agent.example.toml @@ -24,7 +24,7 @@ permission = "ask" # ask | auto | always-approve # `always-approve` asks about nothing. Headless (-p) runs always use it. Before this version, # `auto` meant "ask about nothing": use `always-approve` for that. -thinking_level = "off" # off | minimal | low | medium | high | xhigh +thinking_level = "off" # off | minimal | low | medium | high | xhigh | max max_turns = 50 [model] diff --git a/packages/pi-agent-cli/pi_agent_cli/config.py b/packages/pi-agent-cli/pi_agent_cli/config.py index 6e8d3c86..0ca50849 100644 --- a/packages/pi-agent-cli/pi_agent_cli/config.py +++ b/packages/pi-agent-cli/pi_agent_cli/config.py @@ -10,7 +10,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field, replace from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, get_args from pi_agent_core.coding_tools.bash import MAX_TIMEOUT_SECONDS from pi_agent_core.home import pi_home @@ -22,7 +22,9 @@ _VALID_PERMISSION: set[str] = {"ask", "auto", "always-approve"} _VALID_DEFAULT_TRUST: set[str] = {"ask", "never", "always"} -_VALID_THINKING: set[str] = {"off", "minimal", "low", "medium", "high", "xhigh"} +# The levels come from the type, so a level that the core learns is valid here at once. +_THINKING_LEVELS: tuple[str, ...] = get_args(ThinkingLevel) +_VALID_THINKING: set[str] = set(_THINKING_LEVELS) # ``pi_home`` (imported above and re-exported from here) is the one place that decides where # pi-python keeps its files; the extension loader and the extensions use the same function. @@ -375,7 +377,14 @@ def _from_toml(data: dict[str, Any]) -> CliConfig: ) permission = "ask" thinking = data.get("thinking_level", "off") - if thinking not in _VALID_THINKING: + thinking_note: str | None = None + if not isinstance(thinking, str) or thinking not in _VALID_THINKING: + # No thinking, and the user is told. This was silent, so a typo looked like "off". + thinking_note = ( + "thinking_level in agent.toml must be " + + ", ".join(f'"{level}"' for level in _THINKING_LEVELS[:-1]) + + f' or "{_THINKING_LEVELS[-1]}". The agent uses "off".' + ) thinking = "off" max_turns = data.get("max_turns") if max_turns is not None: @@ -450,7 +459,9 @@ def _optional_str(value: object) -> str | None: ) auto = _auto_config(data.get("auto")) - leading_notes = [note for note in (permission_note, *window_notes) if note is not None] + leading_notes = [ + note for note in (permission_note, thinking_note, *window_notes) if note is not None + ] if leading_notes: auto = replace(auto, notes=(*leading_notes, *auto.notes)) diff --git a/packages/pi-agent-cli/tests/test_config.py b/packages/pi-agent-cli/tests/test_config.py index 817dcc8e..f95e4a06 100644 --- a/packages/pi-agent-cli/tests/test_config.py +++ b/packages/pi-agent-cli/tests/test_config.py @@ -264,7 +264,7 @@ def test_model_reasoning_follows_the_thinking_level_unless_set(): ``reasoning`` setting overrides that either way.""" assert CliConfig().model_reasoning is False assert CliConfig(thinking_level="off").model_reasoning is False - for level in ("minimal", "low", "medium", "high", "xhigh"): + for level in ("minimal", "low", "medium", "high", "xhigh", "max"): assert CliConfig(thinking_level=level).model_reasoning is True # type: ignore[arg-type] assert CliConfig(thinking_level="high", reasoning=False).model_reasoning is False assert CliConfig(thinking_level="off", reasoning=True).model_reasoning is True @@ -603,3 +603,45 @@ def test_a_file_of_the_project_cannot_set_the_auto_table(tmp_path: Path): assert config.permission == "ask" assert config.auto == AutoConfig() + + +# --------------------------------------------------------------------------------------------- +# ``thinking_level``: the levels, and what a value that is not one of them does +# --------------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("level", ["off", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_every_thinking_level_is_read_and_needs_no_note(tmp_path: Path, level: str): + config = _load(tmp_path, f'thinking_level = "{level}"\n') + + assert config.thinking_level == level + assert config.auto.notes == () + + +def test_no_thinking_level_is_off_and_needs_no_note(tmp_path: Path): + config = _load(tmp_path, 'permission = "ask"\n') + + assert config.thinking_level == "off" + assert config.auto.notes == () + + +def test_max_asks_the_model_to_reason(tmp_path: Path): + """The level is how a user says that the model can reason, ``max`` included.""" + assert _load(tmp_path, 'thinking_level = "max"\n').model_reasoning is True + + +@pytest.mark.parametrize( + "raw", + ['"ultra"', '"MAX"', '""', "5", "true", '["high"]'], + ids=["unknown", "other-case", "empty", "number", "boolean", "list"], +) +def test_a_thinking_level_that_is_not_known_is_off_and_the_user_is_told(tmp_path: Path, raw: str): + """It was silent: a typo looked like ``off``. A list used to stop the loading too.""" + config = _load(tmp_path, f"thinking_level = {raw}\n") + + assert config.thinking_level == "off" + assert config.model_reasoning is False + (note,) = config.auto.notes + assert note.startswith("thinking_level in agent.toml must be ") + assert '"xhigh" or "max"' in note # every level is named, in the order of the type + assert note.endswith('The agent uses "off".') diff --git a/packages/pi-agent-cli/tests/test_model_reasoning.py b/packages/pi-agent-cli/tests/test_model_reasoning.py index 5d8edd09..1972b0d8 100644 --- a/packages/pi-agent-cli/tests/test_model_reasoning.py +++ b/packages/pi-agent-cli/tests/test_model_reasoning.py @@ -40,6 +40,7 @@ async def build_model(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config: C (CliConfig(thinking_level="off"), False), (CliConfig(thinking_level="high"), True), (CliConfig(thinking_level="xhigh"), True), + (CliConfig(thinking_level="max"), True), (CliConfig(thinking_level="high", reasoning=False), False), (CliConfig(thinking_level="off", reasoning=True), True), ], diff --git a/pi_agent_core/adapters/langchain_stream.py b/pi_agent_core/adapters/langchain_stream.py index 87312c3b..2eca90c1 100644 --- a/pi_agent_core/adapters/langchain_stream.py +++ b/pi_agent_core/adapters/langchain_stream.py @@ -46,6 +46,8 @@ logger = logging.getLogger("pi_agent_core.langchain_stream") +# Each table must name every ThinkingLevel (a test checks it). A level that a table lacks sends no +# parameter, and the model then thinks at the default of the provider, or not at all. _ANTHROPIC_BUDGET: dict[ThinkingLevel, int | None] = { "off": None, "minimal": 1024, @@ -53,6 +55,8 @@ "medium": 10000, "high": 20000, "xhigh": 40000, + # A budget has no step above xhigh, so max is the budget of xhigh. + "max": 40000, } _OPENAI_EFFORT: dict[ThinkingLevel, str | None] = { @@ -62,11 +66,14 @@ "medium": "medium", "high": "high", "xhigh": "high", + # Sent as it is. A model that does not take it (an older OpenAI model, for example) answers + # with an error, and nothing is lowered here: the user asked for max. + "max": "max", } # DeepSeek's own mapping for pi ("reasoningEffortMap" in # https://api-docs.deepseek.com/quick_start/agent_integrations/pi_mono): every level below -# xhigh is "high", xhigh is "max". +# xhigh is "high", xhigh is "max". The level max is "max" too. _DEEPSEEK_EFFORT: dict[ThinkingLevel, str | None] = { "off": None, "minimal": "high", @@ -74,6 +81,7 @@ "medium": "high", "high": "high", "xhigh": "max", + "max": "max", } # Anthropic requires max_tokens to exceed the thinking budget; reserve room for diff --git a/pi_agent_core/tests/test_deepseek_replay.py b/pi_agent_core/tests/test_deepseek_replay.py index 725d60b6..d867e903 100644 --- a/pi_agent_core/tests/test_deepseek_replay.py +++ b/pi_agent_core/tests/test_deepseek_replay.py @@ -258,7 +258,7 @@ def test_the_text_of_an_answer_is_still_plain_text(self): assert second["content"] == "done" - @pytest.mark.parametrize("level", ["minimal", "low", "medium", "high", "xhigh"]) + @pytest.mark.parametrize("level", ["minimal", "low", "medium", "high", "xhigh", "max"]) def test_every_thinking_level_sends_it(self, level: str): first, _ = self.assistants(deepseek(), level) diff --git a/pi_agent_core/tests/test_deepseek_thinking.py b/pi_agent_core/tests/test_deepseek_thinking.py index 18a4e9dc..ef022b8b 100644 --- a/pi_agent_core/tests/test_deepseek_thinking.py +++ b/pi_agent_core/tests/test_deepseek_thinking.py @@ -92,6 +92,7 @@ class TestThinkingOn: ("medium", "high"), ("high", "high"), ("xhigh", "max"), + ("max", "max"), ], ) @pytest.mark.parametrize("base_url", [None, "https://api.deepseek.com/v1"]) @@ -131,7 +132,7 @@ def test_no_level_means_do_not_think(self, level: str | None): assert kwargs == {"extra_body": {"thinking": NOT_THINKING}} - @pytest.mark.parametrize("level", ["minimal", "high", "xhigh"]) + @pytest.mark.parametrize("level", ["minimal", "high", "xhigh", "max"]) def test_a_model_not_declared_to_reason_does_not_think_whatever_the_level(self, level: str): kwargs = _apply_reasoning_params({}, deepseek(reasoning=False), level) # type: ignore[arg-type] @@ -149,7 +150,7 @@ def test_what_the_caller_already_set_is_kept_and_not_modified(self): class TestGatewaysAreLeftAlone: @pytest.mark.parametrize("base_url", GATEWAYS) @pytest.mark.parametrize("reasoning", [True, False]) - @pytest.mark.parametrize("level", ["off", "low", "xhigh", None]) + @pytest.mark.parametrize("level", ["off", "low", "xhigh", "max", None]) def test_nothing_is_added(self, base_url: str, reasoning: bool, level: str | None): model = deepseek(reasoning=reasoning, base_url=base_url) @@ -170,6 +171,7 @@ def test_openai_pointed_at_deepseek_keeps_its_own_rules(self): ) assert _apply_reasoning_params({}, model, "high") == {"reasoning_effort": "high"} + assert _apply_reasoning_params({}, model, "max") == {"reasoning_effort": "max"} assert _apply_reasoning_params({}, model, "off") == {} diff --git a/pi_agent_core/tests/test_thinking.py b/pi_agent_core/tests/test_thinking.py index 27b97100..e0753ed2 100644 --- a/pi_agent_core/tests/test_thinking.py +++ b/pi_agent_core/tests/test_thinking.py @@ -2,11 +2,19 @@ from __future__ import annotations +from typing import get_args + import pytest -from pi_agent_core.adapters.langchain_stream import _apply_reasoning_params, _extract_thinking_delta +from pi_agent_core.adapters.langchain_stream import ( + _ANTHROPIC_BUDGET, + _DEEPSEEK_EFFORT, + _OPENAI_EFFORT, + _apply_reasoning_params, + _extract_thinking_delta, +) from pi_agent_core.tests.mock_stream import mock_thinking_stream -from pi_agent_core.types import LlmContext, Model +from pi_agent_core.types import LlmContext, Model, ThinkingLevel def test_apply_reasoning_anthropic(): @@ -25,6 +33,32 @@ def test_apply_reasoning_openai(): assert "model_kwargs" not in kwargs +def test_apply_reasoning_max(): + """``max`` is the budget of ``xhigh`` on Anthropic, and "max" as it is in the OpenAI format.""" + anthropic = Model(provider="anthropic", model_id="claude-3", reasoning=True) + kwargs = _apply_reasoning_params({}, anthropic, "max") + assert kwargs["thinking"] == {"type": "enabled", "budget_tokens": 40000} + assert kwargs["max_tokens"] > 40000 + + openai = Model(provider="openai", model_id="gpt-5.6", reasoning=True) + assert _apply_reasoning_params({}, openai, "max") == {"reasoning_effort": "max"} + # The level below keeps its mapping: older OpenAI models take no "xhigh". + assert _apply_reasoning_params({}, openai, "xhigh") == {"reasoning_effort": "high"} + + +def test_max_needs_the_capability_flag_like_every_level(): + openai = Model(provider="openai", model_id="gpt-4o", reasoning=False) + assert _apply_reasoning_params({}, openai, "max") == {} + + +def test_every_thinking_level_has_a_key_in_each_table(): + """A level that a table lacks sends no parameter, so the model silently does not think.""" + levels = set(get_args(ThinkingLevel)) + for table in (_ANTHROPIC_BUDGET, _OPENAI_EFFORT, _DEEPSEEK_EFFORT): + assert set(table) == levels + assert [level for level, value in table.items() if value is None] == ["off"] + + def test_apply_reasoning_off(): model = Model(provider="openai", model_id="gpt-4", reasoning=True) assert _apply_reasoning_params({}, model, "off") == {} diff --git a/pi_agent_core/types.py b/pi_agent_core/types.py index e2f0027e..822d1e13 100644 --- a/pi_agent_core/types.py +++ b/pi_agent_core/types.py @@ -22,7 +22,7 @@ UsageCost, ) -ThinkingLevel = Literal["off", "minimal", "low", "medium", "high", "xhigh"] +ThinkingLevel = Literal["off", "minimal", "low", "medium", "high", "xhigh", "max"] ToolExecutionMode = Literal["sequential", "parallel"] QueueMode = Literal["all", "one-at-a-time"] From 27a853da818008fd3eb7482ce8083193acb2c4d9 Mon Sep 17 00:00:00 2001 From: zhangy <40223340+zy1233@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:11:39 +0800 Subject: [PATCH 3/4] feat(cli): add `auto_compact`, answer `max_turn_requests`, and raise the example `max_turns` The default of `max_turns` is `None` (no limit) in the core, the harness and the CLI. The 50 that a user sees is the value in `agent.example.toml`. Three small changes make a high limit and a window of 1M tokens work. `auto_compact` (a key at the top of agent.toml, `true` or `false`) Compaction by the agent was on only when `max_turns >= 30`, so a user with no `max_turns` (every user of `--setup`) never compacted, and a user who wanted a high limit and no compaction could not have it. `factory.auto_compact_enabled` decides now. The key wins. When it is not set, compaction is on for a model with a declared `context_window` and for `max_turns >= 30` (the old rule, which the benchmark path uses), and off otherwise. After a change of model the rule decides again for the new model. A value that is not a boolean is ignored, with a note. `auto_compact = true` for a model with no window assumes the placeholder of 128,000 tokens, so a model with a window of 1M needs its real `context_window`: with the placeholder, compaction starts above 111,616 tokens, which is 11.2 % of that window. `max_turn_requests` A prompt that hit `max_turns` ended with the stop reason `end_turn` and a line of error text. ACP has a stop reason for it, and now the agent sends it. `MAX_TURNS_MESSAGE` is the one source of the text of `MaxTurnsExceededError`, and `_stop_reason` finds the failure by it. Any other error is still `end_turn`. The VS Code extension shows "The turn ended: max_turn_requests." for it without a change, and `zypi` takes it as "done". `agent.example.toml` `max_turns` is 200 (it was 50), with a comment that says it guards against a loop that does not end and does not limit the context. In the benchmark a long task used up to 100 calls, and the official `pi` ran one task for 334. The example `context_window` is 1000000 (it was 128000, which is wrong for a model with 1M). The default in the code stays `None`: a finite default would put a limit that nobody can see on every user who has no `max_turns`. Test helper: `make_config` in test_context_usage.py sets `auto_compact=False` unless a test says otherwise, so that a window of 40 tokens does not compact after every answer of the tests that start a compaction themselves. Five rules were broken one at a time, and each break made a test fail. Record: docs/specs/2026-11-16-thinking-max-and-turn-limit.md. --- docs/DESIGN.md | 2 +- docs/specs/2026-11-13-agent-context-usage.md | 2 +- .../2026-11-16-thinking-max-and-turn-limit.md | 72 +++++++++ packages/pi-agent-cli/AGENTS.md | 2 + packages/pi-agent-cli/agent.example.toml | 17 +- packages/pi-agent-cli/pi_agent_cli/agent.py | 14 +- packages/pi-agent-cli/pi_agent_cli/config.py | 16 +- packages/pi-agent-cli/pi_agent_cli/factory.py | 23 ++- packages/pi-agent-cli/tests/test_acp_agent.py | 49 ++++++ .../pi-agent-cli/tests/test_context_usage.py | 147 +++++++++++++++++- pi_agent_core/agent_loop.py | 3 +- pi_agent_core/tests/test_agent_loop.py | 6 +- pi_agent_core/types.py | 6 + 13 files changed, 342 insertions(+), 17 deletions(-) create mode 100644 docs/specs/2026-11-16-thinking-max-and-turn-limit.md diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 35bca20b..ff760d1c 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -163,7 +163,7 @@ messages → normalize_tool_call_ids → downgrade_thinking → strip_unsupporte ### 5.4 流式重试与失控保护 - **重试**:首 token 前指数退避 + 抖动(`Retry-After` 感知),可配 `max_retries` -- **`max_turns`**:超限抛 `MaxTurnsExceededError`,Agent 转为 error-stop assistant message +- **`max_turns`**:超限抛 `MaxTurnsExceededError`,Agent 转为 error-stop assistant message。错误文本以 `MAX_TURNS_MESSAGE` 开头;ACP CLI 靠它把这次停止报成 `max_turn_requests`(其他错误仍是 `end_turn`)。默认值是 `None`(不限制) - **`tool_timeout`**:单工具超时 → error tool result(LLM 可见可自纠) ### 5.5 Guardrail Hooks 与可观测性 diff --git a/docs/specs/2026-11-13-agent-context-usage.md b/docs/specs/2026-11-13-agent-context-usage.md index 6101a123..466f34f4 100644 --- a/docs/specs/2026-11-13-agent-context-usage.md +++ b/docs/specs/2026-11-13-agent-context-usage.md @@ -54,7 +54,7 @@ The change follows the proposal of the first version of this record. The differe ## 5. Risks and side effects -- **Compaction changes (on purpose).** For a model with a `context_window`, `should_compact` uses the real window, in each session that has `auto_compact` on (`max_turns >= 30`; `agent.example.toml` sets 50). A model with a window under 128,000 compacts earlier (this is correct). A model with a larger window compacts later. A model with no `context_window` does not change. Test: `test_compaction_uses_the_window_that_the_user_declared`. +- **Compaction changes (on purpose).** For a model with a `context_window`, `should_compact` uses the real window, in each session that has `auto_compact` on (since 2026-11-16 the key `auto_compact` and a declared window decide, and `max_turns >= 30` still counts: see `2026-11-16-thinking-max-and-turn-limit.md`). A model with a window under 128,000 compacts earlier (this is correct). A model with a larger window compacts later. A model with no `context_window` does not change. Test: `test_compaction_uses_the_window_that_the_user_declared`. - **The count is partly an estimate.** A tool result that the model has not read yet is an estimate. The count after a compaction is an estimate for the messages only: it has no system prompt and no tool descriptions, and 4 characters for 1 token is too low for Chinese, Japanese and Korean text. The next model call gives an exact count. - **Gateways.** A gateway that sends no usage gives no update. The agent does not show an estimate that has no number of a provider behind it. - **A window that is too small.** A user who writes a window under the real one can see more than 100 %. The agent sends the number as it is. diff --git a/docs/specs/2026-11-16-thinking-max-and-turn-limit.md b/docs/specs/2026-11-16-thinking-max-and-turn-limit.md new file mode 100644 index 00000000..a195e67d --- /dev/null +++ b/docs/specs/2026-11-16-thinking-max-and-turn-limit.md @@ -0,0 +1,72 @@ +# The `max` thinking level and the turn limit: investigation record + +Date: 2026-11-16. Status: built in the branch of PR #131, as separate commits. Not released. + +## 1. Question + +The owner asked for two small changes: + +1. Add `max` to `thinking_level`. +2. Raise the default of `max_turns`, because the main models now have a context of 1M tokens. + +## 2. What was checked + +### 2.1 `thinking_level = "max"` + +| # | Question | Answer | +|---|----------|--------| +| 1 | Does the reference have it? | Yes. `pi` (TypeScript) added a separate, opt-in level `max` above `xhigh` in v0.80.6 (2026-07-09). `pi` also decides per model which levels are open: `Model.thinkingLevelMap` (a string is sent, `null` means "not supported"), with `getSupportedThinkingLevels()` and `clampThinkingLevel()`. | +| 2 | Do providers use the name? | Yes. The `pi` changelog names `max` as a level of DeepSeek V4, OpenAI GPT-5.6 and later, Anthropic adaptive models (`effort`), Kimi K3, GLM 5.x and MiniMax M3.1. | +| 3 | What did the agent do before? | It had six levels. `provider = "openai"` sent `xhigh` as "high", so a gateway for a DeepSeek V4 model could not be asked for "max". Only DeepSeek's own API turned `xhigh` into "max". | +| 4 | Did an unknown level warn? | No. A `thinking_level` that was not one of the six was `off` with no word, so a typo looked like `off`. A list as the value stopped the loading with a `TypeError`. | +| 5 | Does the budget form of Anthropic work for new models? | **Not verified.** The documentation of Anthropic says that Claude 4.7 and later refuse `thinking: {type: enabled, budget_tokens}` (400) and want `thinking: {type: adaptive}` with `effort`. `ChatAnthropic` has `effort`, but `pyproject.toml` allows `langchain-anthropic>=0.2.0`, which is too old for it. `langchain_anthropic` is not installed in `.venv`. | + +### 2.2 `max_turns` + +| # | Question | Answer | +|---|----------|--------| +| 1 | What is the default? | `None` (no limit) in the core, the harness and the CLI. The `50` is only the value in `agent.example.toml`. `--setup` writes only `[model]`, so a user of `--setup` has no limit. | +| 2 | What else does `max_turns` do? | `factory.py` turned compaction on when `max_turns >= 30`. With no `max_turns` there was no compaction. | +| 3 | Which window does compaction use? | The placeholder of the core (128,000) when `context_window` is not declared. Computed with the real code for a gateway with `provider = "openai"`, `max_turns = 150` and no `context_window`: compaction starts above 111,616 tokens, which is 11.2 % of a window of 1M. With `context_window = 1000000` it starts above 983,616. | +| 4 | What do other agents do? | Claude Agent SDK: no limit. Gemini CLI: no limit (`maxSessionTurns = -1`, with loop detection). Goose: 1000. OpenHands (V0): 100. | +| 5 | How many calls does a long task need? | `docs/benchmarks/FRONTIER-HARNESS-30-EVAL-REPORT.md`: nine DeepSWE tasks all used the whole limit of 60 calls. Six were run again with a limit of 100, and four passed. Three of these finished by themselves (httpx 28, katex 78 and expr 89 calls), and `arktype` used all 100. The official `pi`, with no limit, ran `arktype` for 334 calls and $10.43. That was another model, so it is only a measure of size. | +| 6 | What does a client see at the limit? | `end_turn`, and a message of the agent `Error: Agent loop exceeded max_turns=N`. ACP has a stop reason for it: `max_turn_requests`. The extension shows "The turn ended: …" for a stop reason that is not `end_turn`, `cancelled` or `error` (`chat-controller.ts`). `zypi` takes it as "done" (`turn_completion.rs`). | + +## 3. Decisions (the owner, 2026-11-16) + +1. `max`: **Plan A**. Add the level and the three mappings. No table of levels per model (Plan B, not built). The change goes in the branch of PR #131, as its own commit. +2. `max_turns`: **all three** of these. + 1. The value in `agent.example.toml` is 200 (it was 50), with a comment that says it guards against a loop that does not end. The default in the code stays `None`: a finite default would put a limit that nobody can see on every user who has no `max_turns`, and the `--setup` users have none. + 2. A key `auto_compact`, so that compaction does not depend on `max_turns` (section 4). + 3. The stop reason `max_turn_requests`. + +## 4. What was built + +- `ThinkingLevel` has `max` (`pi_agent_core/types.py`). OpenAI format: "max", as it is. DeepSeek's own API: "max". Anthropic: the budget of `xhigh` (40000), because a budget has no step above. Each of the three tables names every level, and a test checks it: a level that a table lacks sends no parameter, and the model then silently does not think. +- A `thinking_level` that is not one of the levels is `off`, and a note says so at the start of the session (the same channel as `context_window`: `auto.notes`). The valid levels come from the type, so a level that the core learns is valid in the CLI at once. +- `auto_compact` is a key at the top of `agent.toml` (`true` or `false`; another type is ignored, with a note). `factory.auto_compact_enabled` decides. The key wins. When it is not set, compaction is on for a model with a declared `context_window` and for `max_turns >= 30`, and off otherwise. After a change of model the rule decides again. `auto_compact = true` for a model with no window assumes 128,000 tokens. +- `MAX_TURNS_MESSAGE` (`pi_agent_core/types.py`) is the one source of the text of `MaxTurnsExceededError`. `_stop_reason` answers `max_turn_requests` when the text of an error holds it. Any other error is still `end_turn`. +- `agent.example.toml`: `max_turns = 200`, the text for `auto_compact`, and `context_window = 1000000` as the example. It was 128000, which is wrong for a model with a window of 1M. + +Tests: `test_thinking.py`, `test_deepseek_thinking.py`, `test_deepseek_replay.py`, `test_config.py`, `test_model_reasoning.py`, `test_agent_loop.py`, `test_acp_agent.py` and `test_context_usage.py`. Five rules were broken one at a time (the stop reason branch, the window rule, the key rule, the decision after a change of model, and the text of the error): each break made a test fail. + +## 5. Risks and side effects + +- **A declared window turns compaction on.** A user who has `context_window` and no `max_turns` did not compact before. This is on purpose: the agent knows when the context is full. `auto_compact = false` turns it off. `context_window` is not released yet (PR #131), so nobody depends on the old behaviour. +- **`max` is open for every model.** A model that does not take it answers with an error, and nothing is lowered. Plan B would lower it. +- **Old agents.** An agent that does not know `max` takes `thinking_level = "max"` as `off`, and it gives no note. Release the agent first. Then add `max` to the menu of the extension, only for an agent that knows it. +- **A client sees `max_turn_requests`, not `end_turn`.** The extension shows a line of notice. `zypi` takes it as "done". Another client may show it as a failure: not checked. + +## 6. Not verified + +- A live gateway or provider that gets `reasoning_effort = "max"` (a gateway for DeepSeek V4, and api.openai.com). +- The Anthropic path (no `langchain_anthropic` in `.venv`), and the claim about Claude 4.7 and later (it comes from the documentation). +- The VS Code extension with `thinking_level = "max"` in the file. Its menu has six levels. +- What happens when a session with compaction off reaches the limit of the model. + +## 7. Open + +1. The extension: add `max` to the Thinking menu, only when `agentInfo.version` is new enough. After the agent release. +2. Anthropic: adaptive thinking with `effort` for new models, and a higher floor for `langchain-anthropic`. A separate change. +3. Plan B: a table of levels per model, as in `pi`, so that a level that a model does not take is lowered, and a menu can hide it. +4. A compaction threshold of its own. With a window of 1M, compaction starts above 983,616 tokens, which can be late (cost, quality). Not investigated. diff --git a/packages/pi-agent-cli/AGENTS.md b/packages/pi-agent-cli/AGENTS.md index f80a7859..fb120235 100644 --- a/packages/pi-agent-cli/AGENTS.md +++ b/packages/pi-agent-cli/AGENTS.md @@ -44,6 +44,8 @@ The agent tells a client how full the context window is, with the standard ACP s - **When it is sent.** At each `turn_end` (the number of the provider for the assistant message, plus an estimate for the tool results of the turn). After a compaction (an estimate for the messages that remain, only for a session that has a count already). After `session/load` and `session/resume` (the saved history, only a number from after the last compaction). After a change of model (the same count with the new window; no window, nothing). A provider that gives no usage, and a turn that ends with `error` or `aborted`, give no update. - **A send never raises.** The harness turns the error of a subscriber into an error of the turn, so every send is guarded. - **Compaction uses the same window.** A model with a `context_window` compacts by it. `factory.compaction_settings` cuts `reserve_tokens` and `keep_recent_tokens` for a small window (a quarter and a third of it), so that a small window does not compact after every turn. +- **Who turns compaction on.** `auto_compact` in `agent.toml` (`true` or `false`) decides when it is set. When it is not set, compaction is on for a model with a `context_window` and for a session with `max_turns` of 30 or more (`factory.auto_compact_enabled`; it decides again for each model after a change of model). A model with no window and no turn limit does not compact by itself: its window is the placeholder of the core, and nothing is guessed. `auto_compact = true` with no window assumes 128,000 tokens, so a 1M model needs its real `context_window`. Record: `docs/specs/2026-11-16-thinking-max-and-turn-limit.md`. +- **The turn limit and ACP.** A prompt that hits `max_turns` ends with the stop reason `max_turn_requests` (`_stop_reason` finds it by `MAX_TURNS_MESSAGE`, the text of `MaxTurnsExceededError`). The client still gets the line `Error: Agent loop exceeded max_turns=N` as a message of the agent. Any other error is still `end_turn`. - **Clients.** The VS Code extension shows the update (a Context row). `zypi` reads it and ignores it. It can read it only because its ACP crate has the feature `unstable` (`tui/Cargo.toml`): keep that feature. ## Project trust diff --git a/packages/pi-agent-cli/agent.example.toml b/packages/pi-agent-cli/agent.example.toml index 472a3bdf..b3d10d1e 100644 --- a/packages/pi-agent-cli/agent.example.toml +++ b/packages/pi-agent-cli/agent.example.toml @@ -25,7 +25,13 @@ permission = "ask" # ask | auto | always-approve # `auto` meant "ask about nothing": use `always-approve` for that. thinking_level = "off" # off | minimal | low | medium | high | xhigh | max -max_turns = 50 +# The most model calls that one prompt may make. It stops a loop that does not end, and it does +# not limit the size of the context. Unset: no limit. +max_turns = 200 +# Does the agent compact the context by itself when it is nearly full? Unset: yes when the model +# has a `context_window` (below) or `max_turns` is 30 or more. `false` turns it off. `true` turns +# it on for a model with no `context_window` too (the agent then assumes 128,000 tokens). +# auto_compact = true [model] provider = "deepseek" @@ -37,9 +43,10 @@ api_key_env = "REAL_LLM_API_KEY" # key for `provider` only; other providers use # reasoning = true # The context window of the model, in tokens (a whole number). With it the agent tells a client how # full the context is (ACP `usage_update`; the VS Code extension shows it as a Context row), and -# compaction (`max_turns` of 30 or more) uses it. Unset: the agent sends no context size and does not -# guess one. Use the real window of your model: a wrong number gives a wrong percent. -# context_window = 128000 +# compaction (see `auto_compact`) starts before the window is full. Unset: the agent sends no +# context size and does not guess one (compaction then assumes 128,000 tokens). Use the real window +# that your provider serves: a wrong number gives a wrong percent. +# context_window = 1000000 # Extra models for the TUI `/model` picker (ACP session config option "model"). # `[model]` above is always offered first; unset fields inherit from it, but @@ -48,7 +55,7 @@ api_key_env = "REAL_LLM_API_KEY" # key for `provider` only; other providers use # [[models]] # id = "deepseek-ai/DeepSeek-V4-Pro" # name = "DeepSeek V4 Pro" -# context_window = 128000 +# context_window = 1000000 # # [[models]] # provider = "openai" diff --git a/packages/pi-agent-cli/pi_agent_cli/agent.py b/packages/pi-agent-cli/pi_agent_cli/agent.py index ac4c38af..dff17637 100644 --- a/packages/pi-agent-cli/pi_agent_cli/agent.py +++ b/packages/pi-agent-cli/pi_agent_cli/agent.py @@ -101,6 +101,7 @@ ) from pi_agent_cli.factory import ( api_key_for_choice, + auto_compact_enabled, compaction_settings, create_session_harness, default_stream_fn, @@ -137,7 +138,7 @@ from pi_agent_cli.trust_store import TrustStore from pi_agent_core.coding_tools.path_utils import normalize_host_path from pi_agent_core.messages import ImageContent -from pi_agent_core.types import StreamFn +from pi_agent_core.types import MAX_TURNS_MESSAGE, StreamFn from pi_agent_harness import ( AgentHarness, AgentHarnessError, @@ -575,9 +576,10 @@ async def set_config_option( {"configId": config_id, "value": value, "reason": "unknown model"} ) await harness.set_model(model_for_choice(choice, reasoning=self._config.model_reasoning)) - # The limits of compaction depend on the window of the model (``compaction_settings``). + # Whether the agent compacts by itself, and the limits of compaction, depend on the + # window of the model (``auto_compact_enabled``, ``compaction_settings``). harness.compaction = compaction_settings( - choice, auto_compact=harness.compaction.auto_compact + choice, auto_compact=auto_compact_enabled(self._config, choice) ) harness.get_api_key = api_key_for_choice(choice) self._session_models[session_id] = choice @@ -1579,6 +1581,10 @@ def _stop_reason(message: Any) -> str: ACP stopReason enum: 'end_turn' | 'max_tokens' | 'max_turn_requests' | 'refusal' | 'cancelled'. Internal stopReason values: 'stop' | 'length' | 'toolUse' | 'error' | 'aborted'. + + An 'error' is 'max_turn_requests' when the loop hit its ``max_turns`` limit (the one error + that ACP has a word for), 'refusal' when the text says the provider refused, and 'end_turn' + for any other error: the client shows the text of the error as a message of the agent. """ reason = getattr(message, "stopReason", None) or "stop" if reason == "aborted": @@ -1587,6 +1593,8 @@ def _stop_reason(message: Any) -> str: return "max_tokens" if reason == "error": err = str(getattr(message, "errorMessage", "") or "") + if MAX_TURNS_MESSAGE in err: + return "max_turn_requests" if any(w in err.lower() for w in ("refus", "policy", "filter", "safety")): return "refusal" return "end_turn" diff --git a/packages/pi-agent-cli/pi_agent_cli/config.py b/packages/pi-agent-cli/pi_agent_cli/config.py index 0ca50849..7f5601e1 100644 --- a/packages/pi-agent-cli/pi_agent_cli/config.py +++ b/packages/pi-agent-cli/pi_agent_cli/config.py @@ -203,6 +203,9 @@ class CliConfig: base_url: str | None = None thinking_level: ThinkingLevel = "off" max_turns: int | None = None + # Whether the agent compacts the context by itself. ``None``: not set, and then + # ``factory.auto_compact_enabled`` decides. + auto_compact: bool | None = None api_key_env: str | None = None supports_images: bool | None = None # ``[model] context_window``: the window of the default model, in tokens. ``None``: not set @@ -390,6 +393,14 @@ def _from_toml(data: dict[str, Any]) -> CliConfig: if max_turns is not None: max_turns = int(max_turns) + auto_compact_raw = data.get("auto_compact") + auto_compact: bool | None = None + compact_note: str | None = None + if isinstance(auto_compact_raw, bool): + auto_compact = auto_compact_raw + elif auto_compact_raw is not None: + compact_note = "auto_compact in agent.toml must be true or false, and it is ignored." + raw_paths = skills.get("paths") or skills.get("dirs") or [] skills_dirs: tuple[str, ...] = () if isinstance(raw_paths, list): @@ -460,7 +471,9 @@ def _optional_str(value: object) -> str | None: auto = _auto_config(data.get("auto")) leading_notes = [ - note for note in (permission_note, thinking_note, *window_notes) if note is not None + note + for note in (permission_note, thinking_note, *window_notes, compact_note) + if note is not None ] if leading_notes: auto = replace(auto, notes=(*leading_notes, *auto.notes)) @@ -472,6 +485,7 @@ def _optional_str(value: object) -> str | None: base_url=model.get("base_url") or data.get("base_url"), thinking_level=thinking, # type: ignore[arg-type] max_turns=max_turns, + auto_compact=auto_compact, api_key_env=model.get("api_key_env") or data.get("api_key_env"), supports_images=supports_images, context_window=context_window, diff --git a/packages/pi-agent-cli/pi_agent_cli/factory.py b/packages/pi-agent-cli/pi_agent_cli/factory.py index 53b3df9f..db53119f 100644 --- a/packages/pi-agent-cli/pi_agent_cli/factory.py +++ b/packages/pi-agent-cli/pi_agent_cli/factory.py @@ -75,6 +75,27 @@ def model_for_choice(choice: ModelChoice, *, reasoning: bool = False) -> Model: return model +# With ``auto_compact`` not set, a session of this many turns (``max_turns``) or more compacts by +# itself. This was the only rule before the key existed, and the benchmark path relies on it. +AUTO_COMPACT_MIN_TURNS = 30 + + +def auto_compact_enabled(config: CliConfig, choice: ModelChoice) -> bool: + """Whether the agent compacts the context by itself for ``choice``. + + ``auto_compact`` in agent.toml decides when it is set. When it is not set, compaction is on + for a model with a declared ``context_window`` (the agent knows when the context is full), + and for a session with ``max_turns`` of ``AUTO_COMPACT_MIN_TURNS`` or more. A model with no + window in a session with no such limit does not compact by itself: its window is the + placeholder of the core, and the agent does not guess one. + """ + if config.auto_compact is not None: + return config.auto_compact + if config.max_turns is not None and config.max_turns >= AUTO_COMPACT_MIN_TURNS: + return True + return choice.context_window is not None + + def compaction_settings(choice: ModelChoice, *, auto_compact: bool) -> CompactionSettings: """Compaction limits for ``choice``. @@ -296,7 +317,7 @@ async def system_prompt_callback(ctx: dict[str, Any]) -> str: system_prompt_options=options, ) - auto_compact = config.max_turns is not None and config.max_turns >= 30 + auto_compact = auto_compact_enabled(config, choice) harness = AgentHarness( session=session, model=model, diff --git a/packages/pi-agent-cli/tests/test_acp_agent.py b/packages/pi-agent-cli/tests/test_acp_agent.py index 36331f4a..74886e68 100644 --- a/packages/pi-agent-cli/tests/test_acp_agent.py +++ b/packages/pi-agent-cli/tests/test_acp_agent.py @@ -416,6 +416,40 @@ async def _bash_once_stream(model, context, options=None): return stream +async def _read_forever_stream(model, context, options=None): + """A model that asks for one more `read` after every result: the loop never ends by itself.""" + stream = AssistantMessageEventStream() + tc: ToolCallContent = { + "type": "toolCall", + "id": f"call_read_{len(context.messages)}", + "name": "read", + "arguments": {"path": "missing.txt"}, + } + partial = _base_partial(model, [tc]) + partial.stopReason = "toolUse" + stream.push(StartEvent(partial=partial.model_copy(deep=True))) + stream.push(DoneEvent(partial=partial.model_copy(deep=True), reason="toolUse")) + stream.set_final_message(partial) + stream.end() + return stream + + +@pytest.mark.asyncio +async def test_a_prompt_that_reaches_max_turns_ends_with_max_turn_requests(tmp_path): + """The client got "end_turn" and one line of error text. ACP has a stop reason for this.""" + agent = PiAcpAgent( + stream_fn=_read_forever_stream, + home=tmp_path, + config=CliConfig(permission="ask", provider="mock", model_id="mock", max_turns=2), + ) + agent.on_connect(FakeClient(allow=False)) # `read` declares itself harmless: nothing is asked + session = await agent.new_session(cwd=str(tmp_path)) + + response = await agent.prompt(session_id=session.session_id, prompt=[text_block("go on")]) + + assert response.stop_reason == "max_turn_requests" + + @pytest.mark.asyncio async def test_permission_ask_denies_bash(tmp_path): agent = _agent(tmp_path, stream_fn=_bash_once_stream, permission="ask") @@ -533,6 +567,21 @@ def test_stop_reason_mapping(): ) == "refusal" ) + # The loop's own limit is the one error that ACP names. Any other error stays "end_turn". + assert ( + _stop_reason( + AssistantMessage( + content=[], stopReason="error", errorMessage="Agent loop exceeded max_turns=200" + ) + ) + == "max_turn_requests" + ) + assert ( + _stop_reason( + AssistantMessage(content=[], stopReason="error", errorMessage="429 rate limit") + ) + == "end_turn" + ) def test_tool_title_extraction(): diff --git a/packages/pi-agent-cli/tests/test_context_usage.py b/packages/pi-agent-cli/tests/test_context_usage.py index 622dd9f2..ab231cd2 100644 --- a/packages/pi-agent-cli/tests/test_context_usage.py +++ b/packages/pi-agent-cli/tests/test_context_usage.py @@ -33,7 +33,7 @@ usage_update, ) from pi_agent_cli.extension_trust import TRUST_ENV -from pi_agent_cli.factory import compaction_settings, model_for_choice +from pi_agent_cli.factory import auto_compact_enabled, compaction_settings, model_for_choice from pi_agent_core.event_stream import AssistantMessageEventStream from pi_agent_core.extensions import ExtensionLoader from pi_agent_core.messages import AssistantMessage, ToolResultMessage, Usage, UserMessage @@ -423,6 +423,10 @@ def kinds(self) -> list[str]: def make_config(window: int | None = 200_000, **extra: Any) -> CliConfig: + # The agent compacts by itself only when a test asks for it. A window of 40 tokens would + # otherwise compact after every answer, and the tests that count after a compaction that + # they start themselves would lose their steps. ``auto_compact=None`` is the rule of the file. + extra.setdefault("auto_compact", False) return CliConfig( permission="ask", provider="mock", model_id="mock", context_window=window, **extra ) @@ -703,7 +707,8 @@ async def test_compaction_uses_the_window_that_the_user_declared(tmp_path): async def run(window: int | None) -> bool: home = tmp_path / ("declared" if window else "undeclared") home.mkdir() - config = make_config(window=window, max_turns=30) # 30 turns or more: auto-compaction + # The rule of the file (``auto_compact=None``), and 30 turns or more: both say "on". + config = make_config(window=window, max_turns=30, auto_compact=None) script = Script(*(Step(total=900) for _ in range(3))) agent, _ = make_agent(home, script, config) session_id = (await agent.new_session(cwd=str(home))).session_id @@ -713,3 +718,141 @@ async def run(window: int | None) -> bool: assert await run(window=1_000) is True # 900 tokens is above 1,000 - 250 assert await run(window=None) is False # 900 tokens is far below 128,000 - 16,384 + + +# --- Compaction: when the agent does it by itself ----------------------------------------- + + +@pytest.mark.parametrize( + ("auto_compact", "max_turns", "window", "expected"), + [ + (None, None, None, False), # nothing says so, and the window is a guess + (None, 29, None, False), + (None, 30, None, True), # the rule from before the key existed + (None, 200, None, True), + (None, None, 1_000_000, True), # a declared window + (None, 5, 1_000_000, True), + (False, 200, 1_000_000, False), # the key wins over both rules + (False, None, None, False), + (True, None, None, True), + (True, None, 1_000_000, True), + ], +) +def test_the_rule_for_compaction_by_the_agent(auto_compact, max_turns, window, expected): + config = CliConfig(provider="mock", auto_compact=auto_compact, max_turns=max_turns) + choice = ModelChoice(id="m", context_window=window) + + assert auto_compact_enabled(config, choice) is expected + + +@pytest.mark.parametrize(("raw", "expected"), [("true", True), ("false", False)]) +def test_auto_compact_is_read_from_the_file(tmp_path, raw, expected): + config = _load(tmp_path, f"auto_compact = {raw}\n") + + assert config.auto_compact is expected + assert config.auto.notes == () + + +def test_a_file_with_no_auto_compact_leaves_it_not_set(tmp_path): + config = _load(tmp_path, "max_turns = 200\n") + + assert config.auto_compact is None + assert config.auto.notes == () + + +@pytest.mark.parametrize( + "raw", + ['"yes"', '"true"', "1", "0", "[true]"], + ids=["text", "text-true", "one", "zero", "list"], +) +def test_an_auto_compact_that_is_not_a_boolean_is_ignored_and_the_user_is_told(tmp_path, raw): + config = _load(tmp_path, f"auto_compact = {raw}\n") + + assert config.auto_compact is None + (note,) = config.auto.notes + assert note == "auto_compact in agent.toml must be true or false, and it is ignored." + + +async def _compacts(home: Path, config: CliConfig, *, total: int, size: int = 4_000) -> bool: + """Whether the agent compacted by itself, after three answers that each count ``total``. + + ``size`` is the length of each request. A window of the placeholder size keeps the last + 20,000 tokens, so there is nothing to compact until the requests are longer than that. + """ + home.mkdir(parents=True, exist_ok=True) + agent, _ = make_agent(home, Script(*(Step(total=total) for _ in range(3))), config) + session_id = (await agent.new_session(cwd=str(home))).session_id + for index in range(3): + await say(agent, session_id, f"request {index} " + "a" * size) + return any(isinstance(e, CompactionEntry) for e in await _entries(agent, session_id)) + + +@pytest.mark.asyncio +async def test_a_declared_window_turns_compaction_on_with_no_turn_limit(tmp_path): + declared = make_config(window=1_000, auto_compact=None) + undeclared = make_config(window=None, auto_compact=None) + + assert await _compacts(tmp_path / "declared", declared, total=900) is True + # No window and no turn limit: nothing is guessed, even above the placeholder of 128,000. + # The requests are long enough that a compaction would find something to compact. + guessed = await _compacts(tmp_path / "undeclared", undeclared, total=120_000, size=100_000) + assert guessed is False + + +@pytest.mark.asyncio +async def test_auto_compact_false_turns_compaction_off_for_a_long_run(tmp_path): + config = make_config(window=1_000, max_turns=200, auto_compact=False) + + assert await _compacts(tmp_path, config, total=900) is False + + +@pytest.mark.asyncio +async def test_auto_compact_true_turns_it_on_for_a_model_with_no_window(tmp_path): + """The placeholder of 128,000 tokens applies then, which is why `agent.example.toml` says to + set the real window.""" + config = make_config(window=None, auto_compact=True) + + over = await _compacts(tmp_path / "over", config, total=120_000, size=100_000) # > 111,616 + under = await _compacts(tmp_path / "under", config, total=100_000, size=100_000) + + assert (over, under) == (True, False) + + +@pytest.mark.asyncio +async def test_a_change_of_model_decides_again_whether_the_agent_compacts(tmp_path): + config = make_config( + window=None, + auto_compact=None, + models=( + ModelChoice(id="small", provider="mock", context_window=32_000), + ModelChoice(id="unknown", provider="mock"), + ), + ) + agent, _ = make_agent(tmp_path, Script(), config) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await settle(agent) + harness = agent._harnesses[session_id] + assert harness.compaction.auto_compact is False # the default model has no window + + await agent.set_config_option(config_id="model", session_id=session_id, value="small") + assert harness.compaction.auto_compact is True + + await agent.set_config_option(config_id="model", session_id=session_id, value="unknown") + assert harness.compaction.auto_compact is False + + +@pytest.mark.asyncio +async def test_a_change_of_model_keeps_what_the_key_says(tmp_path): + config = make_config( + window=None, + auto_compact=False, + models=(ModelChoice(id="small", provider="mock", context_window=32_000),), + ) + agent, _ = make_agent(tmp_path, Script(), config) + session_id = (await agent.new_session(cwd=str(tmp_path))).session_id + await settle(agent) + harness = agent._harnesses[session_id] + + await agent.set_config_option(config_id="model", session_id=session_id, value="small") + + assert harness.compaction.auto_compact is False diff --git a/pi_agent_core/agent_loop.py b/pi_agent_core/agent_loop.py index 4f6da9f7..1e21d8ea 100644 --- a/pi_agent_core/agent_loop.py +++ b/pi_agent_core/agent_loop.py @@ -16,6 +16,7 @@ ToolResultMessage, ) from pi_agent_core.types import ( + MAX_TURNS_MESSAGE, AfterToolCallContext, AfterToolCallResult, AgentContext, @@ -226,7 +227,7 @@ async def _run_loop( # the Agent wrapper turns this into an error-stop assistant message. turn_count += 1 if config.max_turns is not None and turn_count > config.max_turns: - raise MaxTurnsExceededError(f"Agent loop exceeded max_turns={config.max_turns}") + raise MaxTurnsExceededError(f"{MAX_TURNS_MESSAGE}{config.max_turns}") if not first_turn: await _emit(emit, TurnStartEvent()) diff --git a/pi_agent_core/tests/test_agent_loop.py b/pi_agent_core/tests/test_agent_loop.py index 40100d76..dfa9a542 100644 --- a/pi_agent_core/tests/test_agent_loop.py +++ b/pi_agent_core/tests/test_agent_loop.py @@ -674,7 +674,7 @@ async def noop(_id, params, signal, on_update) -> AgentToolResult: @pytest.mark.asyncio async def test_max_turns_raises(): """#2: a tool loop that never converges must stop at max_turns.""" - from pi_agent_core.types import MaxTurnsExceededError + from pi_agent_core.types import MAX_TURNS_MESSAGE, MaxTurnsExceededError ctx = AgentContext(system_prompt="", messages=[], tools=[_noop_tool()]) config = AgentLoopConfig(model=_model(), convert_to_llm=default_convert_to_llm, max_turns=3) @@ -685,7 +685,7 @@ async def emit(e): if e.type == "turn_start": turn_starts["n"] += 1 - with pytest.raises(MaxTurnsExceededError, match="max_turns=3"): + with pytest.raises(MaxTurnsExceededError, match="max_turns=3") as raised: await run_agent_loop( [UserMessage(content="go", timestamp=int(time.time() * 1000))], ctx, @@ -694,6 +694,8 @@ async def emit(e): stream_fn=_endless_tool_stream(), ) assert turn_starts["n"] == 3 + # The ACP agent finds this failure by the text (``MAX_TURNS_MESSAGE``): keep the two tied. + assert str(raised.value) == f"{MAX_TURNS_MESSAGE}3" @pytest.mark.asyncio diff --git a/pi_agent_core/types.py b/pi_agent_core/types.py index 822d1e13..0e3b5a85 100644 --- a/pi_agent_core/types.py +++ b/pi_agent_core/types.py @@ -37,6 +37,12 @@ class MaxTurnsExceededError(RuntimeError): """ +# The text of MaxTurnsExceededError, up to the number. The wrapper keeps only the text of an error +# (`errorMessage`), so a caller that must know that the limit was the reason (the ACP agent, for +# the stop reason `max_turn_requests`) finds it by this text. +MAX_TURNS_MESSAGE = "Agent loop exceeded max_turns=" + + # Extensible custom messages — apps can subclass or use TypedDict merging patterns CustomAgentMessages = dict[str, Any] AgentMessage = Message | Any From 8dc233436a4ae793d4a773d74f0a5a44e2ed270a Mon Sep 17 00:00:00 2001 From: zhangy <40223340+zy1233@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:35:21 +0800 Subject: [PATCH 4/4] feat(vscode): offer the max thinking level to an agent that knows it The agent has the thinking level `max` (b688701). An agent that does not know it reads `thinking_level = "max"` as `off` and says nothing. So the Thinking menu lists `max` only for an agent of MAX_LEVEL_SINCE (1.0.0) or later. The version is `agentInfo.version` of `initialize`. The numbers are compared as PEP 440: a pre-release of 1.0.0 (`1.0.0rc1`) and a version that cannot be read do not count. The extension read agent.toml before it started the agent. It still reads the text there (the agent reads it at its start), and now decides what the text says after `initialize`, because only then it has the version. So `max` in the file is `max` for a new agent, and `off` with the note "which the agent does not know" for an old one. The page takes `max`. The controller drops a request for `max` when the running agent does not know it. Tests: - version.test.ts: agentVersionAtLeast (numbers as numbers, pre-releases, post-releases, versions that cannot be read). - thinking.test.ts: the levels per version, read and write of `max`, and the controller with the fake agent at 0.9.1 and at 1.0.0 (menu, request, file, restart, note). - webview.test.ts: the page sends `max`. Without the guard in webview/main.ts this test fails (posted is empty). README.md and CHANGELOG.md of the extension stay as they are until the extension release: their sentences are true for the agent 0.9.1 that it bundles today. PACKAGING.md has a new step: check MAX_LEVEL_SINCE when the pin moves. --- AGENTS.md | 2 +- .../2026-11-04-vscode-extension-design.md | 8 +- .../2026-11-16-thinking-max-and-turn-limit.md | 12 ++- vscode/PACKAGING.md | 3 +- vscode/src/core/chat-controller.ts | 33 ++++-- vscode/src/core/protocol.ts | 10 +- vscode/src/core/thinking.ts | 39 ++++++- vscode/src/core/version.ts | 36 +++++++ vscode/src/webview/main.ts | 2 +- vscode/test/thinking.test.ts | 101 ++++++++++++++++-- vscode/test/version.test.ts | 51 ++++++++- vscode/test/webview.test.ts | 11 ++ 12 files changed, 277 insertions(+), 31 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 61e69326..d542a07c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ AgentMessage[] → transform_context() → convert_to_llm() → LangChain BaseMe | `packages/pi-mcp/` | Optional extension `pi-mcp-py` (entry point `pi-mcp`): connects the MCP servers (stdio, http, sse) that an ACP client passes in `session/new`, one runner task per server, tools named `mcp____`, `/mcp` command. Core, harness and CLI have no MCP client; they hand the servers over as data (`ClientMcpServer`, `ExtensionAPI.client_mcp_servers`). Design: `docs/specs/2026-10-23-mcp-extension-design.md`. Its tests are left out of a run that did not install the package (the header says so) | — | | `packaging/agent/` | Builds the agent as a one-file executable (PyInstaller; `build.py`, `zycode-agent.spec`, `entry.py`) and the smoke test over ACP with a fake model server (`smoke.py`). `.github/workflows/agent-binaries.yml` runs both on five platforms and attaches the archives to the GitHub Release after the `Release` workflow. A frozen build has no interpreter: `host.py` starts a frozen `child.py`, and only the OpenAI-format provider is in it. Design and the registry entry draft: `docs/specs/2026-10-31-agent-binaries.md` | — | | `packages/pi-agent-cli/` | Standard-ACP CLI (`python -m pi_agent_cli`); config `agent.example.toml`, home `~/.pi-python`; TUI binary `zypi` under `tui/`. See [`packages/pi-agent-cli/AGENTS.md`](packages/pi-agent-cli/AGENTS.md) for spawn, config, first run (`--setup` and the terminal login that `initialize` offers to a client that can run one; `zypi` still gets `authMethods: []`), prompt pipeline, and the three permission modes `ask`, `auto` and `always-approve` (Auto: rules, and a model step for a `bash` call that no rule decides: `auto_rules.py`, `auto_classifier.py`, `auto_mode.py`, `permission_modes.py`; design `docs/specs/2026-11-02-auto-mode-design.md`; the pager offers all three, and a clean start is `ask`). The items on the ACP wire beyond standard ACP (`pi/session/delete`, `_meta.sessionId`, and others) are in the register `docs/specs/2026-11-05-acp-extension-registry.md` | — | -| `vscode/` | The VS Code extension `zycode.zycode` (TypeScript, `src/core/` has no `vscode` import). It is an ACP client and a chat view for zycode in VS Code: it starts the agent as a child process, shows permission questions and a diff for `write`/`edit`, runs the first-run setup in a terminal (a walkthrough, a banner and hints guide the first start; the agent's own `--setup` and `auth_required` carry it, no wire item and no agent change), lists past chats of the folder in a view (`session/list`, `session/load`, `pi/session/delete`), and offers the three permission modes `ask`, `auto` and `always-approve` in the chat menu (`always-approve` only after a modal warning, never restored for a new chat, a past chat or a restart, and not in an untrusted workspace; the `zycode.permissionMode` setting stays `ask`/`auto`; it moves the agent with the agent's own `pi/yolo_mode_changed`). The extension never writes `always-approve` through `set_config_option`. A Thinking menu shows `thinking_level` of `agent.toml`; agent 0.9.1 cannot change it live, so a change edits that one line (checked by parsing, after a modal), restarts the agent and reopens the chat (design section 7c). One VSIX per platform, each with the bundled `zycode-agent` binary of one Release (pin and sha256 in `agent-assets.json`). Not a Python package: pytest does not read it, and ruff reads only its three Python helpers (`scripts/python_licenses.py`, `test/support/fake_model.py`, `test/support/pty_answers.py`). `.github/workflows/vscode-extension.yml` runs `npm run check` and a `linux-x64` VSIX smoke test for a PR that changes it. `ci.yml` (Ruff, pytest) skips a PR that touches only `vscode/`, `docs/` and that workflow, with no `.py` file (a `paths` filter with `!` patterns; no check is required on `main`, so a skipped run blocks no merge). The same workflow publishes (Trusted Publishing, by hand on `main`, environment `vscode-publish`). 1.0.0 was the first stable release (the pre-release 0.1.0 was uploaded by hand; the 1.0.0 run, run `43181052994` on 2026-11-12, was the first real OIDC publish and worked on the first try on Open VSX and the Marketplace, five targets, no token; the record is in `vscode/RELEASING.md`; the owner tested the install of 1.0.0 and the update from 0.1.0 on Windows (2026-11-12), and the rest of the Windows flow, macOS and Linux in a real VS Code are not verified; stable = even minor, pre-release = odd minor). Do not rename that file or the environment: the registries name them. Local check: `cd vscode; npm run check`. Design: `docs/specs/2026-11-04-vscode-extension-design.md`; make and test a VSIX: `vscode/PACKAGING.md`; releases: `vscode/RELEASING.md` | — | +| `vscode/` | The VS Code extension `zycode.zycode` (TypeScript, `src/core/` has no `vscode` import). It is an ACP client and a chat view for zycode in VS Code: it starts the agent as a child process, shows permission questions and a diff for `write`/`edit`, runs the first-run setup in a terminal (a walkthrough, a banner and hints guide the first start; the agent's own `--setup` and `auth_required` carry it, no wire item and no agent change), lists past chats of the folder in a view (`session/list`, `session/load`, `pi/session/delete`), and offers the three permission modes `ask`, `auto` and `always-approve` in the chat menu (`always-approve` only after a modal warning, never restored for a new chat, a past chat or a restart, and not in an untrusted workspace; the `zycode.permissionMode` setting stays `ask`/`auto`; it moves the agent with the agent's own `pi/yolo_mode_changed`). The extension never writes `always-approve` through `set_config_option`. A Thinking menu shows `thinking_level` of `agent.toml`; agent 0.9.1 cannot change it live, so a change edits that one line (checked by parsing, after a modal), restarts the agent and reopens the chat (design section 7c). The menu has `max` only for an agent of `MAX_LEVEL_SINCE` (`src/core/thinking.ts`, 1.0.0) or later, because an older agent reads `max` as `off` and says nothing; check that number when the pin moves (`vscode/PACKAGING.md`). One VSIX per platform, each with the bundled `zycode-agent` binary of one Release (pin and sha256 in `agent-assets.json`). Not a Python package: pytest does not read it, and ruff reads only its three Python helpers (`scripts/python_licenses.py`, `test/support/fake_model.py`, `test/support/pty_answers.py`). `.github/workflows/vscode-extension.yml` runs `npm run check` and a `linux-x64` VSIX smoke test for a PR that changes it. `ci.yml` (Ruff, pytest) skips a PR that touches only `vscode/`, `docs/` and that workflow, with no `.py` file (a `paths` filter with `!` patterns; no check is required on `main`, so a skipped run blocks no merge). The same workflow publishes (Trusted Publishing, by hand on `main`, environment `vscode-publish`). 1.0.0 was the first stable release (the pre-release 0.1.0 was uploaded by hand; the 1.0.0 run, run `43181052994` on 2026-11-12, was the first real OIDC publish and worked on the first try on Open VSX and the Marketplace, five targets, no token; the record is in `vscode/RELEASING.md`; the owner tested the install of 1.0.0 and the update from 0.1.0 on Windows (2026-11-12), and the rest of the Windows flow, macOS and Linux in a real VS Code are not verified; stable = even minor, pre-release = odd minor). Do not rename that file or the environment: the registries name them. Local check: `cd vscode; npm run check`. Design: `docs/specs/2026-11-04-vscode-extension-design.md`; make and test a VSIX: `vscode/PACKAGING.md`; releases: `vscode/RELEASING.md` | — | | `web/` | Product site for zycode, live at `https://zycode.dev` (static Astro, Node ≥ 22.23.4; English at `/`, Chinese at `/zh/`; Cloudflare Workers static assets, no server of ours). Not a Python package: ruff and pytest do not read it; `.github/workflows/web-ci.yml` runs `npm run check` and `npm run build` for a PR that changes `web/`. A merge to `main` that changes `web/` deploys to `zycode.dev` by itself (Cloudflare Workers Builds, not GitHub Actions). Its claims must stay equal to the README table "Status" and to the benchmark report, and `en.ts` / `zh.ts` keep the same keys. See [`web/README.md`](web/README.md) for commands, deploy, the domain, and the Cloudflare settings that are not in the repository | — | ### Invariants diff --git a/docs/specs/2026-11-04-vscode-extension-design.md b/docs/specs/2026-11-04-vscode-extension-design.md index e0c96846..6525ddce 100644 --- a/docs/specs/2026-11-04-vscode-extension-design.md +++ b/docs/specs/2026-11-04-vscode-extension-design.md @@ -231,7 +231,7 @@ All messages are JSON objects with a string `type`. The types live in one file ( | `permissionAnswer` | `requestId`, `optionId` | The user chose. The host accepts only an `optionId` that the request listed. | | `showDiff` | `toolCallId` | Open the diff. | | `setMode` | `value` | `ask`, `auto` or `always-approve`. The host sets the mode of the open session only (section 9); it does not write the setting. | -| `setThinking` | `value` | One of `xhigh`, `high`, `medium`, `low`, `minimal`, `off`. The host asks first, then changes `agent.toml` and restarts the agent (section 7c). | +| `setThinking` | `value` | One of `max`, `xhigh`, `high`, `medium`, `low`, `minimal`, `off`. The host drops `max` when the running agent does not know it. The host asks first, then changes `agent.toml` and restarts the agent (section 7c). | | `runSetup` | | Run the first-run setup in a terminal (section 7). | | `openGuide` | | Open the getting-started walkthrough (section 7a). | | `openAgentConfig` | | Open `agent.toml` of the agent (section 7a). | @@ -324,7 +324,7 @@ Nothing else is needed, so there is **no wire item to add and no agent change**. **So no live switch is possible without an agent change.** The extension adds no wire item and changes nothing in the agent. The only input that the agent has is the file, and the agent reads it at its start. What the extension does: -1. **Shows** the level that the running agent uses: the extension reads `agent.toml` just before it starts the agent (`readThinking`, a TOML parser; `smol-toml`, BSD-3-Clause, bundled and named in `THIRD-PARTY-NOTICES`). The menu has the six levels. It shows nothing until the agent is up. +1. **Shows** the level that the running agent uses: the extension reads `agent.toml` just before it starts the agent (`readThinking`, a TOML parser; `smol-toml`, BSD-3-Clause, bundled and named in `THIRD-PARTY-NOTICES`). The menu has the six levels, and `max` when the agent knows it (below). It shows nothing until the agent is up. 2. **Disables** the menu, with a reason in its tooltip, when: `reasoning = false` is set (the model does not think); an answer is running; `agent.toml` does not exist (the setup creates it) or is not valid TOML. A `thinking_level` that the agent does not know is shown as `off` with a note, as the agent reads it. 3. **Changes** the level in four steps, from the state `ready` only: (a) a modal names the file and the line, and says that the agent restarts, that the chat opens again and that the level then applies to every chat and to `zypi`; (b) the extension edits `agent.toml` (below); (c) it stops the agent and starts it again; (d) it opens the same chat with `session/load` (the agent replays the history). A chat with no message is not loaded: a new chat opens. Always approve is off again after the restart, and the notice says so. If the chat cannot be loaded, the existing path opens a new chat with a notice. 4. **The edit** (`setThinkingLevelInText`, `writeThinkingLevel`): it replaces the string of an existing top-level `thinking_level` line (comment, spacing and line ends stay), or adds the line above the first table header (above the comment lines that belong to the header; at the end if the file has no table). It is a text edit, so it is checked: both texts are parsed, and the new one must be the old one with only `thinking_level` changed. If the check fails, or the key is written in a form that the edit does not know (an array, a dotted key, an inline table), the file is not touched and a notice says to edit the line by hand. The file is written beside the file and moved over it (a symbolic link is followed, the mode is kept). The extension never creates `agent.toml`, and it never touches `local.env` (the key). @@ -333,6 +333,8 @@ Nothing else is needed, so there is **no wire item to add and no agent change**. **Tested against a fake model server** (`test/real-thinking.test.ts`, opt in, Python agent and the 0.9.0 and 0.9.1 binaries): at `low` the model request has `reasoning_effort: "low"`; after a change to `high` the agent restarts, the chat comes back with its history, and the next request has `reasoning_effort: "high"` and the earlier question; at `off` the request has no thinking parameter; a model with `reasoning = false` has no menu, and the request has no thinking parameter at `high`; a new controller (a restart of VS Code) starts at the level of the file. `test/thinking.test.ts` runs the same flows against the fake ACP agent, and the edit and the file write on their own. +**The level `max` and the version of the agent** (`docs/specs/2026-11-16-thinking-max-and-turn-limit.md`). A later agent adds `max` (`ThinkingLevel` in `pi_agent_core/types.py`). An agent that does not know a level reads it as `off` and says nothing. A menu with `max` would then tell the user that the model reasons at the highest level, while it does not reason at all. So the menu has `max` only for an agent of `MAX_LEVEL_SINCE` or later (`src/core/thinking.ts`, now 1.0.0), and the level in the file is read for that agent: `max` is `max` for an agent that knows it, and `off` with the note "which the agent does not know" for an older one. The version is `agentInfo.version` of `initialize` (a PEP 440 string: `agentVersionAtLeast` in `src/core/version.ts` compares the numbers, and a pre-release of the version itself, such as `1.0.0rc1`, does not count, because it can come from before `max`). A version that is missing or cannot be read gets the six levels. The extension reads the text of `agent.toml` before it starts the agent, and decides what the text says after `initialize`. The page takes every level (`parseWebviewMessage`); the controller drops a level that the running agent does not know (`applyThinkingLevel`). The number must be the first release that has `max`, or a later one: a number that is too low shows `max` to an agent that does not know it, and a number that is too high only hides `max`. Check it when the pin of the bundled agent moves (`vscode/PACKAGING.md`). A user of the extension 1.0.0 who writes `max` in `agent.toml` sees `off` and a note that is wrong for a new agent: that extension has six levels. + **Not verified:** a real provider. The agent passes `reasoning_effort` (the OpenAI-format provider) and the provider's own parameters for the others; the extension does not know which model of which provider accepts a level. ## 7d. The status row (after 1.0.0) @@ -571,7 +573,7 @@ From the report, section 3.3 (Phase 2 and later), and from the limits above: | D27 (adopted) | `smoke-vsix.mjs` also checks the list, load, delete and the always-approve signal on the frozen binary of each release; the vitest tests of the real agent stay opt-in. | The frozen binary is what ships; CI has no Python environment for the extension. | | D28 (adopted) | **Recommended: as built (option B).** The Thinking menu edits `thinking_level` in the user's `agent.toml`, restarts the agent, and opens the same chat again. Options: **A**, a read-only label plus the **Open agent.toml** button (the extension writes nothing; the user edits the file and presses Restart); **B**, as built (one verified line edit, after a modal); **C**, an agent change (a `thinking_level` session config option that calls `set_thinking_level` and is saved in the chat). C is the right long-term answer and needs the user's word. | Agent 0.9.0 has no live setting (section 7c). B gives the user the switch that was asked for, with the one input that the agent has. It writes a file of the user, so the modal names the file, the line and the restart. A is the zero-write fallback: remove the write path and keep the label. | | D29 (adopted) | **Recommended: as built.** No setting `zycode.thinkingLevel`. A new chat, a past chat, a restart of the agent and a restart of VS Code start at the level that `agent.toml` has. | The file is the one source, and `zypi` uses it too. A setting would be a second source that can differ from the file, and the agent cannot take a level for one session. The agent does not save a level in a chat, so a past chat has no level of its own. | -| D30 (adopted) | **Recommended: as built.** The menu has the six levels of the agent (`xhigh`, `high`, `medium`, `low`, `minimal`, `off`), in the order of the `zypi` menu (strongest first), with the names and the descriptions of `zypi`. The list is the same for every model. | The agent lists no levels for a model; the pager's fallback menu is the only precedent. `minimal` and `off` are accepted by the agent. | +| D30 (adopted) | **Recommended: as built.** The menu has the six levels of the agent (`xhigh`, `high`, `medium`, `low`, `minimal`, `off`), in the order of the `zypi` menu (strongest first), with the names and the descriptions of `zypi`. The list is the same for every model. For an agent that knows `max`, the menu has `max` on top (section 7c). | The agent lists no levels for a model; the pager's fallback menu is the only precedent. `minimal` and `off` are accepted by the agent. | | D31 (adopted) | **Recommended: as built.** A model without thinking is a model with `reasoning = false` in `agent.toml`: the menu is off and its tooltip says why. With `reasoning` unset the menu is on. | The agent does not say it over ACP, and `reasoning` is the agent's own switch (`Model.reasoning`). A guess from the model id would be wrong for gateways. | | D32 (adopted) | **Recommended: as built.** The edit changes one top-level line, keeps comments and line ends, is checked by parsing both texts (`smol-toml`, BSD-3-Clause, a build-time dependency that is bundled), refuses a form it does not know, and writes through a temporary file. A modal asks first. | A wrong edit of a config file is worse than no edit. A regular expression alone is a guess about the layout; the parse of both texts is the proof. | | D33 (adopted) | **Recommended: as built.** The change is possible only while no answer runs. After the restart the same chat is opened with `session/load` (history replayed); a chat with no message gets a new chat; Always approve is off again, and a notice says so. | A restart ends a running turn. `session/load` keeps the history of the chat (D21 to D24). Always approve is never restored (D19). | diff --git a/docs/specs/2026-11-16-thinking-max-and-turn-limit.md b/docs/specs/2026-11-16-thinking-max-and-turn-limit.md index a195e67d..f0d917e4 100644 --- a/docs/specs/2026-11-16-thinking-max-and-turn-limit.md +++ b/docs/specs/2026-11-16-thinking-max-and-turn-limit.md @@ -1,6 +1,6 @@ # The `max` thinking level and the turn limit: investigation record -Date: 2026-11-16. Status: built in the branch of PR #131, as separate commits. Not released. +Date: 2026-11-16. Status: built in the branch of PR #131, as separate commits (the agent, then the extension). Not released. ## 1. Question @@ -39,6 +39,7 @@ The owner asked for two small changes: 1. The value in `agent.example.toml` is 200 (it was 50), with a comment that says it guards against a loop that does not end. The default in the code stays `None`: a finite default would put a limit that nobody can see on every user who has no `max_turns`, and the `--setup` users have none. 2. A key `auto_compact`, so that compaction does not depend on `max_turns` (section 4). 3. The stop reason `max_turn_requests`. +3. The VS Code extension gets `max` in the same PR, as its own commit (the owner, later the same day). The menu offers `max` only to an agent that knows it (section 4). ## 4. What was built @@ -47,6 +48,7 @@ The owner asked for two small changes: - `auto_compact` is a key at the top of `agent.toml` (`true` or `false`; another type is ignored, with a note). `factory.auto_compact_enabled` decides. The key wins. When it is not set, compaction is on for a model with a declared `context_window` and for `max_turns >= 30`, and off otherwise. After a change of model the rule decides again. `auto_compact = true` for a model with no window assumes 128,000 tokens. - `MAX_TURNS_MESSAGE` (`pi_agent_core/types.py`) is the one source of the text of `MaxTurnsExceededError`. `_stop_reason` answers `max_turn_requests` when the text of an error holds it. Any other error is still `end_turn`. - `agent.example.toml`: `max_turns = 200`, the text for `auto_compact`, and `context_window = 1000000` as the example. It was 128000, which is wrong for a model with a window of 1M. +- The VS Code extension (`vscode/`, its own commit). `ThinkingLevel` has `max`, and the menu lists it first, but only for an agent of `MAX_LEVEL_SINCE` (`src/core/thinking.ts`, 1.0.0) or later. The version is `agentInfo.version` of `initialize`. `agentVersionAtLeast` (`src/core/version.ts`) compares it as PEP 440: a pre-release of 1.0.0 (`1.0.0rc1`) and a version that cannot be read do not count. The extension reads `agent.toml` before it starts the agent and decides what the text says after `initialize`. So `max` in the file is `max` for a new agent, and `off` with the note "which the agent does not know" for an old one. The page takes `max` (`parseWebviewMessage`), and the controller drops a request for `max` when the running agent does not know it (`applyThinkingLevel`). Design: section 7c of `docs/specs/2026-11-04-vscode-extension-design.md`. Release step: `vscode/PACKAGING.md`, "Change the agent version", step 3. Tests: `test_thinking.py`, `test_deepseek_thinking.py`, `test_deepseek_replay.py`, `test_config.py`, `test_model_reasoning.py`, `test_agent_loop.py`, `test_acp_agent.py` and `test_context_usage.py`. Five rules were broken one at a time (the stop reason branch, the window rule, the key rule, the decision after a change of model, and the text of the error): each break made a test fail. @@ -54,19 +56,21 @@ Tests: `test_thinking.py`, `test_deepseek_thinking.py`, `test_deepseek_replay.py - **A declared window turns compaction on.** A user who has `context_window` and no `max_turns` did not compact before. This is on purpose: the agent knows when the context is full. `auto_compact = false` turns it off. `context_window` is not released yet (PR #131), so nobody depends on the old behaviour. - **`max` is open for every model.** A model that does not take it answers with an error, and nothing is lowered. Plan B would lower it. -- **Old agents.** An agent that does not know `max` takes `thinking_level = "max"` as `off`, and it gives no note. Release the agent first. Then add `max` to the menu of the extension, only for an agent that knows it. +- **Old agents.** An agent that does not know `max` takes `thinking_level = "max"` as `off`, and it gives no note. The extension of this PR does not offer `max` to such an agent (section 4). But `MAX_LEVEL_SINCE` is a statement about a release that is not made yet (1.0.0): it must be the first release that has `max`, or a later one. A number that is too low shows `max` to an agent that reads it as `off`. A number that is too high only hides `max`. If PR #130 releases 1.0.0 without this PR, change the number before the extension release that bundles that agent. +- **The extension 1.0.0 and `max` in the file.** The extension reads `thinking_level` itself (`vscode/src/core/thinking.ts`) and has six levels. For a user who writes `max` by hand, its menu shows `off`, and a note says that the agent does not know `max` (`chat-controller.ts`). For the new agent that note is wrong: the agent uses `max`. Nothing breaks. The note goes away when the extension knows `max`. - **A client sees `max_turn_requests`, not `end_turn`.** The extension shows a line of notice. `zypi` takes it as "done". Another client may show it as a failure: not checked. ## 6. Not verified - A live gateway or provider that gets `reasoning_effort = "max"` (a gateway for DeepSeek V4, and api.openai.com). - The Anthropic path (no `langchain_anthropic` in `.venv`), and the claim about Claude 4.7 and later (it comes from the documentation). -- The VS Code extension with `thinking_level = "max"` in the file. Its menu has six levels. +- The VS Code extension in a real VS Code window, with a real agent that has `max`. The tests use a fake agent that reports a version. +- The behaviour of the extension 1.0.0 with `max` in the file (section 5) is read from its source (`thinking.ts`, `chat-controller.ts`). It was not run. - What happens when a session with compaction off reaches the limit of the model. ## 7. Open -1. The extension: add `max` to the Thinking menu, only when `agentInfo.version` is new enough. After the agent release. +1. The extension release: move the pin (`vscode/PACKAGING.md`), check `MAX_LEVEL_SINCE`, and change the sentences that say the menu has six levels (`vscode/README.md`, `vscode/CHANGELOG.md`). They stay as they are until then, because they are true for the agent 0.9.1 that the extension bundles today. 2. Anthropic: adaptive thinking with `effort` for new models, and a higher floor for `langchain-anthropic`. A separate change. 3. Plan B: a table of levels per model, as in `pi`, so that a level that a model does not take is lowered, and a menu can hide it. 4. A compaction threshold of its own. With a window of 1M, compaction starts above 983,616 tokens, which can be late (cost, quality). Not investigated. diff --git a/vscode/PACKAGING.md b/vscode/PACKAGING.md index 83ca9c3b..614d6f13 100644 --- a/vscode/PACKAGING.md +++ b/vscode/PACKAGING.md @@ -46,4 +46,5 @@ Check before you upload a file made on your machine: `sha256sum -c out/SHA256SUM 1. Publish the agent Release first (`agent-binaries.yml`). 2. Change `zycodeAgentVersion` in `package.json`, and `agent-assets.json` (version, tag, `pythonPackages`, and for each target `size` and `sha256` of the asset). `test/packaging.test.ts` fails when these disagree. -3. Make the files again. +3. Check `MAX_LEVEL_SINCE` in `src/core/thinking.ts`. It must be the first agent release that has the thinking level `max` (`ThinkingLevel` in `pi_agent_core/types.py`), or a later one. The Thinking menu offers `max` to every agent of that version or later. A number that is too low shows `max` to an agent that reads it as `off` and says nothing. +4. Make the files again. diff --git a/vscode/src/core/chat-controller.ts b/vscode/src/core/chat-controller.ts index c78fa8be..1adab987 100644 --- a/vscode/src/core/chat-controller.ts +++ b/vscode/src/core/chat-controller.ts @@ -50,10 +50,10 @@ import { } from "./session-list.js"; import { readUsageUpdate, sameUsage } from "./status-row.js"; import { - THINKING_LEVELS, isThinkingLevel, noThinkingReason, readThinking, + thinkingLevelsFor, writeThinkingLevel, type ThinkingRead, } from "./thinking.js"; @@ -103,6 +103,9 @@ interface TerminalAuthMethod { env: Record; } +/** What was read from `agent.toml` before the agent started: its text, or why there is none. */ +type ThinkingText = { ok: true; text: string } | { ok: false; reason: string }; + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -151,6 +154,9 @@ export class ChatController { private deleteSupported = true; private refreshing: Promise | undefined; private refreshAgain = false; + /** `agent.toml` as it was just before the agent started (the agent reads it at its start). */ + private thinkingText: ThinkingText | undefined; + /** What that text says for the agent that runs. It is set after `initialize`: it needs the version. */ private thinkingFile: ThinkingRead | undefined; private reopenId: string | undefined; private changingThinking = false; @@ -241,7 +247,7 @@ export class ChatController { const file = this.thinkingFile; if (!file) return undefined; if (this.status !== "ready" && this.status !== "busy" && this.status !== "cancelling") return undefined; - const levels = [...THINKING_LEVELS]; + const levels = [...this.thinkingLevels()]; if (!file.ok) return { levels, disabled: file.reason }; const state: ThinkingState = { current: file.value.level, levels }; if (file.value.invalidLevel !== undefined) { @@ -254,10 +260,22 @@ export class ChatController { return state; } - private readThinkingFile(): ThinkingRead { + /** The levels that the running agent knows. Its version comes from `initialize`; before that, the levels of every agent. */ + private thinkingLevels() { + return thinkingLevelsFor(this.initResult?.agentInfo?.version); + } + + /** What the text read before the start says, for the agent that runs. It needs the version of the agent, so it runs after `initialize`. */ + private parseThinkingText(): ThinkingRead | undefined { + const read = this.thinkingText; + if (!read) return undefined; + return read.ok ? readThinking(read.text, this.thinkingLevels()) : read; + } + + private readThinkingText(): ThinkingText { const file = this.host.agentConfigPath(); try { - return readThinking(readFileSync(file, "utf8")); + return { ok: true, text: readFileSync(file, "utf8") }; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") { return { ok: false, reason: "agent.toml does not exist yet, so there is no thinking level to set. Run the setup first." }; @@ -274,7 +292,8 @@ export class ChatController { async applyThinkingLevel(level: ThinkingLevel): Promise { if (this.changingThinking) return; const state = this.thinkingState(); - if (!state || !isThinkingLevel(level)) return; + // Only a level that the running agent knows: the page cannot tell which these are. + if (!state || !isThinkingLevel(level, this.thinkingLevels())) return; if (state.disabled !== undefined) { this.pushNotice("info", `The thinking level cannot be changed now: ${state.disabled}`, [{ id: "openAgentConfig", label: "Open agent.toml" }]); this.emit(this.stateMessage()); @@ -414,7 +433,8 @@ export class ChatController { return; } this.command = resolved.command; - this.thinkingFile = this.readThinkingFile(); + this.thinkingText = this.readThinkingText(); + this.thinkingFile = undefined; this.setStatus("starting"); const generation = ++this.generation; try { @@ -437,6 +457,7 @@ export class ChatController { const init = await this.withTimeout(conn.initialize(this.host.extensionVersion), proc); if (generation !== this.generation) return; this.initResult = init; + this.thinkingFile = this.parseThinkingText(); const verdict = checkAgentIdentity({ protocolVersion: init.protocolVersion, agentName: init.agentInfo?.name, diff --git a/vscode/src/core/protocol.ts b/vscode/src/core/protocol.ts index 20d14819..4ca3fe03 100644 --- a/vscode/src/core/protocol.ts +++ b/vscode/src/core/protocol.ts @@ -33,8 +33,11 @@ export interface ModeState { options: { id: string; name: string }[]; } -/** The levels of `thinking_level` in agent.toml. */ -export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh"; +/** + * The levels of `thinking_level` in agent.toml. An agent knows `max` only from `MAX_LEVEL_SINCE` on + * (`thinking.ts`): the menu gets the list for the agent that runs, and this type has all of them. + */ +export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; export interface ThinkingLevelOption { id: ThinkingLevel; @@ -232,7 +235,8 @@ export function parseWebviewMessage(value: unknown): WebviewMessage | undefined value.value !== "low" && value.value !== "medium" && value.value !== "high" && - value.value !== "xhigh" + value.value !== "xhigh" && + value.value !== "max" ) { return undefined; } diff --git a/vscode/src/core/thinking.ts b/vscode/src/core/thinking.ts index f363467b..61ffd1f2 100644 --- a/vscode/src/core/thinking.ts +++ b/vscode/src/core/thinking.ts @@ -5,6 +5,8 @@ * What agent v0.9.1 offers (read from its source and measured on the 0.9.0 and 0.9.1 binaries): * - The level lives in `agent.toml` only: the top-level key `thinking_level`, one of `off`, * `minimal`, `low`, `medium`, `high`, `xhigh`. A missing or invalid value is `off`. + * A later agent adds `max` (`MAX_LEVEL_SINCE`). An agent that does not know `max` reads it as `off` + * and says nothing, so the menu has `max` only for an agent that knows it (`thinkingLevelsFor`). * - The agent reads it when the process starts. `session/new` has no thinking option or `_meta`, * `session/set_config_option` refuses any id but `model` and `mode`, and the agent has no * custom method for it. The harness has `set_thinking_level`, but the ACP agent never calls it. @@ -24,9 +26,23 @@ import path from "node:path"; import { parse as parseToml } from "smol-toml"; import type { ThinkingLevel, ThinkingLevelOption } from "./protocol.js"; +import { agentVersionAtLeast } from "./version.js"; -/** The order of the pager menu (`effort_levels.rs`): strongest first. The ids are the agent's. */ +/** + * The first agent release that knows the level `max` (`ThinkingLevel` in `pi_agent_core/types.py`). + * An agent before it reads `max` as `off` and says nothing. The user would then think that the model + * reasons at the highest level, while it does not reason at all. So the menu has `max` only for an + * agent of this version or later. + * + * This number must be the first release that has `max`, or a later one. A number that is too low shows + * `max` to an agent that does not know it. A number that is too high only hides `max`. Check it when + * the pin of the bundled agent moves (`vscode/PACKAGING.md`, "Change the agent version"). + */ +export const MAX_LEVEL_SINCE = "1.0.0"; + +/** Every level that an agent can know. The order is the one of the pager menu (`effort_levels.rs`): strongest first. The ids are the agent's. */ export const THINKING_LEVELS: readonly ThinkingLevelOption[] = [ + { id: "max", name: "max", description: "Maximum reasoning" }, { id: "xhigh", name: "xhigh", description: "Extended reasoning" }, { id: "high", name: "high", description: "Heavy reasoning" }, { id: "medium", name: "medium", description: "Balanced reasoning" }, @@ -35,8 +51,17 @@ export const THINKING_LEVELS: readonly ThinkingLevelOption[] = [ { id: "off", name: "off", description: "No reasoning" }, ]; -export function isThinkingLevel(value: unknown): value is ThinkingLevel { - return typeof value === "string" && THINKING_LEVELS.some((level) => level.id === value); +/** + * The levels that an agent of this version knows, strongest first. `agentVersion` is `agentInfo.version`. + * A version that is missing or cannot be read gets the levels that every agent knows. + */ +export function thinkingLevelsFor(agentVersion: string | undefined): readonly ThinkingLevelOption[] { + return agentVersionAtLeast(agentVersion, MAX_LEVEL_SINCE) ? THINKING_LEVELS : THINKING_LEVELS.filter((level) => level.id !== "max"); +} + +/** Whether `value` is one of `levels`. With no `levels`, the ones that every agent knows. */ +export function isThinkingLevel(value: unknown, levels: readonly ThinkingLevelOption[] = thinkingLevelsFor(undefined)): value is ThinkingLevel { + return typeof value === "string" && levels.some((level) => level.id === value); } /** What `agent.toml` says about thinking. */ @@ -70,7 +95,11 @@ function stripBom(text: string): string { return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; } -export function readThinking(text: string): ThinkingRead { +/** + * What `text` (the content of `agent.toml`) says about thinking. `levels` are the levels that the + * agent knows (`thinkingLevelsFor`): any other value is `invalidLevel`, and the agent uses `off`. + */ +export function readThinking(text: string, levels: readonly ThinkingLevelOption[] = thinkingLevelsFor(undefined)): ThinkingRead { let data: Record; try { data = parseToml(stripBom(text)); @@ -80,7 +109,7 @@ export function readThinking(text: string): ThinkingRead { const raw = data.thinking_level; const value: ThinkingFile = { level: "off", reasoning: undefined }; if (raw !== undefined) { - if (isThinkingLevel(raw)) value.level = raw; + if (isThinkingLevel(raw, levels)) value.level = raw; else value.invalidLevel = typeof raw === "string" ? raw : typeof raw; } const model = isTable(data.model) ? data.model : {}; diff --git a/vscode/src/core/version.ts b/vscode/src/core/version.ts index 600e6ac7..c26e11ff 100644 --- a/vscode/src/core/version.ts +++ b/vscode/src/core/version.ts @@ -8,6 +8,42 @@ export type VersionVerdict = | { kind: "warn"; text: string } | { kind: "fail"; text: string }; +/** + * What can follow the release numbers of a version that is a final release: more numbers + * (`1.0.0.1`), a post-release (`.post1`) and a local part (`+local`). A pre-release (`rc1`, `a1`, + * `.dev0`) and any text that is not a PEP 440 suffix do not match. + */ +const FINAL_SUFFIX = /^(?:\.\d+)*(?:[.\-_]?post[.\-_]?\d*)?(?:\+[0-9a-z.\-_]+)?$/i; + +/** + * Whether `version` is `min` or later. `version` is `agentInfo.version` of the agent: the version of + * the Python package, written as PEP 440 (`1.0.0`, `1.0.0rc1`, `1.0.1.dev3`, `0.0.0+unknown`). `min` + * is a plain release (`1.0.0`). + * + * The answer is `false` when it is not sure. A version that cannot be read, a pre-release of `min` + * (it may come from before the change that `min` stands for), and a version with a suffix that is not + * PEP 440 are all `false`. A pre-release of a later release (`1.0.1rc1`) is `true`. + */ +export function agentVersionAtLeast(version: string | undefined, min: string): boolean { + const have = parseRelease(version); + const need = parseRelease(min); + if (!have || !need) return false; + for (let i = 0; i < 3; i++) { + if (have.numbers[i] !== need.numbers[i]) return have.numbers[i]! > need.numbers[i]!; + } + return FINAL_SUFFIX.test(have.suffix); +} + +/** The first three numbers of a version (a missing one is 0), and the text after them. */ +function parseRelease(text: string | undefined): { numbers: [number, number, number]; suffix: string } | undefined { + const match = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(text ?? ""); + if (!match) return undefined; + return { + numbers: [Number(match[1]), Number(match[2] ?? 0), Number(match[3] ?? 0)], + suffix: (text ?? "").slice(match[0].length), + }; +} + /** * Checks what `initialize` returned (docs/specs/2026-11-04-vscode-extension-design.md, section 3.5). * A program that does not speak the protocol fails. A bundled binary of another version warns. diff --git a/vscode/src/webview/main.ts b/vscode/src/webview/main.ts index a261d8c8..88823e8d 100644 --- a/vscode/src/webview/main.ts +++ b/vscode/src/webview/main.ts @@ -216,7 +216,7 @@ function renderHeader(): void { select.disabled = thinking.disabled !== undefined; select.addEventListener("change", () => { const value = select.value; - if (value === "off" || value === "minimal" || value === "low" || value === "medium" || value === "high" || value === "xhigh") { + if (value === "off" || value === "minimal" || value === "low" || value === "medium" || value === "high" || value === "xhigh" || value === "max") { send({ type: "setThinking", value }); } }); diff --git a/vscode/test/thinking.test.ts b/vscode/test/thinking.test.ts index 881cfe3b..e1b60fc7 100644 --- a/vscode/test/thinking.test.ts +++ b/vscode/test/thinking.test.ts @@ -4,7 +4,16 @@ import { afterEach, describe, expect, it } from "vitest"; import type { ChatController } from "../src/core/chat-controller.js"; import { parseWebviewMessage, type HostMessage } from "../src/core/protocol.js"; -import { THINKING_LEVELS, noThinkingReason, readThinking, setThinkingLevelInText, writeThinkingLevel } from "../src/core/thinking.js"; +import { + MAX_LEVEL_SINCE, + THINKING_LEVELS, + noThinkingReason, + readThinking, + setThinkingLevelInText, + thinkingLevelsFor, + writeThinkingLevel, +} from "../src/core/thinking.js"; +import { agentVersionAtLeast } from "../src/core/version.js"; import { TestHost, newController, ofType, readLog, tempDir, waitFor } from "./helpers.js"; const MODEL = '[model]\nprovider = "openai"\nid = "m"\n'; @@ -16,8 +25,27 @@ function edited(text: string, level: Parameters[1 } describe("reading thinking from agent.toml", () => { - it("lists the levels of the agent, strongest first, as the pager does", () => { - expect(THINKING_LEVELS.map((level) => level.id)).toEqual(["xhigh", "high", "medium", "low", "minimal", "off"]); + it("lists every level that an agent can know, strongest first, as the pager does", () => { + expect(THINKING_LEVELS.map((level) => level.id)).toEqual(["max", "xhigh", "high", "medium", "low", "minimal", "off"]); + }); + + it("offers max only to an agent that knows it", () => { + const ids = (version: string | undefined) => thinkingLevelsFor(version).map((level) => level.id); + const six = ["xhigh", "high", "medium", "low", "minimal", "off"]; + expect(ids("0.9.1")).toEqual(six); + expect(ids(MAX_LEVEL_SINCE)).toEqual(["max", ...six]); + expect(ids("99.0.0")).toEqual(["max", ...six]); + // A version that is missing or cannot be read gets the six levels that every agent knows. + expect(ids(undefined)).toEqual(six); + expect(ids("unknown")).toEqual(six); + expect(ids("0.0.0+unknown")).toEqual(six); + }); + + it("keeps the number that max starts at above the agents that were released without max", () => { + // 0.9.1 is the agent that the extension bundles today. It does not know max (it reads it as off). + expect(agentVersionAtLeast("0.9.1", MAX_LEVEL_SINCE)).toBe(false); + // A plain release number: a pre-release text here would make every comparison false. + expect(MAX_LEVEL_SINCE).toMatch(/^\d+\.\d+\.\d+$/); }); it("reads the level and says off when it is missing, as the agent does", () => { @@ -30,6 +58,14 @@ describe("reading thinking from agent.toml", () => { expect(readThinking("thinking_level = 3\n")).toEqual({ ok: true, value: { level: "off", reasoning: undefined, invalidLevel: "number" } }); }); + it("reads max for an agent that knows it, and reports it for an agent that does not, which reads it as off", () => { + const text = `thinking_level = "max"\n${MODEL}`; + expect(readThinking(text, thinkingLevelsFor(MAX_LEVEL_SINCE))).toEqual({ ok: true, value: { level: "max", reasoning: undefined } }); + expect(readThinking(text, thinkingLevelsFor("0.9.1"))).toEqual({ ok: true, value: { level: "off", reasoning: undefined, invalidLevel: "max" } }); + // With no levels given, the answer is the one for an agent that does not know max. + expect(readThinking(text)).toMatchObject({ ok: true, value: { level: "off", invalidLevel: "max" } }); + }); + it("reads reasoning from [model] first, then from the top level, like CliConfig", () => { expect(readThinking('reasoning = true\n[model]\nreasoning = false\n')).toMatchObject({ ok: true, value: { reasoning: false } }); expect(readThinking("reasoning = false\n")).toMatchObject({ ok: true, value: { reasoning: false } }); @@ -67,6 +103,11 @@ describe("the edit of thinking_level", () => { expect(edited("", "high")).toBe('thinking_level = "high"\n'); }); + it("writes max like any other level", () => { + expect(edited(`thinking_level = "high"\n${MODEL}`, "max")).toBe(`thinking_level = "max"\n${MODEL}`); + expect(edited(MODEL, "max")).toBe(`thinking_level = "max"\n\n${MODEL}`); + }); + it("keeps Windows line ends", () => { const next = edited(`max_turns = 5\r\n${MODEL.replace(/\n/g, "\r\n")}`, "high"); expect(next.replace(/\r\n/g, "")).not.toContain("\n"); @@ -140,11 +181,15 @@ describe("writing the file", () => { }); describe("the message of the page", () => { - it("takes the six levels and nothing else", () => { - for (const value of ["off", "minimal", "low", "medium", "high", "xhigh"]) { + it("takes the seven levels and nothing else (the controller checks the level against the agent)", () => { + for (const value of ["off", "minimal", "low", "medium", "high", "xhigh", "max"]) { expect(parseWebviewMessage({ type: "setThinking", value })).toEqual({ type: "setThinking", value }); } - expect(parseWebviewMessage({ type: "setThinking", value: "max" })).toBeUndefined(); + // The page takes every level that the menu can list. + for (const level of THINKING_LEVELS) { + expect(parseWebviewMessage({ type: "setThinking", value: level.id })).toEqual({ type: "setThinking", value: level.id }); + } + expect(parseWebviewMessage({ type: "setThinking", value: "ultra" })).toBeUndefined(); expect(parseWebviewMessage({ type: "setThinking" })).toBeUndefined(); }); }); @@ -223,6 +268,50 @@ describe("the thinking picker in the controller", () => { expect(thinkingOf(controller)?.disabled).toBeUndefined(); }); + it("offers no max to an agent that is older than MAX_LEVEL_SINCE, and does not write it", async () => { + // 0.9.1 reads max as off and says nothing, so a menu with max would lie. + const { controller, host, toml } = rig({ env: { FAKE_AGENT_VERSION: "0.9.1" } }); + await controller.start(); + expect(thinkingOf(controller)?.levels.map((l) => l.id)).toEqual(["xhigh", "high", "medium", "low", "minimal", "off"]); + // The page cannot tell which levels the agent knows: a message for max is dropped here. + await controller.applyThinkingLevel("max"); + expect(host.confirmations).toHaveLength(0); + expect(readFileSync(toml, "utf8")).toContain('thinking_level = "low"'); + }); + + it("offers max, first in the list, to an agent that knows it, and sets it", async () => { + const { controller, host, toml, log, messages } = rig({ env: { FAKE_AGENT_VERSION: MAX_LEVEL_SINCE } }); + await controller.start(); + expect(thinkingOf(controller)?.levels.map((l) => l.id)).toEqual(["max", "xhigh", "high", "medium", "low", "minimal", "off"]); + + await controller.applyThinkingLevel("max"); + + expect(host.confirmations).toHaveLength(1); + expect(host.confirmations[0]?.detail).toContain('thinking_level = "max"'); + expect(readFileSync(toml, "utf8")).toBe(`thinking_level = "max"\n${MODEL}`); + expect(starts(log)).toEqual(["start thinking_level=low", "start thinking_level=max"]); + expect(thinkingOf(controller)).toMatchObject({ current: "max" }); + expect(thinkingOf(controller)?.note).toBeUndefined(); + expect(notices(messages).at(-1)).toMatch(/Thinking level: max\./); + await say(controller, "level"); + expect(agentText(controller)).toContain("level:max"); + }); + + it("shows max from the file for an agent that knows it", async () => { + // The file is read before the agent starts, and the version comes with `initialize`: the two meet here. + const { controller } = rig({ toml: 'thinking_level = "max"\n', env: { FAKE_AGENT_VERSION: MAX_LEVEL_SINCE } }); + await controller.start(); + expect(thinkingOf(controller)).toMatchObject({ current: "max" }); + expect(thinkingOf(controller)?.note).toBeUndefined(); + }); + + it("says that an agent older than MAX_LEVEL_SINCE does not know the max in the file, and shows off", async () => { + const { controller } = rig({ toml: 'thinking_level = "max"\n', env: { FAKE_AGENT_VERSION: "0.9.1" } }); + await controller.start(); + expect(thinkingOf(controller)).toMatchObject({ current: "off" }); + expect(thinkingOf(controller)?.note).toBe('thinking_level in agent.toml is "max", which the agent does not know. It uses off.'); + }); + it("is disabled, with the reason, for a model that has reasoning = false", async () => { const { controller, host, toml } = rig({ toml: `thinking_level = "high"\n[model]\nreasoning = false\n` }); await controller.start(); diff --git a/vscode/test/version.test.ts b/vscode/test/version.test.ts index fa063d3c..c20686e9 100644 --- a/vscode/test/version.test.ts +++ b/vscode/test/version.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { checkAgentIdentity } from "../src/core/version.js"; +import { agentVersionAtLeast, checkAgentIdentity } from "../src/core/version.js"; const ok = { protocolVersion: 1, agentName: "pi-agent-cli", agentVersion: "0.8.0", source: "bundled" as const, expectedVersion: "0.8.0" }; @@ -26,3 +26,52 @@ describe("checkAgentIdentity", () => { expect(checkAgentIdentity({ ...ok, agentName: undefined, source: "setting" }).kind).toBe("fail"); }); }); + +describe("agentVersionAtLeast", () => { + const atLeast = (version: string | undefined) => agentVersionAtLeast(version, "1.0.0"); + + it("compares the numbers as numbers, not as text", () => { + expect(atLeast("0.9.1")).toBe(false); + expect(atLeast("0.10.0")).toBe(false); + expect(atLeast("1.0.0")).toBe(true); + expect(atLeast("1.0.1")).toBe(true); + expect(atLeast("1.2.0")).toBe(true); + expect(atLeast("2.0.0")).toBe(true); + expect(atLeast("10.0.0")).toBe(true); + expect(agentVersionAtLeast("1.10.0", "1.9.0")).toBe(true); + expect(agentVersionAtLeast("1.9.0", "1.10.0")).toBe(false); + }); + + it("reads a version with fewer than three numbers, and one with more", () => { + expect(atLeast("1.0")).toBe(true); + expect(atLeast("1")).toBe(true); + expect(atLeast("0.9")).toBe(false); + expect(atLeast("1.0.0.1")).toBe(true); + }); + + it("says no to a pre-release of the version itself, which can come from before the change", () => { + for (const version of ["1.0.0rc1", "1.0.0a1", "1.0.0b2", "1.0.0.dev0", "1.0.0.dev3", "1.0.dev0"]) { + expect(atLeast(version), version).toBe(false); + } + }); + + it("says yes to a pre-release of a later version, to a post-release and to a local version", () => { + expect(atLeast("1.0.1rc1")).toBe(true); + expect(atLeast("1.1.0.dev2")).toBe(true); + expect(atLeast("1.0.0.post1")).toBe(true); + expect(atLeast("1.0.0+local")).toBe(true); + expect(atLeast("1.0.0+abc.5")).toBe(true); + }); + + it("says no when it cannot read the version", () => { + for (const version of [undefined, "", "unknown", "v1.0.0", "0.0.0+unknown", "a.b.c", "1.0.0!"]) { + expect(atLeast(version), String(version)).toBe(false); + } + }); + + it("says no when the version after the numbers is not a PEP 440 suffix", () => { + expect(atLeast("1.0.0-beta")).toBe(false); + expect(atLeast("1.0.0abc")).toBe(false); + expect(atLeast("1!1.0.0")).toBe(false); + }); +}); \ No newline at end of file diff --git a/vscode/test/webview.test.ts b/vscode/test/webview.test.ts index 4b321f56..ed9e3903 100644 --- a/vscode/test/webview.test.ts +++ b/vscode/test/webview.test.ts @@ -362,6 +362,17 @@ describe.skipIf(!chrome || !built)("webview: modes, first run and notices", () = }); }); + it("shows max when the host lists it, and sends it", () => { + const withMax = `[{ id: "max", name: "max", description: "Maximum reasoning" }, ${levels.slice(1)}`; + const out = run(`${state} + state({ mode: { current: "ask", options: ${modes} }, thinking: { current: "low", levels: ${withMax} } }); + const select = document.querySelector(".thinking select"); + const options = [...select.options].map((o) => o.value); + select.value = "max"; select.dispatchEvent(new Event("change")); + window.__result = { options, posted: window.__posted.filter((m) => m.type === "setThinking") };`); + expect(out).toMatchObject({ ok: true, options: ["max", "xhigh", "high", "low", "off"], posted: [{ type: "setThinking", value: "max" }] }); + }); + it("disables the picker and gives the reason for a model that does not think", () => { const reason = "agent.toml sets reasoning = false for the model."; const out = run(`${state}