From 9ab8c3acb0747cf99afb63e78ce1a2333ee70483 Mon Sep 17 00:00:00 2001 From: aaron Date: Tue, 20 Oct 2026 14:56:58 -0700 Subject: [PATCH 1/2] Make openrouter the default provider Settings.provider and Settings.from_env now default to "openrouter" when LLM_PROVIDER is unset. Breaking: callers relying on the anthropic default must set LLM_PROVIDER=anthropic or Settings(provider="anthropic"). Co-Authored-By: Claude Sonnet 5.5 --- .env.example | 2 +- CHANGELOG.md | 4 ++++ README.md | 2 +- docs/api/config.md | 2 +- docs/getting-started/configuration.md | 2 +- src/agentloop/config.py | 4 ++-- tests/test_config.py | 2 +- 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index b330bb6..07e425f 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ # Read by the CLI only. The library never reads this file. # One of: anthropic | openrouter | modal | openai -LLM_PROVIDER=anthropic +LLM_PROVIDER=openrouter ANTHROPIC_API_KEY= ANTHROPIC_MODEL=claude-opus-5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 96aeda7..0190958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,10 @@ while `0.x`, a minor bump may break the public API (`agentloop.__all__`). o-series reasoning models no longer receive `parallel_tool_calls`, which they reject. ### Changed +- **Breaking:** the default provider is now `openrouter`, not `anthropic`, for `Settings.provider` + and `Settings.from_env` when `LLM_PROVIDER` is unset. Set `LLM_PROVIDER=anthropic` (or + `Settings(provider="anthropic")`) to keep the old behavior; otherwise the missing + `OPENROUTER_API_KEY` raises `AuthError`. - Resuming a transcript that ends in unanswered tool calls (the process died while tools ran) runs those calls before the first model call instead of sending an invalid request. Any other invalid transcript raises `TranscriptError` before a model call, not a provider 400. diff --git a/README.md b/README.md index 45fef6f..2e9e303 100644 --- a/README.md +++ b/README.md @@ -285,7 +285,7 @@ and the rest of the package. | Variable | Default | Purpose | |---|---|---| -| `LLM_PROVIDER` | `anthropic` | `anthropic`, `openai`, `openrouter`, or `modal` | +| `LLM_PROVIDER` | `openrouter` | `anthropic`, `openai`, `openrouter`, or `modal` | | `LLM_MAX_ATTEMPTS` | `3` | Total attempts, including the first request | | `LLM_TIMEOUT_SECONDS` | provider default | Read timeout for one model call (Anthropic: the SDK's overall timeout) | | `ANTHROPIC_API_KEY` | unset | Anthropic credential | diff --git a/docs/api/config.md b/docs/api/config.md index 3573084..82a7c7b 100644 --- a/docs/api/config.md +++ b/docs/api/config.md @@ -12,7 +12,7 @@ A frozen Pydantic model. Unknown fields are rejected. | Field | Type | Default | |---|---|---| -| `provider` | `Provider` | `"anthropic"` | +| `provider` | `Provider` | `"openrouter"` | | `anthropic_api_key` | `SecretStr | None` | `None` | | `anthropic_model` | `str` | `"claude-opus-5"` | | `openrouter_api_key` | `SecretStr | None` | `None` | diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 68225fd..071b43e 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -20,7 +20,7 @@ finally: | Variable | Default | Meaning | |---|---|---| -| `LLM_PROVIDER` | `anthropic` | `anthropic`, `openai`, `openrouter`, or `modal` | +| `LLM_PROVIDER` | `openrouter` | `anthropic`, `openai`, `openrouter`, or `modal` | | `LLM_MAX_ATTEMPTS` | `3` | Total attempts, including the first request | | `LLM_TIMEOUT_SECONDS` | provider default | Read timeout for one model call (Anthropic: the SDK's overall timeout) | | `ANTHROPIC_API_KEY` | unset | Anthropic credential | diff --git a/src/agentloop/config.py b/src/agentloop/config.py index 9e77904..44564bb 100644 --- a/src/agentloop/config.py +++ b/src/agentloop/config.py @@ -40,7 +40,7 @@ class Settings(BaseModel): protected_namespaces=(), # several fields are legitimately named *_model ) - provider: Provider = "anthropic" + provider: Provider = "openrouter" anthropic_api_key: SecretStr | None = None anthropic_model: str = "claude-opus-5" @@ -109,7 +109,7 @@ def from_env(cls, env: Mapping[str, str]) -> Settings: program understands, in one greppable place. """ return cls( - provider=env.get("LLM_PROVIDER", "anthropic"), # type: ignore[arg-type] + provider=env.get("LLM_PROVIDER", "openrouter"), # type: ignore[arg-type] anthropic_api_key=env.get("ANTHROPIC_API_KEY"), # type: ignore[arg-type] anthropic_model=env.get("ANTHROPIC_MODEL", "claude-opus-5"), openrouter_api_key=env.get("OPENROUTER_API_KEY"), # type: ignore[arg-type] diff --git a/tests/test_config.py b/tests/test_config.py index be8631e..b1d2e6a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -81,7 +81,7 @@ def test_from_env_ignores_the_real_environment() -> None: os.environ["LLM_PROVIDER"] = "openrouter" try: - assert Settings.from_env({}).provider == "anthropic" # the default + assert Settings.from_env({}).provider == "openrouter" # the default finally: del os.environ["LLM_PROVIDER"] From efa30bbde66c099740495a0edf8d48caaa7bc4e5 Mon Sep 17 00:00:00 2001 From: aaron Date: Tue, 20 Oct 2026 19:11:15 -0700 Subject: [PATCH 2/2] Refresh agentloop documentation and examples --- BACKLOG.md | 6 + CLAUDE.md | 2 +- DOCUMENTATION.md | 425 ++++++++++++++++++++++ README.md | 156 +++++--- RELEASING.md | 77 ++++ docs/api/agent.md | 139 ++++--- docs/api/agentloop.md | 22 +- docs/api/config.md | 49 ++- docs/api/errors.md | 36 +- docs/api/events.md | 62 +++- docs/api/index.md | 11 +- docs/api/loop.md | 92 +++-- docs/api/metrics.md | 103 +++++- docs/api/observer.md | 107 ++++-- docs/api/provider.md | 85 ++++- docs/api/providers/anthropic.md | 58 ++- docs/api/providers/index.md | 26 +- docs/api/providers/modal.md | 63 +++- docs/api/providers/openai.md | 69 +++- docs/api/providers/openrouter.md | 57 ++- docs/api/retry.md | 74 +++- docs/api/summary.md | 108 ++++-- docs/api/tooling.md | 77 +++- docs/api/tracing.md | 108 +++++- docs/api/transcript.md | 99 +++-- docs/examples/custom-observer.md | 84 +++-- docs/examples/custom-provider.md | 22 +- docs/examples/index.md | 18 +- docs/examples/minimal-agent.md | 33 +- docs/examples/tool-enabled-agent.md | 55 ++- docs/examples/traced-agent.md | 59 ++- docs/getting-started/configuration.md | 46 ++- docs/getting-started/hello-world.md | 70 +++- docs/getting-started/installation.md | 56 ++- docs/guides/agents.md | 102 ++++-- docs/guides/conversations.md | 137 +++---- docs/guides/custom-providers.md | 43 ++- docs/guides/event-trace.md | 86 ++++- docs/guides/hosted-tools.md | 146 ++++---- docs/guides/installing.md | 53 ++- docs/guides/observers.md | 80 +++- docs/guides/providers.md | 127 ++++--- docs/guides/releasing.md | 80 ---- docs/guides/retries-and-errors.md | 145 +++++--- docs/guides/summaries.md | 116 ++++-- docs/guides/tools.md | 106 ++++-- docs/guides/tracing.md | 91 +++-- docs/guides/transcript-storage.md | 53 ++- docs/index.md | 19 +- src/agentloop/providers/_openai_compat.py | 2 +- tests/test_openai_compat.py | 8 + 51 files changed, 2920 insertions(+), 1028 deletions(-) create mode 100644 DOCUMENTATION.md create mode 100644 RELEASING.md delete mode 100644 docs/guides/releasing.md diff --git a/BACKLOG.md b/BACKLOG.md index 904f16b..ba64aa9 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -43,6 +43,12 @@ Left over: - **Hosted-tools guide examples (item 12).** The `agentloop-tools` parts of `docs/guides/hosted-tools.md` (`native_web_search`, `DomainPolicy`) were written from this backlog and not run here. +- **Companion package version compatibility.** `based-models-agentloop-tools` 0.1.0 pins + `based-models-agentloop>=0.4,<0.5`, while this checkout is 0.5.0. Update the companion + package's dependency range after verifying it against 0.5.0 so both can be installed + together by pip. +- **Public documentation hosting.** Add the `docs/` content to the website, then replace + the PyPI README's documentation placeholder with a link to the published pages. --- diff --git a/CLAUDE.md b/CLAUDE.md index b3559c7..d119c0b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,5 +79,5 @@ the relevant `docs/api/*` and `docs/guides/*` pages, `docs/examples/*`, and `__a ## Releasing Use the `/release` skill (`.claude/skills/release/SKILL.md`; background in -`docs/guides/releasing.md`). Pushing a `vX.Y.Z` tag publishes to PyPI immediately and a version +`RELEASING.md`). Pushing a `vX.Y.Z` tag publishes to PyPI immediately and a version can never be reused. Never `git push --tags`, force-push a tag or `twine upload`. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md new file mode 100644 index 0000000..659af54 --- /dev/null +++ b/DOCUMENTATION.md @@ -0,0 +1,425 @@ +# Documentation review and update plan + +This checklist tracks a page-by-page review against `src/agentloop`, the provider adapters, +the public exports in `src/agentloop/__init__.py`, and the tests. Update **one Markdown page +at a time** and pause for review before starting the next page. Check a page off only after +its claims have been compared with code and every executable example on that page has been +run. Use OpenAI for agent-running examples. When credentials are unavailable, exercise the +OpenAI adapter with a local HTTP transport and record that limitation here. Use the simple +caption `Output:` for documented results. Provider setup commands can be checked locally +without making a paid request. + +## Review rules + +- [x] Inventory the documentation, package surface, provider implementations, and test suite. +- [x] Identify the available Python 3.12+ environment (`conda` environment `agentloop`). +- [ ] Use small, generic, self-contained OpenAI examples for agent runs. Reserve `FakeClient` + for pages that specifically explain offline testing. +- [ ] Replace placeholder output, invented JSON, and comments like “For example” with + verified output where stable. Do not imply that local HTTP responses are live model output. + Give each example the imports, objects, and setup it needs. +- [ ] Check package-level imports against `__all__`; use submodule imports for other names. +- [ ] After each page, run its Python examples, check links and commands as applicable, and + record the verification before pausing for review. +- [ ] After all pages, run the repository's documentation-relevant checks and a final README + review of the rendered PyPI description. + +## Getting started and navigation + +1. [ ] `docs/getting-started/hello-world.md`: revise the completed offline-first page to + lead with OpenAI and remove the `FakeClient` example; keep provider recipes accurate. +2. [x] `docs/getting-started/installation.md`: check Python version, extras, checkout install, + and lazy-import claims against `pyproject.toml` and package imports; verify commands. +3. [x] `docs/getting-started/configuration.md`: check all defaults and accepted environment + variables against `Settings.from_env`, `build_client`, and provider builders; run both + configuration examples without network calls. +4. [ ] `docs/index.md`: revise its offline-first navigation after the Hello World page is + changed to OpenAI; links were already verified. + +## Guides + +5. [x] `docs/guides/agents.md`: use OpenAI for one-shot, multi-turn, and borrowed-client + examples; check client ownership and traced-agent limits. +6. [x] `docs/guides/tools.md`: correct the `ToolCallPart` import (it is in + `agentloop.transcript`, not the package root); run examples of tool execution, + dependencies, and approval; verify result order in the tooling tests. +7. [x] `docs/guides/conversations.md`: supply the missing `agent` setup; run conversation, + manual tool-result ordering, invariant, and failure examples. +8. [x] `docs/guides/providers.md`: use OpenAI for complete examples; verify provider + constructors, transport ownership, API selection, redirects, and connection-error + descriptions against implementation and tests. +9. [x] `docs/guides/hosted-tools.md`: separate this package's `HostedTool` behavior from + optional `agentloop-tools` helpers; check options and provider support against adapters; + remove stale or unverifiable experiment claims and make the example complete. +10. [x] `docs/guides/custom-providers.md`: run the `LLMClient` example and show actual output; + verify pause, hosted-tool capability, replay, and empty-turn rules. +11. [x] `docs/guides/summaries.md`: replace the fabricated summary JSON with output from an + OpenAI adapter run; supply missing imports and objects; check status, usage, timing, + warnings, timeline, and failure semantics against summary/collector/loop code. +12. [x] `docs/guides/event-trace.md`: replace undefined `tools`/`client` with a runnable + client; capture real JSONL output; check redaction and event fields. +13. [x] `docs/guides/observers.md`: run the observer examples with explicit client and usage; + verify callback signatures, guarding, and console output. +14. [x] `docs/guides/tracing.md`: verify setup and lifecycle against `tracing.py`/`metrics.py`; + run examples with local in-memory exporters where possible, without requiring OTLP. +15. [x] `docs/guides/retries-and-errors.md`: provide a scripted failure/retry example and + real output; check error taxonomy, `request_sent`, and `RetryPolicy` fields. +16. [x] `docs/guides/transcript-storage.md`: create the missing `transcript` locally, perform + a JSONL round trip in a temporary path, and show actual output. +17. [x] `docs/guides/installing.md`: reconcile with the getting-started install page and + release metadata; check version pinning guidance and verification commands. + +## Maintainer documentation + +18. [x] `RELEASING.md`: move release instructions out of user documentation; compare every + documented command and release claim with `scripts/`, package metadata, and CI + configuration; avoid running publish steps. + +## Complete examples + +19. [x] `docs/examples/minimal-agent.md`: show a direct OpenAI agent run and verify its code + through the provider adapter. +20. [x] `docs/examples/tool-enabled-agent.md`: replace the simulated weather forecast with + a generic local tool; use OpenAI and verify the tool-call path. +21. [x] `docs/examples/custom-observer.md`: fix undefined `tools`/`client`, account for + optional token values correctly, and capture actual observer/JSONL output. +22. [x] `docs/examples/traced-agent.md`: remove dependence on an external collector for the + runnable example and verify produced spans locally. +23. [x] `docs/examples/custom-provider.md`: run the complete fixed client and show its output; + verify ownership guidance. +24. [x] `docs/examples/index.md`: check descriptions and links after examples are updated. + +## API reference + +25. [x] `docs/api/agentloop.md`: reconcile the export list with `__all__`, including + `AttemptRecord` and `FailureCount`; run the import example. +26. [x] `docs/api/agent.md`: replace the `yfinance` example with a generic OpenAI tool + example; update the documented `tool()` signature to include `timeout_seconds` and + `parallel_safe`; verify agent methods, client ownership, and output. +27. [x] `docs/api/config.md`: verify fields, defaults, constructors, optional transport, and + executable settings example. +28. [x] `docs/api/errors.md`: verify the hierarchy and structured error fields; make the + catch example self-contained and run it. +29. [x] `docs/api/loop.md`: verify defaults, status and failure behavior, and observer example; + run the low-level example. +30. [x] `docs/api/observer.md`: verify all optional hooks and callback signatures; run the + subclass example. +31. [x] `docs/api/summary.md`: verify every field and status against models; run any example. +32. [x] `docs/api/events.md`: verify constructor, event names, data policy, and sample output. +33. [x] `docs/api/metrics.md`: verify instruments and attribute names against implementation. +34. [x] `docs/api/provider.md`: verify request/response types, usage and hosted-tool semantics; + run the model example. +35. [x] `docs/api/retry.md`: verify policy defaults/fields and retry behavior; run example. +36. [x] `docs/api/tooling.md`: verify schemas, registration, execution, timeouts, and concurrency; + run the `ToolSpec` example. +37. [x] `docs/api/tracing.md`: verify OTel setup, parameters and content capture; run example. +38. [x] `docs/api/transcript.md`: add missing imports to snippets; check current part fields, + invariant rules, JSONL behavior, and actual output against `transcript.py`. +39. [x] `docs/api/providers/anthropic.md`: check constructor, options, wire translation, + hosted search, and credentials against adapter code; run translation examples offline. +40. [x] `docs/api/providers/openai.md`: check Responses/Completions behavior and constructor; + run offline translation examples. +41. [x] `docs/api/providers/openrouter.md`: check defaults, request translation and constructor; + run the provider example and an offline translation check. +42. [x] `docs/api/providers/modal.md`: check endpoint, authentication and constructor; + run offline examples. +43. [x] `docs/api/providers/index.md`: check provider table and links. +44. [x] `docs/api/index.md`: check module coverage and navigation once references are current. + +## PyPI README + +45. [x] `README.md`: check every claim and configuration default against the finished pages + and implementation; use OpenAI for agent-running examples and validate them through the + adapter; include verified output where stable. Keep the documentation placeholder until + the website includes these docs. + +## Verified pages + +Record each completed page here with the exact command or method used and the observed +output. Do not mark a page complete based on syntax inspection alone. + +- `docs/getting-started/hello-world.md`: ran the exact offline Python block with + `conda run -n agentloop python /tmp/verify_agentloop_hello.py`; output was + `Hello, world!` and `model requests: 1`. Compiled the live Python block and + constructed/closed agents with `Agent.from_env` for Anthropic, OpenAI, OpenRouter, + and Modal using local dummy settings, without making provider requests. All four + constructors succeeded with the documented default models. A live model reply + requires a real credential or deployed endpoint and is intentionally not shown + as fixed output. +- `docs/getting-started/installation.md`: ran the documented `python -c` import check + with `conda run -n agentloop`; output was `agentloop ready`. Checked Python 3.12.14, + package version 0.5.0, `requires-python`, and all extras against `pyproject.toml`. + A fresh import confirmed `agentloop` does not load `anthropic`, `httpx`, or + `opentelemetry.sdk`. A local editable-install dry run could not complete because + this existing conda environment lacks the `hatchling` build backend and network + installation is unavailable in the sandbox; installation commands are therefore + checked against package metadata, not claimed as executed. +- `docs/getting-started/configuration.md`: extracted and ran both Python blocks with + `conda run -n agentloop python /tmp/verify_agentloop_configuration.py`. They printed + `openai example-model`, `max attempts: 2`, and `openai example-model 2`, matching + the page. Compared the environment table with every key read by `Settings.from_env`; + all 14 variables are listed. Client construction and close made no provider request. +- `docs/index.md`: checked the provider list against `Settings` and the offline example + against the shipped `FakeClient`; resolved all 19 relative Markdown links with a local + link check. This navigation page contains no executable example. +- `docs/guides/agents.md`: after the OpenAI revision, extracted and ran all three Python + blocks with `conda run -n agentloop python /tmp/verify_agentloop_agents_openai.py`. + A local `httpx.MockTransport` supplied four Responses API replies; the OpenAI adapter + sent four `/v1/responses` requests, and the second conversation request contained the + earlier turn. The project `.env` contains an OpenAI key, though it is not exported to the + process by default. Loaded it without printing its value and ran the same three blocks + against live OpenAI with `conda run -n agentloop python /tmp/verify_agentloop_agents_live.py`; + all four requests succeeded. The page now shows the actual live output with the `Output:` + caption. The prior `tests/test_agent.py` run passed all 29 tests. +- `docs/guides/tools.md`: extracted and ran both complete Python blocks against live + OpenAI with `conda run -n agentloop python /tmp/verify_agentloop_tools_live.py`, loading + the project `.env` key without printing it. The arithmetic tool ran once and returned + `27 plus 15 is 42.` The public note was read from application deps; the private call + was denied before its handler ran and was counted in both `tool_calls` and + `tool_errors`. The page contains the actual output. Ran + `conda run -n agentloop python -m pytest -q tests/test_tooling.py` (25 passed), + including the executor's ordering and failure behavior. After simplifying `add`, ran + just that Python block again with + `conda run -n agentloop python /tmp/verify_agentloop_tools_add_live.py`; it returned + the same documented output. +- `docs/guides/conversations.md`: ran its OpenAI conversation block with + `conda run -n agentloop python /tmp/verify_agentloop_conversations_live.py` using the + project `.env` key without printing it; the reply and `messages: 2` output are on the + page. Ran the manual result-order and invariant blocks with + `conda run -n agentloop python /tmp/verify_agentloop_conversations_local.py`; their + output matches the page. Ran `conda run -n agentloop python -m pytest -q + tests/test_transcript.py` (37 passed). +- `docs/guides/providers.md`: extracted and ran all three Python blocks with + `conda run -n agentloop python /tmp/verify_agentloop_providers_live.py`, using the + project `.env` OpenAI key without printing it. They produced `Hello`, + `openai gpt-5.1 responses`, and `ready` with `HTTP client closed by agent: False`, + matching the page. Removed the unconfigured `truststore`/proxy snippet and checked + the transport and Responses claims against the adapters. Ran the transport, OpenAI + Responses, and OpenAI compatibility test files together (96 passed). +- `docs/guides/hosted-tools.md`: removed package-external policy details and stale + experiment claims, then ran its OpenAI Responses web-search block with + `conda run -n agentloop python /tmp/verify_agentloop_hosted_live.py` using the project + `.env` key without printing it. The provider returned one hosted search call, 12 + sources, and no citations for this answer; the output is on the page. Compared the + option table with both adapters and ran the OpenAI Responses and Anthropic test files + (83 passed). Clarified the contract with the separate `agentloop-tools` package: + `native_web_search()` returns this package's `HostedTool`. Importing its factory from + the sibling source confirmed the type and options. Moved the companion package's + version compatibility issue to `BACKLOG.md`. Corrected the execution attribution, documented + mutually exclusive domain filters and hosted/local tool name collisions, and clarified + Anthropic model support and the Chat Completions error paths. Checked those claims against + source and ran the agent, OpenAI Responses, and Anthropic test files (112 passed). +- `docs/guides/custom-providers.md`: kept the deliberately minimal custom `EchoClient` to + demonstrate the `LLMClient` protocol, combined its two code blocks into one complete + example, and ran the block in the `agentloop` conda environment. It printed `Echo: hello`, + as documented. Checked pause, hosted capability, replay, stop reasons, empty turns, and + client ownership against the provider, loop, transcript, and agent code. Ran the agent, + loop, and provider tests (69 passed). +- `docs/guides/summaries.md`: replaced fabricated JSON and incomplete snippets with live + OpenAI examples showing a selected summary, an observer callback, and an optional timeline. + Ran its Python blocks with `conda run -n agentloop python + /tmp/verify_agentloop_summaries_live.py` using the project `.env` key without printing it; + the actual output is on the page. Checked fields and failure behavior against the summary, + collector, loop, and observer code; ran summary and loop tests (94 passed). Rebuilt the page + as three independent OpenAI examples after feedback: first the result summary, then an + observer, then `SummaryOptions`. Ran all three code blocks and updated their output. +- `docs/guides/event-trace.md`: replaced undefined `tools` and `client` with an OpenAI run + that writes JSONL to a temporary file and prints selected fields from its four records. + Added a separate content-capture example with an application redactor. Ran both Python + blocks with `conda run -n agentloop python /tmp/verify_agentloop_event_trace_live.py` + using the project `.env` key without printing it; their output is on the page. Checked + event fields and redaction behavior against `events.py`; ran event trace tests (20 passed). +- `docs/guides/observers.md`: made the custom usage observer and combined built-in observer + examples complete OpenAI runs. Ran both Python blocks with `conda run -n agentloop python + /tmp/verify_agentloop_observers_live.py` using the project `.env` key without printing it; + the response count, token usage, console text, and recorded events match the page. Checked + callback signatures, optional hooks, and guarding against `observer.py`; ran agent, loop, + and summary tests (123 passed). After feedback, kept the usage observer simple and printed + the input prompt and returned answer beside the provider's input/output token counts; + reran both blocks and confirmed the displayed values. Then changed the first prompt to + the one-digit week question, printed `response.usage`, and confirmed its 19 input and 11 + output tokens against the raw OpenAI response body from the same request. Reran the updated + guide blocks; the documented output matches. +- `docs/guides/tracing.md`: replaced collector-dependent examples with complete OpenAI runs + using in-memory span and metric exporters. Ran both Python blocks with `conda run -n + agentloop python /tmp/verify_agentloop_tracing_live.py` using the project `.env` key without + printing it; the recorded span operations and metric names match the page. Checked provider + setup, content capture, lifecycle, instruments, and metric attributes against source; ran + tracing and metrics tests (41 passed). +- `docs/guides/retries-and-errors.md`: replaced incomplete snippets with one runnable example + that injects a retryable connection failure, then lets the OpenAI client answer. Ran its + Python block with `conda run -n agentloop python /tmp/verify_agentloop_retries_live.py` + using the project `.env` key without printing it; the retry log, answer, attempt count, + summary retry count, and send state match the page. Checked error types, send-state phases, + and retry policy against source; ran retry, transport error, and summary tests (115 passed). + Reworked the guide after feedback to introduce retryable errors and policy before the + example, then explain the recorded attempts and three send states. Reran the shortened + example; its output still matches. +- `docs/guides/transcript-storage.md`: replaced the undefined transcript with a complete + two-message JSONL round trip in a temporary directory. Ran the example in the `agentloop` + conda environment; its serialized lines and equality result match the page. Checked + serialization, parsing, invariant checks, and origin replay against `transcript.py`. +- `docs/guides/installing.md`: updated the version range to `0.5`, aligned extras and + Python requirements with `pyproject.toml` and the getting-started guide, and added a + runnable import check. Ran the import command outside the source tree in the `agentloop` + conda environment; its output matches the page. Parsed the example requirement to verify + its syntax. Checked the 0.5.0 release files and provenance on PyPI and the repository's + release workflow. A live GitHub release query was unavailable from this environment. +- `RELEASING.md`: moved the maintainer release procedure out of the user documentation and + removed its navigation link. Matched the tag trigger, job sequence, publisher environment + names, version checks, and release artifacts to `.github/workflows/release.yml`. Replaced + stale pending-publisher and GitHub plan claims, aligned the local checks with + `scripts/check.sh`, and corrected the `test-wheel.sh` description and yanking guidance. + Ran `conda run -n agentloop bash scripts/check.sh`: ruff and mypy passed, 586 tests passed + with 94.81% coverage, and the import check passed. Shell syntax checks passed. Attempted + `scripts/test-wheel.sh` in a fresh temporary clone; build isolation could not install + `hatchling` because PyPI network access was unavailable. No tag or publish step was run. +- `docs/examples/minimal-agent.md`: replaced the incomplete provider setup and larger + one-off script with a short OpenAI agent example. Ran the exact Python block through the + OpenAI provider in the `agentloop` conda environment; the output was `4`, as shown on the + page. Checked `Agent.from_env()`, `run()`, and context-manager ownership against `agent.py`. +- `docs/examples/tool-enabled-agent.md`: replaced the fictional weather forecast with a + small `count_letters` tool and removed the observer from the example. Ran the OpenAI agent + in the `agentloop` conda environment; it returned `5` and recorded one `count_letters` + call, matching the page. Checked decorator schema requirements, tool dispatch, and + `run_result()` summary fields against `agent.py`, `tooling.py`, and the loop. +- `docs/examples/minimal-agent.md`, `docs/examples/tool-enabled-agent.md`, and + `docs/guides/tracing.md`: added a provider-support sentence before their OpenAI install + instructions. The examples and their recorded output were unchanged. +- `docs/examples/custom-observer.md`: replaced the undefined `tools` and `client` with + complete OpenAI examples for a custom token counter and an in-memory JSONL trace. Ran + both blocks in the `agentloop` conda environment; the 19 input tokens, 11 output tokens, + and four trace events match the page. Checked callback signatures and usage field types + against `observer.py` and `provider.py`. +- `docs/examples/traced-agent.md`: replaced the external OTLP collector with an in-memory + tracer provider. Ran the OpenAI example in the `agentloop` conda environment and verified + that the `chat` span is a child of the `invoke_agent` span; the output matches the page. + Checked `trace=True` setup and provider ownership against `agent.py` and `tracing.py`. +- `docs/examples/custom-provider.md`: ran the exact `FixedClient` Python block in the + `agentloop` conda environment; it printed the documented response. Checked the + `LLMClient` protocol and borrowed-client ownership against `provider.py` and `agent.py`, + and confirmed that closing a directly constructed agent leaves a closable client open. +- `docs/examples/index.md`: updated the summaries of all five example pages and clarified + which examples call OpenAI. Checked that each relative Markdown link resolves. +- `docs/api/agentloop.md`: added the two missing root exports, `AttemptRecord` and + `FailureCount`, and checked the list against all 37 names in `__all__`. Ran the import + example in the `agentloop` conda environment and verified that a bare import leaves + `anthropic`, `httpx`, and `opentelemetry.sdk` unloaded. +- `docs/api/agent.md`: replaced the stock-price example with a small `read_note` tool that + shows model-supplied arguments and application-supplied `Deps`. Ran the exact Python block + with OpenAI in the `agentloop` conda environment; it printed `Hello from the agent loop.` + and `{'read_note': 1}`, matching the page. Updated the `tool()` signature and checked + defaults, tool validation, client ownership, and agent methods against the source. +- `docs/api/config.md`: checked every setting and default against `config.py`, made the + settings example self-contained, and showed client construction and cleanup. Ran the + exact Python block in the `agentloop` conda environment; its output matches the page. + Clarified attempt and timeout behavior, credential errors, Pydantic validation, and + ownership of an optional HTTP transport against the provider implementations. +- `docs/api/errors.md`: verified the exception hierarchy, retryable classes, budget kinds, + and provider error fields against `errors.py`, `retry.py`, and the loop. Replaced the + undefined-agent example with a real missing-credential failure from `build_client()`. + Ran the exact Python block in the `agentloop` conda environment; it raised `AuthError` + and printed the metadata shown on the page. +- `docs/api/loop.md`: checked constants, signatures, transcript updates, truncation, + deadlines, summaries, and `ListObserver` against `loop.py`. Replaced undefined-client + and empty-observer snippets with one complete low-level OpenAI example. Ran the exact + Python block in the `agentloop` conda environment; it printed `7` and the request and + response event names shown on the page. +- `docs/api/observer.md`: checked callback signatures, optional hooks, guarded callback + behavior, and tool event timing against `observer.py` and `tooling.py`. Replaced the + unconnected `RetryCounter` snippet with a complete OpenAI `ResponseCounter` example. + Ran its exact Python block in the `agentloop` conda environment; it printed `7` and one + response, matching the page. +- `docs/api/summary.md`: expanded the status table, distinguished statuses from the + `succeeded` and `budget_exhausted` properties, and documented every public field of + `ConversationSummary`, `TurnRecord`, and `ConversationContext`. Checked field and status + coverage against the live models. Ran the exact OpenAI example in the `agentloop` conda + environment; it printed `7`, `completed`, `True`, and `1`, matching the page. +- `docs/api/events.md`: verified the constructor, record keys, event names, content policy, + redaction order, and concurrency rule against `events.py`. Added a complete metadata-only + OpenAI trace example and ran its exact Python block in the `agentloop` conda environment; + the event list and `1 False` request check match the page. +- `docs/api/metrics.md`: documented all 12 instruments with their exact attribute sets and + corrected when stall and budget-warning counters record. Verified each instrument and + attribute set with a `MetricsObserver`, a complete sample `ConversationSummary`, and an + in-memory OpenTelemetry meter in the `agentloop` conda environment. Confirmed the `otel` + extra. Added a complete OpenAI example on the API page that passes a `Meter` from + `metrics.get_meter("agentloop")` to `MetricsObserver`. Ran its exact Python block; it + printed `7` and `completed 1`, matching the page. +- `docs/api/provider.md`: checked all fields of `ToolSchema`, `ModelRequest`, `Usage`, + `ModelResponse`, `AttemptRecord`, and `HostedTool` against the live Pydantic models. + Clarified empty truncated responses, retry metadata, and hosted-tool support. Replaced + the placeholder response with a complete direct OpenAI `ModelRequest` example and ran + its exact Python block in the `agentloop` conda environment; it printed `7` and + `end_turn`, matching the page. +- `docs/api/retry.md`: checked policy fields and defaults, delay calculation, observer + callback timing, delegated client properties, and attempt logging against `retry.py`. + Replaced the undefined-client snippet with an OpenAI client wrapped by a one-time + retryable failure. Ran the exact Python block in the `agentloop` conda environment; it + printed the documented retry warning, answer, attempt count, and retry count. +- `docs/api/tooling.md`: checked `ToolSpec` fields, schema use, registry behavior, + sequential and parallel execution, timeouts, and error results against `tooling.py`. + Replaced the incomplete `ping` snippet with a self-contained `ToolSpec`, `Registry`, + and `run_tools` example. Ran the exact Python block in the `agentloop` conda + environment; it printed `hello` and `False`, matching the page. +- `docs/api/tracing.md`: checked span creation, observer callbacks, content capture, + redaction and error-detail clipping, concurrency, and `configure_tracing()` against + `tracing.py`. Replaced the constructor-only snippet with an in-memory OpenTelemetry + example using a direct `OTelObserver` and OpenAI. Ran its exact Python block in the + `agentloop` conda environment; it printed `ready` and the `chat` and `invoke_agent` + span operations shown on the page. Added a separate `configure_tracing()` collector + example, ran its exact Python block with OpenAI and a temporary local OTLP receiver, + and verified that it printed `ready` and exported spans to `/v1/traces`. +- `docs/api/transcript.md`: checked content-part fields, `Origin`, message and transcript + behavior, invariants, JSONL serialization, and text extraction against `transcript.py`. + Replaced incomplete snippets with one self-contained round-trip example. Ran its exact + Python block in the `agentloop` conda environment; it printed `Hi!`, two messages, and + two JSONL lines, matching the page. +- `docs/api/providers/anthropic.md`: checked the constructor, credential handling, HTTP + client ownership, hosted web search options, and translation helpers against the + Anthropic adapter. Replaced the undefined-request snippet with a self-contained + offline `ModelRequest` example. Ran its exact Python block in the `agentloop` conda + environment; the model, prompt text, and hosted search tool type match the page. +- `docs/api/providers/openai.md`: checked the constructor, defaults, API paths, hosted + tool support, and timeout behavior against the OpenAI and shared adapters. Replaced + the undefined-request snippet with a complete example that calls both OpenAI APIs. + Ran its exact Python block in the `agentloop` conda environment; both returned + `ready`, matching the page. Offline translation checks confirmed `store: false`, + token-limit fields, and omission of `parallel_tool_calls` without tools. The live + check revealed that Chat Completions rejected that parameter without tools, so the + adapter now omits it in that case. The targeted adapter suite passed (43 tests). +- `docs/api/providers/openrouter.md`: checked its constructor, headers, timeouts, Chat + Completions format, hosted-tool support, and reasoning replay against the adapter. + Replaced the placeholder construction snippet with a complete OpenRouter request and + output. Ran its exact Python block in the `agentloop` conda environment; it printed + `openrouter completions` and `ready`, matching the page. An offline translation check + confirmed the default model, `max_tokens` field, and omission of `parallel_tool_calls` + when no tools are present. +- `docs/api/providers/modal.md`: checked the constructor, endpoint path, proxy headers, + timeout defaults, and vLLM quirks against the adapter. Replaced placeholder credentials + with a runnable client configuration example. Ran its exact Python block in the + `agentloop` conda environment; it printed the documented client API, default model, + and empty hosted-tool set. Offline checks confirmed the request path, headers, + `max_tokens` field, and absence of `parallel_tool_calls` without tools. A live request + reached the configured Modal endpoint, but it returned `no upstreams available`, so + no model output was available to document. +- `docs/api/providers/index.md`: added a provider comparison table and links to client + selection, extension, and API details. Checked all seven relative links on disk. + Constructed direct OpenAI, OpenRouter, and Modal clients offline in the `agentloop` + conda environment, and checked Anthropic's class capability; API mode and hosted + search support matched every table row. +- `docs/api/index.md`: checked the table against every top-level module in + `src/agentloop` and added `collector` and `stall` navigation. Verified all 20 relative + links on disk and confirmed that all 17 top-level modules, including the package root + and providers package, are represented. This index has no executable example. +- `README.md`: reconciled provider support, configuration defaults, tool behavior, hosted + search, summaries, tracing, and event capture with the implementation and completed + documentation pages. Replaced incomplete examples with six self-contained Python blocks + and captured their actual output in the `agentloop` conda environment. Five used live + OpenAI calls; the missing-credential example ran offline. The tool run recorded one + `add` call, and the hosted search run recorded a `HostedToolPart`. Kept the + documentation placeholder at the user's request until the website includes these docs. + Revised the reasons list to cover local tool approval, hosted search domain restrictions, + and content redaction, with each bullet at two lines or fewer. + Rendered the Markdown with `markdown_it` (four tables and 15 code blocks) and confirmed + no em dashes remain. diff --git a/README.md b/README.md index 2e9e303..e2dae8f 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,14 @@ The package is installed as `based-models-agentloop` and imported as `agentloop` ## Why use agentloop? - Use the same agent code with Anthropic, OpenAI, OpenRouter, or Modal/vLLM. -- Run local Python tools or provider-hosted web search. -- Keep complete, typed, serializable transcripts—including tool activity, sources, and - citations. -- Execute independent tool calls concurrently while preserving their original result order. -- Get consistent retries, errors, usage data, time limits, and iteration limits across - providers. -- Record outcomes, event logs, distributed traces, and metrics without changing agent - behavior. +- Run local Python tools with approval hooks and trusted dependencies. +- Constrain provider-hosted web search with domain allowlists or blocklists. +- Keep typed, serializable transcripts with tool activity, sources, and citations. +- Run independent tool calls concurrently while preserving result order. +- Get consistent retries, error types, usage data, and run limits across providers. +- Record summaries, event logs, traces, and metrics with metadata-only defaults; redact + captured content before logging or export. - Test agent behavior offline with the included scripted `FakeClient`. -- Install only the provider dependencies your application needs. Requires Python 3.12 or newer. @@ -41,15 +39,17 @@ import os from agentloop import Agent -with Agent.from_env( - os.environ, - tools=[], - system="Answer clearly and concisely.", -) as agent: - print(agent.run("Why is the sky blue?")) +with Agent.from_env(os.environ, tools=[]) as agent: + print(agent.run("Reply with exactly: ready")) +``` + +Output: + +```text +ready ``` -`Agent.from_env()` receives an environment mapping explicitly—the library never reads the +`Agent.from_env()` receives an environment mapping explicitly. The library never reads the process environment or loads a `.env` file on its own. The context manager closes the client created by `from_env()` when the block exits. @@ -97,7 +97,7 @@ contract shown to the model and is never inferred from the function signature. ```python import os -from agentloop import Agent, ToolError, tool +from agentloop import Agent, tool ADD_SCHEMA = { "type": "object", @@ -112,16 +112,24 @@ ADD_SCHEMA = { @tool(parameters=ADD_SCHEMA) def add(left: int, right: int) -> str: """Add two integers.""" - if abs(left) > 1_000_000 or abs(right) > 1_000_000: - raise ToolError("numbers must be at most one million") return str(left + right) with Agent.from_env( os.environ, tools=[add], - system="Use the add tool for arithmetic.", + system="Use the add tool for arithmetic. Reply with only its numeric result.", ) as agent: - print(agent.run("What is 27 plus 15?")) + result = agent.run_result("What is 27 plus 15?") + +print(result.text) +print(result.summary.tool_calls) +``` + +Output: + +```text +42 +{'add': 1} ``` `ToolError` returns safe feedback to the model. Unexpected exceptions are logged and @@ -140,21 +148,33 @@ search is supported by Anthropic and by OpenAI's Responses API: ```python import os -from agentloop import Agent, HostedTool +from agentloop import Agent, HostedTool, HostedToolPart -search = HostedTool( - kind="web_search", - options={ - "allowed_domains": ["europa.eu"], - "search_context_size": "high", - }, -) +search = HostedTool(kind="web_search", options={"allowed_domains": ["python.org"]}) with Agent.from_env(os.environ, tools=[search]) as agent: - print(agent.run("What changed in EU AI Act guidance this month?")) + result = agent.run_result( + "Use web search to find the title of the Python home page. " + "Reply with only the title, without explanation or citation." + ) + +print(result.text) +print(any( + isinstance(part, HostedToolPart) + for message in result.transcript.messages + for part in message.content +)) ``` -Search activity, consulted sources, and citations are preserved in the transcript. Passing a +Output: + +```text +Welcome to Python.org +True +``` + +The `True` means the transcript recorded a hosted search call. Search activity, consulted +sources, and citations are preserved in the transcript. Passing a hosted tool to an unsupported provider or API fails when the agent is constructed instead of silently dropping the capability. @@ -170,7 +190,11 @@ from agentloop import Agent, Transcript, final_text transcript = Transcript() -with Agent.from_env(os.environ, tools=[]) as agent: +with Agent.from_env( + os.environ, + tools=[], + system="When asked for the user's name, reply with only the name.", +) as agent: transcript.add_user_message("My name is Ada.") agent.converse(transcript) @@ -178,7 +202,14 @@ with Agent.from_env(os.environ, tools=[]) as agent: agent.converse(transcript) print(final_text(transcript)) -print(transcript.model_dump_json(indent=2)) +print(len(transcript.messages)) +``` + +Output: + +```text +Ada +4 ``` A transcript can contain text, model reasoning, local tool calls and results, hosted-tool @@ -196,21 +227,22 @@ Rate limits, provider unavailability, timeouts, and invalid responses are retrya authentication, bad-request, context-length, missing-model, and credit errors fail immediately. Server `Retry-After` values are honored within the configured delay limit. -```python -import os +A missing credential fails before a model request is sent: -from agentloop import Agent, AgentLoopError -from agentloop.errors import AuthError, RateLimited +```python +from agentloop import Agent +from agentloop.errors import AuthError try: - with Agent.from_env(os.environ, tools=[]) as agent: - print(agent.run("Hello")) -except AuthError: - print("Check provider credentials") -except RateLimited as error: - print("The provider remained rate limited", error.retry_after) -except AgentLoopError as error: - print(f"Agent failed: {type(error).__name__}: {error}") + Agent.from_env({"LLM_PROVIDER": "openai"}, tools=[]) +except AuthError as error: + print(type(error).__name__, error) +``` + +Output: + +```text +AuthError OPENAI_API_KEY is not set ``` The loop also enforces iteration and wall-clock budgets, protects against truncated tool @@ -228,21 +260,30 @@ Choose the level of detail that fits the task: | Build dashboards and alerts | OpenTelemetry metrics | | Send conversation events to an existing system | A custom observer | -Every conversation produces a `ConversationSummary` on success and failure. It records the -status, model calls, elapsed time, tool calls and errors, retries, token use, and provider- -reported cost. It contains metadata, not prompts, tool arguments, results, or exception -messages. +Every started conversation produces a `ConversationSummary` on success and failure. It +records the status, model calls, elapsed time, tool calls and errors, retries, token use, +and provider-reported cost when available. It contains metadata, not prompts, tool +arguments, results, or exception messages. ```python +import os + +from agentloop import Agent + with Agent.from_env(os.environ, tools=[]) as agent: - result = agent.run_result( - "Summarize this account's open support cases.", - correlation_id="case-review-42", - ) + result = agent.run_result("Reply with exactly: ready") print(result.text) -print(result.summary.status) -print(result.summary.usage) +print(result.summary.status.value, result.summary.iterations) +print(result.summary.usage.input_tokens, result.summary.usage.output_tokens) +``` + +Output: + +```text +ready +completed 1 +11 11 ``` Read the summary from a result, from `.summary` on an `AgentLoopError`, or in an observer's @@ -251,8 +292,9 @@ warnings as a conversation approaches its limits, and detection of repeated tool `EventTraceObserver` writes an ordered JSONL event stream to a function supplied by your application, such as a file writer, logger, or queue. Its default output contains metadata -only. Capturing prompts, responses, tool arguments, results, or error text requires an -explicit setting and a redaction function. +only. Capturing prompts, responses, tool arguments, results, or error text requires +`capture_content=True` and a redaction function, unless you explicitly choose +`allow_unredacted=True`. Install the `otel` extra to use OpenTelemetry spans and metrics: diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..e7d9900 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,77 @@ +# Releasing + +Create release tags from the intended commit on `main`. Pushing a `vX.Y.Z` tag starts the +release workflow, which builds the package and publishes it to PyPI through Trusted +Publishing. The workflow does not check which branch contains the tagged commit. Complete +the checks below before pushing the tag. + +## Checklist + +1. Move the `[Unreleased]` entries in `CHANGELOG.md` under a new `## [X.Y.Z]` heading. +2. Set the version in **both** `pyproject.toml` and `src/agentloop/__init__.py`. +3. In a fresh clone, use Python 3.12 or newer to run the checks and test the built package: + + ```bash + git clone . /tmp/agentloop-release && cd /tmp/agentloop-release + python -m pip install -e '.[all,dev]' + bash scripts/check.sh + bash scripts/test-wheel.sh + ``` + +4. Once the checks pass, create and push the tag for that commit: + + ```bash + git tag vX.Y.Z && git push origin vX.Y.Z + ``` + +The `Release` workflow then: + +1. **build**: refuses to continue unless the tag, `pyproject.toml` and `__version__` agree + and the changelog has an entry; runs ruff, mypy, pytest and `scripts/test-wheel.sh`; + builds the wheel and sdist once and uploads them as an artifact. +2. **publish-pypi**: uploads those exact files to PyPI via Trusted Publishing. PyPI records + provenance attestations for the published files. +3. **github-release**: attaches the same files plus `SHA256SUMS` to the GitHub release. + +## Publishing setup + +- The PyPI and TestPyPI projects need trusted publishers for owner `based-models`, repository + `agent-loop`, workflow `release.yml`, and their respective environments: `pypi` and + `testpypi`. +- The GitHub repository needs environments named `pypi` and `testpypi` to match those + publisher configurations. Check their settings before starting a release. + +## TestPyPI rehearsal + +In GitHub Actions, select **Release** then **Run workflow** on the version-bump branch. This +manual run builds and tests the package, then publishes to TestPyPI only. Choose a version +that has not been uploaded there before; an uploaded version cannot be reused. Check the +TestPyPI release page before running it, then install the result in a clean environment: + +```bash +python -m pip install --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple/ 'based-models-agentloop[anthropic]==X.Y.Z' +``` + +The extra index lets pip fetch dependencies that are not available on TestPyPI. Use a +version unique to TestPyPI so the command does not select the same version from PyPI. + +## What `scripts/test-wheel.sh` proves + +It builds the wheel and source distribution, then installs the wheel into clean virtual +environments. An install with only `anthropic` checks that `httpx` and the OpenTelemetry SDK +remain optional and that missing extras produce useful errors. A separate install with `all` +checks provider imports. The script also checks `py.typed`, license metadata, and package +contents. It deletes and recreates the clone's `dist/` directory. + +## Rolling back + +PyPI files are immutable and a version number cannot be reused. If a release is broken, +[yank it on PyPI](https://docs.pypi.org/project-management/yanking/), fix the issue, and +publish a new version. Installers usually skip yanked releases, but an exact `==` or `===` +pin may still select one. Do not move a published tag. + +## Versioning + +This project uses SemVer. While the version is `0.x`, a minor bump may change the public API +exposed through `agentloop.__all__`. diff --git a/docs/api/agent.md b/docs/api/agent.md index 6167497..83948ce 100644 --- a/docs/api/agent.md +++ b/docs/api/agent.md @@ -15,61 +15,58 @@ a capability that can be offered to the model. `Deps` holds runtime-only state a that the capability needs when it executes. `Agent` registers the tools and passes its deps bundle to each tool call. -The separation is important: the model sees a tool's name, description, and JSON Schema, -but it never sees `Deps`. Values chosen by the model belong in the schema; trusted values -supplied by the application belong in `Deps`. +The model sees a tool's name, description, and JSON Schema, but it never sees `Deps`. +Values chosen by the model belong in the schema; application data belongs in `Deps`. + +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses OpenAI. + +Install OpenAI support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +The model chooses a note name. The application supplies the notes through `Deps`: ```python -import datetime as dt import os -import yfinance as yf - from agentloop import Agent, Deps, tool -class MarketDeps(Deps): - use_adjusted_prices: bool = True +class NoteDeps(Deps): + notes: dict[str, str] @tool( parameters={ "type": "object", - "properties": { - "tickers": { - "type": "array", - "items": {"type": "string"}, - "minItems": 1, - }, - "date": {"type": "string", "format": "date"}, - }, - "required": ["tickers", "date"], + "properties": {"name": {"type": "string"}}, + "required": ["name"], "additionalProperties": False, } ) -def closing_prices(tickers: list[str], date: str, *, deps: MarketDeps) -> str: - """Return closing prices for one or more stock tickers on a date.""" - trading_day = dt.date.fromisoformat(date) - next_day = trading_day + dt.timedelta(days=1) - prices: dict[str, float | None] = {} - - for ticker in tickers: - history = yf.Ticker(ticker).history( - start=trading_day, - end=next_day, - auto_adjust=deps.use_adjusted_prices, - ) - prices[ticker] = None if history.empty else float(history["Close"].iloc[0]) - - return str(prices) - -deps = MarketDeps(use_adjusted_prices=True) -with Agent.from_env(os.environ, tools=[closing_prices], deps=deps) as agent: - answer = agent.run("What did AAPL and MSFT close at on 2025-01-02?") +def read_note(name: str, *, deps: NoteDeps) -> str: + """Read a note by name.""" + return deps.notes.get(name, "Note not found.") + +deps = NoteDeps(notes={"welcome": "Hello from the agent loop."}) +with Agent.from_env(os.environ, tools=[read_note], deps=deps) as agent: + result = agent.run_result( + "Read the welcome note with read_note. Reply with its text only." + ) + print(result.text) + print(result.summary.tool_calls) +``` + +Output: + +```text +Hello from the agent loop. +{'read_note': 1} ``` -Here, `tickers` and `date` are model-provided inputs. Whether to adjust prices is an -application policy, so it is injected through `deps` and omitted from the model-visible -tool schema. This example requires the separately installed `yfinance` package; it is not -an `agentloop` dependency. +`tool_calls` confirms the agent ran the tool. ## When to use `Deps` @@ -82,15 +79,23 @@ Provide `Deps` when tools need either: database handle, or request context; or - a `before_tool` hook that approves or denies tool calls before execution. -Subclass `Deps` to declare application-specific fields, then add a keyword-only `deps` -parameter only to handlers that need the bundle. Do not put ordinary model-selected tool -arguments in `Deps`; doing so hides them from the tool schema and prevents the model from -supplying them. +The `read_note` example shows the two kinds of input: + +- `name` is in the tool's JSON Schema. The model chooses it when calling `read_note`. +- `notes` is declared on `NoteDeps` and passed to the agent by the application. The model + can ask for a note by name, but cannot supply or replace the notes dictionary. -The bundle is attached to the `Agent`, not created for each call. If its fields contain -mutable or request-specific state, manage that lifetime deliberately. Also remember that -multiple tool calls may execute concurrently, so shared dependencies and approval hooks -must be safe for concurrent use. +Define a `Deps` subclass for the application values a tool needs. Add `*, deps: NoteDeps` +to a handler that reads those values, as `read_note` does. A handler that needs only its +model-chosen arguments can leave out `deps`. Keep those arguments in the JSON Schema so +the model knows what it can pass. For example, `read_note` lists `name` under `properties` +and `required`, so the model knows to supply a note name when it calls the tool. + +The agent keeps the same `deps` object for every run and every tool call. For example, two +calls to `agent.run_result(...)` would use the same `notes` dictionary. If the notes differ +by request, create an agent with the appropriate `NoteDeps` for each request. If a tool +changes shared data, account for the fact that multiple tool calls can run concurrently; +the same applies to a custom `before_tool` approval hook. ## `Deps` @@ -109,7 +114,7 @@ handlers. ## `tool` ```text -tool(parameters, *, name=None, description=None) +tool(parameters, *, name=None, description=None, timeout_seconds=None, parallel_safe=True) ``` Decorator that converts a callable into a [`ToolSpec`](tooling.md#toolspec). @@ -118,27 +123,13 @@ Decorator that converts a callable into a [`ToolSpec`](tooling.md#toolspec). - `name`: model-visible name; defaults to the function name. - `description`: defaults to the first paragraph of the docstring. If neither exists, decoration raises `ValueError`. +- `timeout_seconds`: optional positive limit for a local tool call. +- `parallel_safe`: set to `False` to run local tool calls in a turn sequentially. - A keyword-only parameter named `deps` receives the configured dependency bundle and is hidden from the model-visible handler signature and JSON Schema. The same bundle is available to every tool registered with the agent, though a handler can omit the parameter when it does not need it. -```python -from agentloop import tool - -@tool( - parameters={ - "type": "object", - "properties": {"name": {"type": "string"}}, - "required": ["name"], - "additionalProperties": False, - } -) -def greet(name: str) -> str: - """Greet a person.""" - return f"Hello, {name}!" -``` - ## `Agent` ```text @@ -164,12 +155,14 @@ kept as internal state. `client` is borrowed: an agent constructed directly never closes it. `tools` accepts local `ToolSpec` objects and provider-run `HostedTool` objects. Values produced by `@tool` can be -passed directly. Duplicate local tool names, unsupported hosted tools, and a local `ToolSpec` named like a -hosted tool kind (for example `web_search`) fail during construction; native and client-side -tools are separate, explicit choices. `deps` is the one runtime dependency bundle passed to local tool handlers, -defaulting to `Deps()`. `trace=True` combines an `OTelObserver` with the supplied observer. -`metrics=True` adds a `MetricsObserver`. `summary_options` configures the optional timeline, -repeated-call detection, and limit warnings described in +passed directly. Construction fails for duplicate local tool names, hosted tools the client +does not support, or a local tool whose name matches a hosted tool passed to the same agent +(for example, both named `web_search`). + +`deps` is the one runtime dependency bundle passed to local tool handlers, defaulting to +`Deps()`. `trace=True` combines an `OTelObserver` with the supplied observer. `metrics=True` +adds a `MetricsObserver`. `summary_options` configures the optional timeline, repeated-call +detection, and limit warnings described in [Conversation summaries](../guides/summaries.md). ### `Agent.from_env` @@ -218,13 +211,13 @@ agent.converse( ) -> Transcript ``` -Advances, mutates, and returns the supplied transcript. The optional identifiers have the +Updates and returns the supplied transcript. The optional identifiers have the same meaning as in `run()`. ### `Agent.converse_result` Accepts the same arguments as `converse()` and returns a `ConversationResult` containing the -transcript and its summary. +transcript, text, and summary. ### `Agent.close` diff --git a/docs/api/agentloop.md b/docs/api/agentloop.md index a8d8099..4e67a40 100644 --- a/docs/api/agentloop.md +++ b/docs/api/agentloop.md @@ -1,9 +1,11 @@ # `agentloop` -The package root is the recommended import surface for ordinary agents: +The package root collects common imports. `Agent` runs conversations, `tool` defines local +Python tools, `Transcript` holds messages, and `ConversationSummary` reports how a run ended. +Import these names directly from `agentloop`: ```python -from agentloop import Agent, ConsoleObserver, ToolError, tool +from agentloop import Agent, ConversationSummary, Transcript, tool ``` It re-exports: @@ -13,15 +15,17 @@ It re-exports: - Errors: `AgentLoopError`, `ToolError`. - Loop: `run_conversation`, `run_conversation_result`, `ListObserver`. - Observers: `LoopObserver`, `NullObserver`, `ConsoleObserver`, `MultiObserver`. -- Reporting: `ConversationContext`, `ConversationResult`, `ConversationStatus`, - `ConversationSummary`, `SummaryCollector`, `SummaryOptions`, `TurnRecord`, and - `EventTraceObserver`. +- Reporting: `AttemptRecord`, `ConversationContext`, `ConversationResult`, + `ConversationStatus`, `ConversationSummary`, `FailureCount`, `SummaryCollector`, + `SummaryOptions`, `TurnRecord`, and `EventTraceObserver`. - Hosted tools: `HostedTool`, `HostedToolPart`, `Source`, `Citation`. - Tools: `Registry`, `ToolDeps`, `ToolSpec`, `allow_all`. - Transcripts: `Transcript`, `Origin`, `final_text`, `join_text`. -`__version__` is the installed package version string. +`__version__` is the package's declared version string. -Provider-specific clients, the full error hierarchy, retry configuration, OpenTelemetry -setup, and metrics remain in their defining modules. Importing `agentloop` does not load -optional provider SDKs or the OpenTelemetry SDK. +This list is an import map; the [API index](index.md) links to method and field details. +Custom providers use `LLMClient`, `ModelRequest`, and `ModelResponse` from +[`agentloop.provider`](provider.md). Provider-specific clients, the full error hierarchy, +retry configuration, OpenTelemetry setup, and metrics remain in their defining modules. +Importing `agentloop` does not load optional provider SDKs or the OpenTelemetry SDK. diff --git a/docs/api/config.md b/docs/api/config.md index 82a7c7b..a9da812 100644 --- a/docs/api/config.md +++ b/docs/api/config.md @@ -8,7 +8,7 @@ Type alias for `"anthropic" | "openrouter" | "modal" | "openai"`. ## `Settings` -A frozen Pydantic model. Unknown fields are rejected. +A frozen Pydantic model. Unknown fields are rejected. The field defaults are: | Field | Type | Default | |---|---|---| @@ -27,9 +27,14 @@ A frozen Pydantic model. Unknown fields are rejected. | `max_attempts` | `int >= 1` | `3` | | `timeout_seconds` | `float > 0 \| None` | `None` (provider default) | +`max_attempts` counts the first request and any retries. `timeout_seconds` applies to one +model call. With OpenAI, OpenRouter, and Modal it replaces the read timeout; with Anthropic +it sets the SDK's overall request timeout. + ### `Settings.model` -Read-only property returning the model selected by `provider`. +Read-only property returning the model selected by `provider`. For example, with +`provider="openai"`, it returns `openai_model`. ### `Settings.from_env` @@ -38,14 +43,37 @@ Settings.from_env(env) -> Settings ``` Translates the variables listed in the [configuration guide](../getting-started/configuration.md). -Empty secret values become `None`. Pydantic validates values such as `LLM_MAX_ATTEMPTS`. +Pass a mapping explicitly, such as `os.environ` or a dictionary. The method does not read +the process environment or a dotenv file on its own. Empty secret values become `None`, +and Pydantic validates values such as `LLM_MAX_ATTEMPTS`. + +This example uses OpenAI settings. It uses a placeholder key because constructing and +closing the client sends no model request: ```python +from agentloop import Settings, build_client + settings = Settings.from_env({ "LLM_PROVIDER": "openai", - "OPENAI_API_KEY": "secret", + "OPENAI_API_KEY": "example-key", + "LLM_MAX_ATTEMPTS": "2", }) -assert settings.model == "gpt-5.1" +print(settings.provider, settings.model) +print(f"max attempts: {settings.max_attempts}") + +client = build_client(settings) +try: + print(client.name, client.model) +finally: + client.close() +``` + +Output: + +```text +openai gpt-5.1 +max attempts: 2 +openai gpt-5.1 ``` ## `build_client` @@ -55,9 +83,14 @@ build_client(settings, *, observer=None, http_client=None) -> LLMClient ``` Constructs the selected provider client and wraps it in `RetryingClient` using -`settings.max_attempts`. The optional observer receives retry events. Raises `AuthError` for -missing required credentials and `ConfigurationError` for incomplete Modal configuration, -an unsupported provider, or an `http_client` from the wrong package. +`settings.max_attempts`. The optional observer receives retry events. The caller owns the +returned client and should close it when finished, as in the example above. + +Missing OpenAI or OpenRouter credentials raise `AuthError` during construction. Anthropic +may resolve credentials through its SDK, so a missing credential can instead raise +`AuthError` on the first model request. Incomplete Modal configuration or an `http_client` +from the wrong package raises `ConfigurationError`. Invalid `Settings` values, including an +unsupported provider, are rejected by Pydantic. `http_client` carries your trust store, proxy or transport (see [corporate networks](../guides/providers.md#corporate-networks-trust-proxy-and-transport)). diff --git a/docs/api/errors.md b/docs/api/errors.md index 15de427..1d4e281 100644 --- a/docs/api/errors.md +++ b/docs/api/errors.md @@ -27,8 +27,9 @@ AgentLoopError - `TranscriptError`: a transcript is inconsistent or unsafe to send. - `ToolError`: a handler failure whose message is intended for the model. Raise this inside application tools; the executor converts it into an error result. -- `LoopBudgetExceeded`: the iteration or elapsed-time limit was reached. Its `kind`, - `limit`, and `actual` attributes identify the limit without requiring message parsing. +- `LoopBudgetExceeded`: the iteration, elapsed-time, or output-token limit was reached. + Its `kind` (`"iterations"`, `"deadline"`, or `"max_tokens"`), `limit`, and `actual` + attributes identify the limit without requiring message parsing. - `ConfigurationError`: required configuration is absent or contradictory. ## `ProviderError` @@ -47,10 +48,11 @@ ProviderError( ) ``` -Public attributes are `provider`, `status_code`, `retry_after`, `request_id`, `attempts`, -`retry_wait_seconds`, and the class attribute `retryable`. `RetryingClient` sets `attempts` -and `retry_wait_seconds`. Only `RateLimited`, `ProviderUnavailable`, `ProviderTimeout`, and -`ResponseInvalid` set `retryable = True`; `ProviderUnreachable` overrides it to `False`. +In addition to its constructor fields, a `ProviderError` exposes `attempts` (1 by default), +`retry_wait_seconds` (0.0 by default), and `attempt_log` (empty by default). `RetryingClient` +fills in these fields when a model request fails. The class attribute `retryable` is `True` +for `RateLimited`, `ProviderUnavailable`, `ProviderTimeout`, and `ResponseInvalid`. +`ProviderUnreachable` overrides it to `False`. ### Where it failed, and whether the request was sent @@ -70,14 +72,30 @@ reset connection and a read error stay retryable. not retried, since waiting does not add credit. An `AgentLoopError` raised from an active conversation also carries its -`ConversationSummary` as `.summary`. Errors raised before a conversation starts may have +`ConversationSummary` as `.summary`. Errors raised before a conversation starts have `.summary = None`. +This example leaves out the OpenAI key, so `build_client()` raises `AuthError`, a subtype +of `ProviderError`, before sending a request: + ```python +from agentloop import Settings, build_client from agentloop.errors import ProviderError try: - agent.run("hello") + build_client(Settings(provider="openai")) except ProviderError as error: - print(error.provider, error.status_code, error.request_id) + print(type(error).__name__, str(error)) + print(error.status_code, error.request_id, error.summary) +``` + +Output: + +```text +AuthError OPENAI_API_KEY is not set +None None None ``` + +The metadata fields are `None` here because no provider request or conversation began. +For errors from a model request, inspect `status_code`, `phase`, `request_sent`, and +`attempt_log` to understand what happened. diff --git a/docs/api/events.md b/docs/api/events.md index 38296a6..b7ce6b5 100644 --- a/docs/api/events.md +++ b/docs/api/events.md @@ -1,6 +1,7 @@ # `agentloop.events` -Ordered JSONL event records for debugging and audit workflows. +Ordered JSONL event records for debugging and audit workflows. `EventTraceObserver` +receives loop callbacks and writes one JSON object per event through a function you provide. ## `EventTraceObserver` @@ -17,18 +18,69 @@ EventTraceObserver( ``` `write` is a function that accepts a string, such as `file.write`, a queue's `put`, or a -logger method. The library does not choose or open the destination. +logger method. The library does not choose or open the destination. Each call receives a +JSON record followed by `terminator`; set `terminator=""` when the writer already adds a +line break. Set `max_string_chars=None` to disable clipping captured strings. + +Every record has `seq` (starting at 1 for each conversation), `ts` (Unix timestamp), `t` +(seconds since the conversation began), and `event`. Conversation, correlation, and parent +IDs are included when available. The possible event names are `conversation_start`, +`request`, `response`, `retry`, `model_error`, `tool_start`, `tool_end`, `stall`, +`budget_warning`, and `conversation_end`. `retry` reports an attempt that will be retried; +`model_error` reports a model call that failed after its final attempt. The final record +contains the complete `ConversationSummary`. By default, records contain metadata such as event names, times, identifiers, tool names, token counts, status codes, and result sizes. They exclude prompts, responses, tool -arguments, tool results, and error text. +arguments, tool results, and error text. Unknown optional fields are omitted from a record; +the nested summary can still contain `null` values. + +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses +OpenAI. Install its support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +Keep the trace in memory and inspect the event names and request metadata: + +```python +import json +import os + +from agentloop import Agent, EventTraceObserver + +lines = [] +trace = EventTraceObserver(lines.append) +with Agent.from_env(os.environ, tools=[], observer=trace) as agent: + agent.run("How many days are in a week? Reply with one digit.") + +records = [json.loads(line) for line in lines] +print([record["event"] for record in records]) +request = next(record for record in records if record["event"] == "request") +print(request["message_count"], "new_messages" in request) +``` + +Output: + +```text +['conversation_start', 'request', 'response', 'conversation_end'] +1 False +``` + +`message_count` reports the one user message sent to the model. `new_messages` is absent +because content capture is off. Set `capture_content=True` to include content. A `redact` function is then required unless `allow_unredacted=True` is explicitly set. Every captured string is redacted before it is limited to `max_string_chars`. If redaction fails, that event is dropped rather than written -without redaction. +without redaction. Structural values such as tool names, call IDs, and message roles do not +pass through the redactor; review them before retaining a trace. One observer instance supports one active conversation at a time. Create a separate instance for each concurrent conversation. -See the [event trace guide](../guides/event-trace.md) for the event list and examples. +See the [event trace guide](../guides/event-trace.md) for content-capture and redaction +examples. diff --git a/docs/api/index.md b/docs/api/index.md index 0bebe72..6e144ab 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,12 +1,14 @@ # API reference -The reference documents the supported, non-underscore-prefixed API. Most users start with -the [top-level package](agentloop.md); extension authors use the lower-level modules. +This reference covers the main public interfaces. Start with the +[top-level package](agentloop.md) for common imports, or use the lower-level modules +when you need direct control of clients, tools, observers, or conversation results. | Module | Purpose | |---|---| | [`agentloop`](agentloop.md) | Ergonomic re-exports | | [`agentloop.agent`](agent.md) | High-level agent and tool decorator | +| [`agentloop.collector`](../guides/summaries.md) | `SummaryCollector`, the loop's per-conversation usage and tool counter | | [`agentloop.config`](config.md) | Settings and client construction | | [`agentloop.errors`](errors.md) | Exception hierarchy | | [`agentloop.loop`](loop.md) | Core conversation loop | @@ -16,10 +18,15 @@ the [top-level package](agentloop.md); extension authors use the lower-level mod | [`agentloop.metrics`](metrics.md) | OpenTelemetry counters and histograms | | [`agentloop.provider`](provider.md) | Shared request, response, and client types | | [`agentloop.retry`](retry.md) | Retry policy and wrapper | +| [`agentloop.stall`](summary.md#summaryoptions) | `RepeatTracker`, used for `SummaryOptions.stall_threshold` reporting | | [`agentloop.tooling`](tooling.md) | Registry and tool execution | | [`agentloop.tracing`](tracing.md) | OpenTelemetry observer and setup | | [`agentloop.transcript`](transcript.md) | Conversation messages and parts | | [`agentloop.providers`](providers/index.md) | Built-in provider integrations | +The [provider index](providers/index.md) compares the built-in adapters and links to +their constructors. The [custom providers guide](../guides/custom-providers.md) explains +how to implement `LLMClient`. + Pydantic models in this package validate construction. Models described as frozen reject attribute assignment after construction. diff --git a/docs/api/loop.md b/docs/api/loop.md index 3605235..7560135 100644 --- a/docs/api/loop.md +++ b/docs/api/loop.md @@ -1,6 +1,8 @@ # `agentloop.loop` -The provider-independent reason/act/observe loop. +The provider-independent reason/act/observe loop. These functions are the lower-level API: +they run the full agent loop, while you assemble the client, tool registry, and dependencies +yourself. [`Agent`](agent.md) is the convenience API that assembles those pieces for you. ## Constants @@ -32,49 +34,81 @@ run_conversation( ) -> Transcript ``` -Mutates and returns `transcript`. Each iteration sends a provider-independent -`ModelRequest`, appends the response, executes requested tools, and appends their results. A -response truncated mid-tool-call, or truncated before producing anything readable, is discarded -and retried with a doubled token allowance up to the 64,000 ceiling. A budget is never lowered, -and when it cannot grow the run raises `LoopBudgetExceeded` with `kind="max_tokens"`. +Updates and returns the supplied `Transcript`. Each iteration sends a `ModelRequest` to +the client. The loop adds the model's response to the transcript, runs any requested local +tools, and adds their results before the next iteration. + +If a response ends partway through a tool call or before producing readable content, the +loop discards that response and retries with twice the token allowance, up to the 64,000 +ceiling. It never lowers a larger allowance supplied by the caller. If the allowance +cannot grow, it raises `LoopBudgetExceeded` with `kind="max_tokens"`. A provider that pauses a turn (`StopReason.PAUSED`, for example Anthropic's `pause_turn` while a hosted tool runs) is continued without sending a user message: the continuation joins the same assistant message, and each continuation counts as an iteration. -If `transcript` ends in tool calls with no results (a log cut off while tools ran), the loop -runs them before the first model call and then validates the transcript with -`check_invariants()`, so a bad one raises `TranscriptError` instead of a provider 400. +If `transcript` ends with tool calls that have no results, the loop runs them before the +first model call. It then checks the transcript for invalid message order and raises +`TranscriptError` if necessary. + +The deadline is checked before each model call. A provider call, its retries, or a started +tool batch may finish after it. `None` disables the deadline. Exhaustion raises +`LoopBudgetExceeded`; provider errors propagate unchanged. + +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses +OpenAI. Install its support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` -The deadline is checked between turns. A provider call, its retries, or a started tool batch -may finish after it. `None` disables the deadline. Exhaustion raises `LoopBudgetExceeded`; -provider errors propagate unchanged. +The example creates each component explicitly and records the model request and response: ```python -from agentloop import Deps, Registry, Transcript, run_conversation +import os + +from agentloop import ( + Deps, ListObserver, Registry, Settings, Transcript, + build_client, final_text, run_conversation, +) transcript = Transcript() -transcript.add_user_message("hello") -run_conversation(transcript, client, Registry([]), Deps()) +transcript.add_user_message("How many days are in a week? Reply with one digit.") +observer = ListObserver() +client = build_client(Settings.from_env(os.environ)) +try: + run_conversation(transcript, client, Registry([]), Deps(), observer=observer) + print(final_text(transcript)) + print(observer.kinds()) +finally: + client.close() ``` +Output: + +```text +7 +['request', 'response'] +``` + +`Registry([])` supplies no local tools, and `Deps()` supplies the default dependency +bundle. `final_text()` reads the last assistant answer from the updated transcript. + ## `run_conversation_result` Accepts the same parameters as `run_conversation` and returns a `ConversationResult` with -`.transcript`, `.summary`, and `.text`. The agent and conversation identifiers label the -summary. `summary_options` configures the timeline, repeated-call detection, and limit -warnings. See [Conversation summaries](../guides/summaries.md). +`.transcript`, `.summary`, and `.text`. The summary records the conversation's status and +usage. A completed answer, refusal, or truncated final answer returns normally; a budget +or provider failure raises an exception with its summary attached as `.summary`. The agent +and conversation identifiers label the summary. `summary_options` configures the timeline, +repeated-call detection, and limit warnings. See +[Conversation summaries](../guides/summaries.md). ## `ListObserver` -An observer for tests and debugging. Public attribute `events` contains ordered -`(kind, payload)` tuples. It implements `on_request`, `on_response`, `on_retry`, -`on_tool_start`, and `on_tool_end` by appending events. - -```python -observer = ListObserver() -# pass observer to an Agent or run_conversation -assert observer.kinds() == [kind for kind, _ in observer.events] -``` - -`kinds() -> list[str]` returns just the event names. +An observer for tests and debugging. The example above passes one to +`run_conversation()`. Its public `events` list contains ordered `(kind, payload)` tuples +for requests, responses, retries, and local tool starts and ends. `kinds() -> list[str]` +returns just the event names. diff --git a/docs/api/metrics.md b/docs/api/metrics.md index cec68d0..414b5ce 100644 --- a/docs/api/metrics.md +++ b/docs/api/metrics.md @@ -1,6 +1,7 @@ # `agentloop.metrics` -OpenTelemetry metrics for dashboards and alerts. +OpenTelemetry metrics for dashboards and alerts. `MetricsObserver` turns conversation +summaries and selected loop events into counters and histograms. ## `MetricsObserver` @@ -8,17 +9,99 @@ OpenTelemetry metrics for dashboards and alerts. MetricsObserver(meter=None) ``` -Records counters and histograms when a conversation ends. Pass an OpenTelemetry `Meter`, or -let the observer use `opentelemetry.metrics.get_meter("agentloop")`. The observer records -nothing until the application installs a `MeterProvider`. +Pass an OpenTelemetry `Meter`, or let the observer use +`opentelemetry.metrics.get_meter("agentloop")`. Install and configure a `MeterProvider` in +your application to collect the measurements; the SDK is available through the +`based-models-agentloop[otel]` extra. Most instruments are recorded from +`on_conversation_end(summary)`; the stall and budget-warning counters are recorded when +those events occur. `MetricsObserver` keeps no per-conversation state and can be shared by concurrent conversations. Use `Agent(metrics=True)` to add one automatically. -Metric attributes are limited to values such as status, agent name, provider, model, tool, -and error state. Conversation, correlation, and parent IDs are intentionally excluded because -unique values would create a separate metric series for every conversation. Use a trace or -event trace to inspect a specific conversation. +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses +OpenAI. Install its support and the OpenTelemetry SDK, then set the provider credentials: -See [OpenTelemetry tracing and metrics](../guides/tracing.md#metrics) for the full instrument -list. +```bash +python -m pip install 'based-models-agentloop[openai-compat,otel]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +An in-memory reader lets you inspect a metric without running an exporter: + +```python +import os + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader + +from agentloop import Agent +from agentloop.metrics import MetricsObserver + +reader = InMemoryMetricReader() +provider = MeterProvider(metric_readers=[reader]) +metrics.set_meter_provider(provider) +try: + meter = metrics.get_meter("agentloop") + observer = MetricsObserver(meter) + with Agent.from_env(os.environ, tools=[], observer=observer) as agent: + answer = agent.run("How many days are in a week? Reply with one digit.") + + data = reader.get_metrics_data() + conversations = next( + metric + for resource in data.resource_metrics + for scope in resource.scope_metrics + for metric in scope.metrics + if metric.name == "agentloop.conversations" + ) + point = conversations.data.data_points[0] + print(answer) + print(point.attributes["status"], point.value) +finally: + provider.shutdown() +``` + +Output: + +```text +7 +completed 1 +``` + +`metrics.get_meter("agentloop")` returns the OpenTelemetry `Meter` used by +`MetricsObserver`. Passing the observer to the agent records one completed conversation. +You can omit the `meter` argument to let `MetricsObserver()` call `get_meter("agentloop")` +itself. `Agent(metrics=True)` creates and attaches that default observer for you. + +In the table, **base** means `provider` and `model`, plus `agent_name` when one was +configured. **Outcome** means base plus `status`. + +| Instrument | Kind | Attributes | Recorded value | +|---|---|---|---| +| `agentloop.conversations` | Counter | Outcome | One per conversation | +| `agentloop.conversation.duration` | Histogram | Outcome | Elapsed seconds | +| `agentloop.conversation.iterations` | Histogram | Outcome | Model calls attempted | +| `agentloop.conversation.model_duration` | Histogram | Base | Model-call seconds, including retry waits | +| `agentloop.conversation.tool_duration` | Histogram | Base, `tool` | Sum of one tool's call durations in the conversation | +| `agentloop.tool.calls` | Counter | `tool`, `error` | Successful and failed local tool calls | +| `agentloop.tokens` | Counter | Base, `type` | Reported `input`, `output`, `cache_read`, `cache_write`, or `reasoning` tokens | +| `agentloop.cost` | Counter | Base | Provider-reported USD cost | +| `agentloop.provider.retries` | Counter | Base | Attempts beyond the first | +| `agentloop.provider.errors` | Counter | Base, `error_type`, `phase`, `request_sent` | Failed provider attempts | +| `agentloop.stalls` | Counter | `tool` | Repeated tool calls that reached the configured threshold | +| `agentloop.budget_warnings` | Counter | `kind` | Configured iteration or deadline warnings | + +`request_sent` is `"true"`, `"false"`, or `"unknown"`; `phase` is `"unknown"` when the +provider did not report it. Token counters skip zero or unreported values. Cost and retry +counters are recorded only when those values are present. Stall and budget-warning counters +require the corresponding [summary options](summary.md#summaryoptions). + +Conversation, correlation, and parent IDs are excluded from metric attributes because a +unique value per run would create a separate metric series for every conversation. Use a +trace or [event trace](events.md) to inspect a specific conversation. + +See [OpenTelemetry tracing and metrics](../guides/tracing.md#metrics) for a runnable +in-memory example. diff --git a/docs/api/observer.md b/docs/api/observer.md index 213ccf4..60b0637 100644 --- a/docs/api/observer.md +++ b/docs/api/observer.md @@ -1,43 +1,100 @@ # `agentloop.observer` -Callbacks for conversation, model-request, retry, and tool events. +An observer is an object that receives callbacks as the agent loop runs. The loop calls it +before and after model requests and local tool calls, when a request is retried, and when a +conversation ends. These callbacks let an application record what happened while the +conversation is still in progress. + +Pass an observer to `Agent` or `run_conversation()` with `observer=...`. For example, an +observer can count model responses, log tool activity, or send timing data to a tracing or +metrics system. The `ResponseCounter` below shows the smallest useful case: it increments +a counter each time a model response arrives. Subclass `NullObserver` to implement only the +callbacks your application needs. ## `LoopObserver` -A protocol with these methods: +`LoopObserver` is the callback interface accepted by the `observer=` parameter. It is a +Python `Protocol`: it describes the methods an observer can provide and the values each +method receives. You normally create an observer by subclassing `NullObserver` and +overriding the callbacks you need, as `ResponseCounter` does below. + +`conversation()` provides a context around the whole run. The `on_*` methods receive +individual events inside that context. The last six hooks in the table are optional at +runtime; if a custom observer omits them, the loop skips them. `NullObserver` supplies +no-op implementations of every hook. | Method | Called when | |---|---| -| `conversation()` | Enters a context covering one complete conversation | -| `on_request(request)` | Immediately before a provider call | -| `on_response(response)` | After a successful provider call | -| `on_retry(attempt, error)` | Before retrying a provider failure | -| `on_tool_start(call)` | Before a tool handler runs | -| `on_tool_end(call, result)` | After a tool produces its paired result | -| `on_conversation_start(context)` | Once, right after `conversation()` is entered, with the ids and limits; optional | -| `on_conversation_end(summary)` | Once per conversation, on success or failure, inside `conversation()`; optional | -| `on_budget_warning(kind, used, limit)` | A configured share of a limit has been used; optional | -| `on_stall(tool_name, count)` | A tool repeated the same call to the configured threshold; optional | -| `on_model_error(error)` | A model call failed for good (not retryable, or retries exhausted), before the exception leaves the loop; optional | -| `on_text_delta(text)` | Reserved for streaming; unused by the current loop | - -`conversation()` returns `AbstractContextManager[None]`. An observer can use it to create and -clean up state for one conversation. +| `conversation() -> AbstractContextManager[None]` | Enters a context covering one complete conversation | +| `on_request(request: ModelRequest)` | Immediately before a provider call | +| `on_response(response: ModelResponse)` | After a successful provider call | +| `on_retry(attempt: int, error: Exception)` | Before retrying a provider failure; `attempt` is the failed attempt number | +| `on_tool_start(call: ToolCallPart)` | Before a local tool call is handled, including a call to an unknown tool | +| `on_tool_end(call: ToolCallPart, result: ToolResultPart)` | After that call produces a result, including an error result | +| `on_conversation_start(context: ConversationContext)` | Once, after `conversation()` is entered, with the IDs and limits; optional | +| `on_conversation_end(summary: ConversationSummary)` | Once per conversation, on success or failure, inside `conversation()`; optional | +| `on_budget_warning(kind: BudgetKind, used: float, limit: float)` | A configured share of the iteration or deadline limit has been used; optional | +| `on_stall(tool_name: str, count: int)` | A tool repeated the same call to the configured threshold; optional | +| `on_model_error(error: Exception)` | A model call failed after its final attempt, before the exception leaves the loop; optional | +| `on_text_delta(text: str)` | Reserved for streaming; unused by the current loop | + +An observer can use `conversation()` to create and clean up state for one conversation. +`on_response` can run more than once in a conversation when the model calls tools; each +successful model call produces a response event. `on_conversation_end` runs once for the +whole conversation. + +`on_model_error` includes failures that are not retryable, so it can fire without any +`on_retry` event. ## `NullObserver` Implements every protocol method as a no-op and returns `nullcontext()` from -`conversation()`. Subclass it when only some events matter: +`conversation()`. Subclass it when only some events matter. + +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses +OpenAI. Install its support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +Count the successful model responses in one agent run: ```python -class RetryCounter(NullObserver): +import os + +from agentloop import Agent, NullObserver +from agentloop.provider import ModelResponse + + +class ResponseCounter(NullObserver): def __init__(self) -> None: self.count = 0 - def on_retry(self, attempt: int, error: Exception) -> None: + def on_response(self, response: ModelResponse) -> None: self.count += 1 + + +observer = ResponseCounter() +with Agent.from_env(os.environ, tools=[], observer=observer) as agent: + answer = agent.run("How many days are in a week? Reply with one digit.") + +print(answer) +print(observer.count) +``` + +Output: + +```text +7 +1 ``` +`on_response` runs after the provider returns. The counter records one response for this +conversation. + ## `ConsoleObserver` Subclasses `NullObserver`. `on_response()` prints nonblank text parts; @@ -52,8 +109,9 @@ MultiObserver(*observers) ``` Calls each observer in argument order. `conversation()` starts them in argument order and -closes them in reverse order. A failed callback in one observer does not prevent the others -from receiving the event. +closes them in reverse order. A failed `on_*` callback in one observer does not prevent the +others from receiving the event. An exception from a `conversation()` context still +propagates. ## `GuardedObserver` @@ -61,8 +119,9 @@ from receiving the event. GuardedObserver(inner) ``` -Uses the wrapped observer's `conversation()` context directly. All `on_*` methods catch and -log exceptions from the wrapped observer so reporting failures do not interrupt agent work. +Uses the wrapped observer's `conversation()` context directly, so exceptions from that +context propagate. All `on_*` methods catch and log exceptions from the wrapped observer +so reporting failures do not interrupt agent work. ## `guarded` diff --git a/docs/api/provider.md b/docs/api/provider.md index 573f4b1..f16e66c 100644 --- a/docs/api/provider.md +++ b/docs/api/provider.md @@ -1,6 +1,8 @@ # `agentloop.provider` -Provider-independent request and response types. Every model is frozen. +Provider-independent request and response types. The loop sends a `ModelRequest` to an +`LLMClient` and receives a `ModelResponse`; provider adapters translate between these +types and each provider's API. The Pydantic models on this page are frozen. ## `StopReason` @@ -24,6 +26,8 @@ the loop to continue it. | `strict` | `bool` | `True` | `parameters` is the bare JSON Schema, without a vendor-specific function envelope. +`strict=True` asks supporting providers to constrain tool-call arguments to that schema; +adapters omit or disable the setting when the selected API does not support it. ## `ModelRequest` @@ -43,16 +47,21 @@ the loop to continue it. ## `Usage` -Fields: `input_tokens` and `output_tokens` are non-negative integers defaulting to zero; -`cache_read_tokens`, `cache_write_tokens`, and `reasoning_tokens` are optional non-negative -integers; `cost_usd` is an optional non-negative float. `input_includes_cached` reports -whether `input_tokens` already includes cache reads. `None` means not reported, not zero. +| Field | Type | Default | +|---|---|---| +| `input_tokens`, `output_tokens` | non-negative `int` | `0` | +| `cache_read_tokens`, `cache_write_tokens`, `reasoning_tokens` | non-negative `int | None` | `None` | +| `cost_usd` | non-negative `float | None` | `None` | +| `input_includes_cached` | `bool` | `True` | + +`input_includes_cached` reports whether `input_tokens` already includes cache reads. A +`None` optional field means the provider did not report it; this differs from zero. ## `ModelResponse` | Field | Type | Default | |---|---|---| -| `content` | non-empty `tuple[ContentPart, ...]` | required | +| `content` | `tuple[ContentPart, ...]` | required | | `stop_reason` | `StopReason` | required | | `usage` | `Usage` | required | | `model` | `str` | required | @@ -61,18 +70,62 @@ whether `input_tokens` already includes cache reads. `None` means not reported, | `request_id` | `str | None` | `None` | | `attempts` | positive `int` | `1` | | `retry_wait_seconds` | non-negative `float` | `0.0` | +| `attempt_log` | `tuple[AttemptRecord, ...]` | `()` | | `origin` | `Origin | None` | `None` | +`content` may be empty only when `stop_reason=MAX_TOKENS` and the provider stopped before +producing usable content. `attempts`, `retry_wait_seconds`, and `attempt_log` describe +retries; the log is empty when the first attempt succeeded. `origin` holds the adapter's +wire representation for replay when available. + +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example calls +OpenAI directly. Install its support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +Build a request, send it through the configured client, and read the normalized response: + ```python -response = ModelResponse( - content=(TextPart(text="Hello"),), - stop_reason=StopReason.END_TURN, - usage=Usage(input_tokens=3, output_tokens=1), - model="demo-v1", - provider="demo", -) +import os + +from agentloop import Settings, Transcript, build_client +from agentloop.provider import ModelRequest +from agentloop.transcript import TextPart + +transcript = Transcript() +transcript.add_user_message("How many days are in a week? Reply with one digit.") +request = ModelRequest(messages=tuple(transcript.messages)) + +client = build_client(Settings.from_env(os.environ)) +try: + response = client.complete(request) + print("".join(part.text for part in response.content if isinstance(part, TextPart))) + print(response.stop_reason.value) +finally: + client.close() ``` +Output: + +```text +7 +end_turn +``` + +`build_client()` supplies a retrying `LLMClient`. This direct call makes one model request; +the higher-level [`Agent`](agent.md) also handles tool calls and transcript updates. + +## `AttemptRecord` + +One provider attempt in `attempt_log`. Its fields are `attempt` (one-based number), `ok`, +`error_type`, `status_code`, `phase`, `request_sent`, `request_id`, and `wait_seconds`. +Optional error fields are `None` when unknown. `wait_seconds` is the delay after this +attempt and is zero for the last attempt. + ## `LLMClient` Protocol with read-only `name: str` and `model: str` properties and: @@ -96,6 +149,8 @@ new attribute to stay an `LLMClient`. ## `HostedTool` A provider-run capability declared on each request. `kind` names the capability, such as -`"web_search"`, and `options` contains provider-independent settings. The provider client +`"web_search"`, and `options` contains settings for the adapter to translate. `kind` must +be a non-empty string and `options` defaults to an empty dictionary. The provider client must advertise support for the kind; unsupported hosted tools are rejected when the agent -is created. +is created. The separate `agentloop-tools` package offers validated helpers that create +these `HostedTool` objects. See [Hosted tools and web search](../guides/hosted-tools.md). diff --git a/docs/api/providers/anthropic.md b/docs/api/providers/anthropic.md index 4229306..e8ca635 100644 --- a/docs/api/providers/anthropic.md +++ b/docs/api/providers/anthropic.md @@ -10,11 +10,12 @@ Anthropic Messages API integration. The SDK is imported only when needed and req - `DEFAULT_TIMEOUT_SECONDS = 600.0` - `CONTEXT_OVERFLOW_STOP_REASON = "model_context_window_exceeded"` - `API = "messages"` -- `HOSTED_TOOLS = frozenset({"web_search"})`: the hosted kinds `AnthropicClient` runs. +- `HOSTED_TOOLS = frozenset({"web_search"})`: the hosted kinds this adapter supports. - `WEB_SEARCH_BASIC = "web_search_20250305"`: the default web search tool. - `WEB_SEARCH_FILTERED = "web_search_20260209"`: sent only when the hosted tool has - `dynamic_filtering=True` (Claude 4.6+). Claude then runs a program that filters results, so - the response carries no citations; see the [hosted tools guide](../../guides/hosted-tools.md). + `dynamic_filtering=True` (Claude 4.6+). Claude then runs a program that filters results, + which may leave the response without citations even when it has sources; see the + [hosted tools guide](../../guides/hosted-tools.md). ## Translation helpers @@ -33,9 +34,42 @@ These public helpers convert between shared Agentloop types and the Anthropic AP - `classify_sdk_error(exc) -> ProviderError`: maps an Anthropic SDK exception to the common hierarchy. +`request_to_wire` puts `ToolSchema.parameters` in `input_schema`, adds local and hosted +tools to the same `tools` list, and maps `tool_choice="required"` to Anthropic's `any` +when local tools are present. Hosted web search supports `allowed_domains`, +`blocked_domains`, `max_uses`, `user_location`, and `dynamic_filtering`. Supplying both +domain lists raises `ConfigurationError`; `search_context_size` is ignored with a warning. +It omits `temperature` because current Claude models reject that parameter. For usage, +Anthropic reports cache read and cache write tokens separately from input tokens. + +This example builds a request and inspects its translated payload. It needs no API key or +network connection. + ```python +from agentloop import HostedTool, Transcript +from agentloop.provider import ModelRequest +from agentloop.providers.anthropic import request_to_wire + +transcript = Transcript() +transcript.add_user_message("Hello") +request = ModelRequest( + messages=tuple(transcript.messages), + hosted_tools=(HostedTool(kind="web_search"),), + max_tokens=100, +) payload = request_to_wire(request, model="claude-opus-5") -assert payload["model"] == "claude-opus-5" + +print(payload["model"]) +print(payload["messages"][0]["content"][0]["text"]) +print(payload["tools"][0]["type"]) +``` + +Output: + +```text +claude-opus-5 +Hello +web_search_20250305 ``` ## `AnthropicClient` @@ -57,15 +91,17 @@ chosen model. translates SDK and credential failures. - `close() -> None`: closes the SDK connection pool and is safe to repeat. A client passed as `http_client` is left open. +- `__enter__()` returns the client; `__exit__()` calls `close()`. -`http_client` must be an `httpx2.Client`: the Anthropic SDK is built on httpx2, whose -classes are not httpx's. Any other object, an `httpx.Client` included, raises -`ConfigurationError`. It carries your trust store, proxy or transport and stays yours to -close. The SDK follows redirects by default; pass a client with `follow_redirects=False` to -have a 3xx reported. -- `__enter__()` returns the client; `__exit__()` closes it. +If you provide `http_client`, it must use the HTTP client class expected by the installed +Anthropic SDK. The current SDK uses `httpx2.Client`; passing a client from another HTTP +package raises `ConfigurationError`. It carries your trust store, proxy or transport and +stays yours to close. The SDK follows redirects by default; pass a client with +`follow_redirects=False` to have a 3xx reported. `pause_turn` maps to `StopReason.PAUSED`. Assistant turns are replayed from `Origin.raw`, so encrypted search results and signatures go back exactly as received. -When `api_key=None`, credential resolution is delegated to the Anthropic SDK. +When `api_key=None`, credential resolution is delegated to the Anthropic SDK. You can set +`ANTHROPIC_API_KEY` or supply `api_key` explicitly. A missing credential raises `AuthError` +when `complete()` is called. diff --git a/docs/api/providers/index.md b/docs/api/providers/index.md index bf7a787..8bdcc00 100644 --- a/docs/api/providers/index.md +++ b/docs/api/providers/index.md @@ -1,13 +1,21 @@ # `agentloop.providers` -Built-in provider integrations convert vendor APIs to the shared `LLMClient` interface. The -package initializer exports nothing; import constructors from their modules. +Built-in provider integrations convert vendor requests and responses to the shared +`LLMClient` interface. All four support local tools. Hosted web search is available only +with Anthropic Messages and OpenAI Responses. -- [Anthropic](anthropic.md) -- [OpenAI](openai.md) -- [OpenRouter](openrouter.md) -- [Modal](modal.md) +| Provider | Direct client | API | Hosted web search | +|---|---|---|---| +| [Anthropic](anthropic.md) | `AnthropicClient` | Messages | Yes | +| [OpenAI](openai.md) | `build_openai_client` | Responses by default; Chat Completions optional | Responses only | +| [OpenRouter](openrouter.md) | `build_openrouter_client` | Chat Completions | No | +| [Modal](modal.md) | `build_modal_client` | Chat Completions | No | -The modules `_common` and `_openai_compat` are private implementation details despite being -importable in Python. Applications should use `Settings`/`build_client`, the concrete public -constructors, or your own `LLMClient` implementation. +Use [`Settings` and `build_client`](../config.md) for provider selection and automatic +retries. Import a direct client from its module when you need provider-specific options. +The `agentloop.providers` package initializer exports no constructors. + +Modules whose names start with an underscore, such as `_common` and `_openai_compat`, are +internal implementation details. To add another provider, implement the +[`LLMClient` interface](../provider.md); see the +[custom providers guide](../../guides/custom-providers.md). diff --git a/docs/api/providers/modal.md b/docs/api/providers/modal.md index 5703551..c9322b3 100644 --- a/docs/api/providers/modal.md +++ b/docs/api/providers/modal.md @@ -1,13 +1,17 @@ # `agentloop.providers.modal` -Integration for a self-deployed vLLM OpenAI-compatible endpoint on Modal. Requires -`based-models-agentloop[openai-compat]`. +Integration for a self-deployed vLLM OpenAI-compatible endpoint on Modal. Install +`based-models-agentloop[openai-compat]` and set `MODAL_ENDPOINT_URL` to your deployment's +base URL. For automatic provider selection and retries, see the +[providers guide](../../guides/providers.md). ## Constants - `DEFAULT_MODEL = "Qwen/Qwen3.6-35B-A3B-FP8"` -- `TIMEOUT_SECONDS`: extended connect/read timeouts for cold starts. -- `QUIRKS`: compatibility flags for vLLM usage, errors, and parallel tool calls. +- `TIMEOUT_SECONDS`: connect 60, read 900, write 30, and pool 10 seconds. The longer + connect and read timeouts allow for cold starts. +- `QUIRKS`: the adapter does not expect detailed usage fields or JSON error bodies from + this endpoint, and disables parallel tool calls by default. ## `build_modal_client` @@ -19,25 +23,48 @@ build_modal_client( proxy_token_id="", proxy_token_secret="", http_client=None, + timeout_seconds=None, ) -> OpenAICompatClient ``` -Appends `/v1` to `endpoint_url` after removing trailing slashes. Proxy credentials are used -only when both values are nonempty. `Settings` enforces the both-or-neither rule; this -low-level constructor itself treats an incomplete pair as unauthenticated. `http_client` is -an `httpx.Client` carrying your trust store, proxy or transport; it stays yours to close, and -anything else raises `ConfigurationError`. +The builder returns a direct `LLMClient` named `modal`. It removes trailing slashes from +`endpoint_url`, adds `/v1`, and sends Chat Completions requests to +`/v1/chat/completions`. It has no hosted tools (`hosted_tools` is empty). + +If both proxy tokens are nonempty, the builder sends them as `Modal-Key` and +`Modal-Secret` headers. With one or both tokens empty, it sends neither header. +`Settings` rejects an incomplete pair when you use `build_client()`; this lower-level +builder does not. `timeout_seconds` overrides the 900-second read timeout while retaining +the other timeout defaults. + +This example builds a client for the configured endpoint and inspects its public +properties without sending a model request: ```python -client = build_modal_client( - endpoint_url="https://example--model.modal.run", - proxy_token_id="id", - proxy_token_secret="secret", -) +import os + +from agentloop.providers.modal import build_modal_client + +with build_modal_client( + endpoint_url=os.environ["MODAL_ENDPOINT_URL"], + proxy_token_id=os.getenv("MODAL_PROXY_TOKEN_ID", ""), + proxy_token_secret=os.getenv("MODAL_PROXY_TOKEN_SECRET", ""), +) as client: + print(client.name, client.api) + print(client.model) + print(sorted(client.hosted_tools)) ``` -It speaks Chat Completions (`api="completions"`) and has no hosted tools -(`hosted_tools` is empty). +Output: + +```text +modal completions +Qwen/Qwen3.6-35B-A3B-FP8 +[] +``` -The returned client exposes `name`, `model`, `complete(request)`, `close()`, and context -manager methods. Construct through `build_client` or wrap with `RetryingClient` for retries. +The client also exposes `complete(request)` and `close()`. Pass an `httpx.Client` as +`http_client` to configure a trust store, proxy, or transport. The client remains yours to +close, and `close()` leaves it open. Passing another HTTP client type raises +`ConfigurationError`. The builder does not add retries; use `build_client()` or +`RetryingClient` if you need them. diff --git a/docs/api/providers/openai.md b/docs/api/providers/openai.md index db82cc2..ffd9643 100644 --- a/docs/api/providers/openai.md +++ b/docs/api/providers/openai.md @@ -1,15 +1,18 @@ # `agentloop.providers.openai` -OpenAI integration using the shared OpenAI-compatible HTTP client. Requires -`based-models-agentloop[openai-compat]`. +OpenAI integration using the shared OpenAI-compatible HTTP client. Install +`based-models-agentloop[openai-compat]` and set `OPENAI_API_KEY`. For automatic provider +selection and retries, see the [providers guide](../../guides/providers.md). ## Constants - `BASE_URL = "https://api.openai.com/v1"` - `DEFAULT_MODEL = "gpt-5.1"` -- `TIMEOUT_SECONDS`: connect/read/write/pool timeout mapping. -- `QUIRKS`: compatibility settings, including `max_completion_tokens` (Chat Completions - only; Responses sends `max_output_tokens`). +- `TIMEOUT_SECONDS`: connect 10, read 300, write 30, and pool 10 seconds. +- `QUIRKS`: OpenAI's default compatibility settings. Chat Completions uses + `max_completion_tokens`; Responses uses `max_output_tokens`. +- `MODEL_QUIRKS` and `quirks_for_model(model)`: select model-specific settings. For + `o1`, `o3`, and `o4` model families, Chat Completions omits `parallel_tool_calls`. ## `build_openai_client` @@ -21,26 +24,56 @@ build_openai_client( organization=None, project=None, api="responses", + quirks=None, http_client=None, + timeout_seconds=None, ) -> OpenAICompatClient ``` -Constructs an `LLMClient` named `openai`. Optional organization and project values become -OpenAI request headers. +The builder returns a direct `LLMClient` named `openai`. `organization` and `project` add +the `OpenAI-Organization` and `OpenAI-Project` request headers. `quirks` overrides the +settings selected for the model; most applications can leave it unset. `timeout_seconds` +overrides the 300-second read timeout while retaining the other timeout defaults. -`api` picks the wire format: `"responses"` (the default) posts to `/responses`, and -`"completions"` keeps Chat Completions. The client exposes it as `client.api`, and the -hosted tools it can run as `client.hosted_tools` (`web_search` on Responses, none on -Completions). +`api="responses"` posts to `/responses` and supports hosted web search. It sends the +full transcript with `store: false`, so conversation history stays in the local +`Transcript`. `api="completions"` posts to `/chat/completions` and supports local tools, +but no hosted tools. Passing a hosted tool directly to a Completions client raises +`BadRequestError`; `Agent` rejects that combination during configuration. -`http_client` is an `httpx.Client` carrying your trust store, proxy or transport. It stays -yours to close; `close()` leaves it open. Anything else raises `ConfigurationError`. +Both formats return a normalized `ModelResponse`. The client exposes `name`, `model`, +`api`, `hosted_tools`, `complete(request)`, `close()`, and context manager methods. +`hosted_tools` contains `web_search` for Responses and is empty for Completions. + +This example sends the same request through each API and reads the response text: ```python -with build_openai_client(api_key="secret", project="project-id") as client: - response = client.complete(request) +import os + +from agentloop import Transcript, join_text +from agentloop.provider import ModelRequest +from agentloop.providers.openai import build_openai_client + +transcript = Transcript() +transcript.add_user_message("Reply with exactly: ready") +request = ModelRequest(messages=tuple(transcript.messages), max_tokens=500) + +for api in ("responses", "completions"): + with build_openai_client( + api_key=os.environ["OPENAI_API_KEY"], model="gpt-4.1-mini", api=api + ) as client: + response = client.complete(request) + print(client.api, join_text(response.content)) +``` + +Output: + +```text +responses ready +completions ready ``` -The returned client exposes `name`, `model`, `complete(request)`, `close()`, and context -manager methods. Retry wrapping is not added by this constructor; use `build_client` or -`RetryingClient` when retries are required. +Pass an `httpx.Client` as `http_client` to configure a trust store, proxy, or transport. +The client remains yours to close, and `close()` leaves it open. Passing another HTTP +client type raises `ConfigurationError`. The builder does not add retries; use +`build_client()` or `RetryingClient` if you need them. diff --git a/docs/api/providers/openrouter.md b/docs/api/providers/openrouter.md index fd76dd5..e914234 100644 --- a/docs/api/providers/openrouter.md +++ b/docs/api/providers/openrouter.md @@ -1,13 +1,14 @@ # `agentloop.providers.openrouter` -OpenRouter integration using the shared OpenAI-compatible HTTP client. Requires -`based-models-agentloop[openai-compat]`. +OpenRouter integration using the shared OpenAI-compatible HTTP client. Install +`based-models-agentloop[openai-compat]` and set `OPENROUTER_API_KEY`. For automatic +provider selection and retries, see the [providers guide](../../guides/providers.md). ## Constants - `BASE_URL = "https://openrouter.ai/api/v1"` - `DEFAULT_MODEL = "openrouter/free"` -- `TIMEOUT_SECONDS`: connect/read/write/pool timeout mapping. +- `TIMEOUT_SECONDS`: connect 10, read 300, write 30, and pool 10 seconds. ## `build_openrouter_client` @@ -19,25 +20,47 @@ build_openrouter_client( app_url=None, app_name=None, http_client=None, + timeout_seconds=None, ) -> OpenAICompatClient ``` -Constructs an `LLMClient` named `openrouter`. `app_url` and `app_name` set OpenRouter's -optional attribution headers. `http_client` is an `httpx.Client` carrying your trust store, -proxy or transport; it stays yours to close, and anything else raises `ConfigurationError`. +The builder returns a direct `LLMClient` named `openrouter`. `app_url` sets the optional +`HTTP-Referer` attribution header, and `app_name` sets `X-Title`. `timeout_seconds` +overrides the 300-second read timeout while retaining the other timeout defaults. + +OpenRouter uses Chat Completions (`api="completions"`) and has no hosted tools +(`hosted_tools` is empty). For reasoning models, it replays `reasoning_details` from an +assistant message's `Origin` when the message came from OpenRouter. This preserves the +provider's reasoning state across turns. The client exposes `name`, `model`, `api`, +`hosted_tools`, `complete(request)`, `close()`, and context manager methods. + +This example sends one request and reads the normalized response text: ```python -client = build_openrouter_client( - api_key="secret", - app_url="https://example.com", - app_name="Example", -) +import os + +from agentloop import Transcript, join_text +from agentloop.provider import ModelRequest +from agentloop.providers.openrouter import build_openrouter_client + +transcript = Transcript() +transcript.add_user_message("Reply with exactly: ready") +request = ModelRequest(messages=tuple(transcript.messages), max_tokens=100) + +with build_openrouter_client(api_key=os.environ["OPENROUTER_API_KEY"]) as client: + response = client.complete(request) + print(client.name, client.api) + print(join_text(response.content)) ``` -It speaks Chat Completions (`api="completions"`) and has no hosted tools -(`hosted_tools` is empty). For reasoning models it replays `reasoning_details` from the -message's `Origin`, unchanged and in order. +Output: + +```text +openrouter completions +ready +``` -The returned client exposes `name`, `model`, `complete(request)`, `close()`, and context -manager methods. Add `RetryingClient` explicitly or construct through `build_client` for -automatic retries. +Pass an `httpx.Client` as `http_client` to configure a trust store, proxy, or transport. +The client remains yours to close, and `close()` leaves it open. Passing another HTTP +client type raises `ConfigurationError`. The builder does not add retries; use +`build_client()` or `RetryingClient` if you need them. diff --git a/docs/api/retry.md b/docs/api/retry.md index a583fca..4f785b8 100644 --- a/docs/api/retry.md +++ b/docs/api/retry.md @@ -1,6 +1,7 @@ # `agentloop.retry` -Provider-independent retry behavior. +Provider-independent retry behavior. `RetryingClient` wraps any `LLMClient` and makes +another provider request when a failure is classified as retryable. ## Constants @@ -16,6 +17,8 @@ A frozen Pydantic model: | `base_delay` | positive `float` | `1.0` | | `max_delay` | positive `float` | `30.0` | +`max_attempts` includes the initial request. A value of 3 allows at most two retries. + ## `RetryingClient` ```text @@ -23,9 +26,10 @@ RetryingClient(inner, *, policy=None, observer=None) ``` Implements `LLMClient` by delegating to `inner` and retrying `ProviderError` instances whose -`retryable` class attribute is true. +`retryable` class attribute is true. Other exceptions propagate without a retry. -- `name` and `model` are read-only properties delegated to the inner client. +- `name` and `model` are read-only properties delegated to the inner client. The wrapper + also exposes its hosted-tool support and optional `api` value. - `complete(request)` returns the first successful response. Before each retry it calls `observer.on_retry(attempt, error)` and sleeps. - `close()` calls the inner client's `close()` when present. @@ -36,10 +40,66 @@ A successful response after a retry, and a terminal error, carry `attempt_log`: message, so it is safe to log. It is empty when the first attempt succeeded. Use it to answer "did any attempt reach the provider?". -`max_attempts` includes the initial request. `retry_after` is preferred over exponential -backoff, but both are capped at `max_delay`. +Between attempts, the delay doubles from `base_delay` and includes up to `JITTER_SECONDS` +of random jitter. A provider's `retry_after` value takes precedence. Either way, the delay +is capped at `max_delay`. + +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses an +OpenAI client. Install its support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +`FailOnce` can wrap any `LLMClient`; here it wraps OpenAI. It raises one retryable error +before the request is sent, then lets OpenAI answer: ```python -policy = RetryPolicy(max_attempts=2, base_delay=0.1, max_delay=1.0) -client = RetryingClient(raw_client, policy=policy) +import os + +from agentloop import Agent +from agentloop.errors import ProviderUnavailable +from agentloop.providers.openai import build_openai_client +from agentloop.retry import RetryingClient, RetryPolicy + + +class FailOnce: + def __init__(self, inner): + self.inner = inner + self.name = inner.name + self.model = inner.model + self.calls = 0 + + def complete(self, request): + self.calls += 1 + if self.calls == 1: + raise ProviderUnavailable("temporary connection failure", request_sent=False) + return self.inner.complete(request) + + +with build_openai_client(api_key=os.environ["OPENAI_API_KEY"]) as openai: + inner = FailOnce(openai) + client = RetryingClient( + inner, + policy=RetryPolicy(max_attempts=2, base_delay=0.01, max_delay=0.01), + ) + result = Agent([], client=client).run_result("Reply with exactly ready.") + +print("answer:", result.text) +print("attempts:", inner.calls) +print("retries:", result.summary.provider_retries) +``` + +Output: + +```text +temporary connection failure (attempt 1/2); retrying in 0.0s +answer: ready +attempts: 2 +retries: 1 ``` + +The wrapper makes two calls to `FailOnce`; only the second reaches OpenAI. The summary +counts one extra provider attempt. diff --git a/docs/api/summary.md b/docs/api/summary.md index b59c99f..6a9f535 100644 --- a/docs/api/summary.md +++ b/docs/api/summary.md @@ -5,23 +5,46 @@ prompts, model responses, tool arguments, tool results, or exception messages. ## `ConversationStatus` -Describes how a conversation ended: `completed`, `refused`, `truncated`, -`iteration_limit`, `deadline_exceeded`, `token_limit`, `provider_error`, `error`, or `cancelled`. - -Use `status.succeeded` to check whether the call returned normally and -`status.budget_exhausted` to check whether it reached an iteration, time or output-token limit. +Describes how a conversation ended. The `succeeded` and `budget_exhausted` properties are +computed from the status; they are not additional status values. + +| Status | Meaning | `succeeded` | `budget_exhausted` | +|---|---|---|---| +| `completed` | The model finished its answer | `True` | `False` | +| `refused` | The model declined the request | `True` | `False` | +| `truncated` | The final answer reached `max_tokens` | `True` | `False` | +| `iteration_limit` | The loop used all allowed iterations | `False` | `True` | +| `deadline_exceeded` | The loop reached its time limit | `False` | `True` | +| `token_limit` | A truncated tool call or empty response could not fit in a larger token allowance | `False` | `True` | +| `provider_error` | A provider call failed | `False` | `False` | +| `error` | Another exception stopped the loop | `False` | `False` | +| `cancelled` | A cancellation such as `KeyboardInterrupt` stopped the loop | `False` | `False` | + +The first three statuses return normally. The other statuses describe runs that raised; +the exception carries the summary as `.summary` when it is an `AgentLoopError`. ## `ConversationSummary` -A frozen, JSON-serializable record produced once for every conversation. It includes: +A frozen, JSON-serializable record produced once for every conversation. Its fields are: + +| Fields | Meaning | +|---|---| +| `conversation_id`, `correlation_id`, `parent_id` | Conversation ID and optional caller-supplied grouping IDs | +| `agent_name`, `provider`, `model` | Agent and model identity | +| `status`, `iterations`, `iteration_limit`, `deadline_seconds`, `elapsed_seconds` | Outcome, model calls attempted, configured limits, and elapsed time | +| `tool_calls`, `tool_errors`, `provider_retries`, `usage` | Tool counts, extra provider attempts, and aggregated token usage and cost | +| `model_seconds`, `tool_seconds`, `retry_wait_seconds`, `tool_duration_seconds` | Time spent on model calls, tool batches, retry waits, and each tool's calls | +| `max_repeated_tool_calls`, `stalled` | Repeated identical calls and whether the configured stall threshold was reached | +| `timeline` | Optional tuple of `TurnRecord` entries; `None` unless enabled | +| `error_type`, `provider_status_code`, `error_phase`, `error_request_sent`, `error_request_id`, `error_attempts_sent`, `failed_attempts` | Details of a failed run and grouped failed provider attempts | + +`cache_hit_rate` is a computed field: the share of input tokens served from cache, or +`None` when the provider did not report cache reads. The `succeeded` property reflects +`status.succeeded`. -- conversation, correlation, and parent identifiers; -- agent, provider, and model names; -- status, iterations, configured limits, and elapsed time; -- model, tool, and retry-wait time; -- tool calls, tool errors, retries, token use, and provider-reported cost; -- repeated-call and stall information; and -- an optional tuple of `TurnRecord` entries. +`model_seconds` includes retry waits, which are also reported separately in +`retry_wait_seconds`. Tools may run concurrently, so the sum of their individual durations +in `tool_duration_seconds` can exceed the wall time in `tool_seconds`. `None` means that a value was not reported or configured. It does not mean zero. For example, `usage.cost_usd` is `None` when the provider did not report cost. @@ -33,6 +56,40 @@ final assistant text. `Agent.run_result`, `Agent.converse_result`, and `run_conversation_result` return this type. +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses +OpenAI. Install its support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +Inspect the answer and summary after a run: + +```python +import os + +from agentloop import Agent + +with Agent.from_env(os.environ, tools=[]) as agent: + result = agent.run_result("How many days are in a week? Reply with one digit.") + +print(result.text) +print(result.summary.status.value) +print(result.summary.succeeded) +print(result.summary.iterations) +``` + +Output: + +```text +7 +completed +True +1 +``` + ## `SummaryOptions` ```text @@ -48,22 +105,27 @@ SummaryOptions( - `stall_threshold`: report a stall after the same tool is requested with identical arguments this many times. The minimum is 2; `None` disables stall events. - `warn_at`: fractions of the iteration and time limits that trigger budget warnings, such - as `(0.5, 0.8)`. + as `(0.5, 0.8)`. Each fraction must be greater than 0 and at most 1. Duplicate values + are removed and the values are sorted. These options report conditions but do not stop or otherwise change the conversation. ## `TurnRecord` -Metadata for one model call: duration, stop reason, token use, cost, attempts, tools, tool -errors, and whether a truncated call was discarded. When a call fails before returning, -`error_type` contains the exception class name, and `input_tokens` / `output_tokens` are -`None` (not reported), never `0`. `attempt_log` has one `AttemptRecord` per -attempt (empty when the first attempt succeeded), and `attempts` is the real count even for a -call that failed. +Metadata for one model call. `index` is its one-based iteration number; +`model_seconds` and `tool_seconds` measure the call and its tool work. `stop_reason`, +`input_tokens`, `output_tokens`, and `cost_usd` describe the model response. + +`attempts` counts provider attempts, and `attempt_log` holds one `AttemptRecord` per +attempt when retries occurred or the call failed. It is empty when the first attempt +succeeded. `tools` and `tool_errors` list tool names. `discarded` marks a truncated response +that the loop did not add to the transcript. When a call fails before returning, +`error_type` contains the exception class name and `input_tokens` / `output_tokens` are +`None` because no usage was reported. ## `FailureCount` -A count of failed provider attempts sharing one `error_type`, `phase` and `request_sent`. +A `count` of failed provider attempts sharing one `error_type`, `phase` and `request_sent`. `ConversationSummary.failed_attempts` holds one per combination, covering retried and final failures alike. `ConversationSummary.error_phase`, `error_request_sent`, `error_request_id` and `error_attempts_sent` describe the call that ended a failed run; `error_attempts_sent` is @@ -72,6 +134,8 @@ recorded. ## `ConversationContext` -The identifiers, names, and limits passed to an observer when a conversation starts. +The values passed to an observer when a conversation starts: `conversation_id`, +`correlation_id`, `parent_id`, `agent_name`, `provider`, `model`, `iteration_limit`, and +`deadline_seconds`. See [Conversation summaries](../guides/summaries.md) for examples and operational guidance. diff --git a/docs/api/tooling.md b/docs/api/tooling.md index 3e5da9d..e5ad44e 100644 --- a/docs/api/tooling.md +++ b/docs/api/tooling.md @@ -1,6 +1,15 @@ # `agentloop.tooling` -Tool types, registration, approval, and execution. +This module defines how the agent loop runs local Python tools. A tool has a model-visible +name, description, and argument schema, plus a Python handler that does the work. When the +model requests a tool, the loop passes the call to the tool executor and sends the result +back to the model. + +`ToolSpec` holds the definition and handler. `Registry` collects tools by name so the loop +can find them. `ToolDeps` supplies application data and an approval hook to the handler at +runtime. `run_tools()` executes the calls and returns a result for each one, including calls +that fail or are denied. The [`tool` decorator](agent.md#tool) creates a `ToolSpec` from a +function when you use the higher-level `Agent` API. ## Type aliases and constants @@ -25,7 +34,7 @@ bundles may add any other fields their handlers require. ## `ToolSpec` -A frozen Pydantic model containing the model-visible definition and handler. +A frozen Pydantic model containing the model-visible definition and its Python handler. | Field | Type | Meaning | |---|---|---| @@ -36,17 +45,9 @@ A frozen Pydantic model containing the model-visible definition and handler. | `parallel_safe` | `bool` | `True`. Set `False` for a handler that is not thread-safe: a turn that calls it runs all of its tool calls one after another, in call order | | `timeout_seconds` | `float \| None` | Per-call limit; `None` waits forever. On expiry the model gets an error result and the handler thread is abandoned (Python cannot kill it), so the handler should still set its own network timeouts | -```python -def ping(deps: ToolDeps) -> str: - return "pong" - -spec = ToolSpec( - name="ping", - description="Return pong.", - parameters={"type": "object", "properties": {}, "additionalProperties": False}, - handler=ping, -) -``` +`parameters` is sent to the model as the tool's argument schema. The executor binds the +returned arguments to the handler's Python signature; it does not independently validate +them against the full JSON Schema. Use a handler whose parameters match the schema. ## `Registry` @@ -70,10 +71,50 @@ Builds a flat name-to-tool mapping and caches handler signatures. Construction r run_tools(content, registry, deps, observer=None) -> list[ToolResultPart] ``` -Executes every `ToolCallPart` in `content`. No calls returns `[]`; one runs inline; several -run in a thread pool. Results always retain call order and every call produces exactly one -result. +Executes every `ToolCallPart` in `content`. No calls returns `[]`; one runs inline. Several +calls run in a thread pool of at most eight workers unless any called tool has +`parallel_safe=False`, in which case all calls in that batch run sequentially. Results +retain call order, even when calls finish out of order, and every call produces one result. The result is marked as an error for malformed arguments, unknown tools, approval denial, -signature mismatch, `ToolError`, approval failure, or unexpected handler failure. Observer -callbacks are guarded. +signature mismatch, timeout, `ToolError`, approval failure, or unexpected handler failure. +Handler return values are converted to strings. Observer callbacks are guarded. + +This example defines a tool, registers it, and runs one call without a model: + +```python +from agentloop import Deps, Registry, ToolSpec +from agentloop.tooling import run_tools +from agentloop.transcript import ToolCallPart + + +def echo(deps: Deps, text: str) -> str: + return text + + +spec = ToolSpec( + name="echo", + description="Return the supplied text.", + parameters={ + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + "additionalProperties": False, + }, + handler=echo, +) +call = ToolCallPart(id="call-1", tool_name="echo", arguments={"text": "hello"}) +result = run_tools([call], Registry([spec]), Deps())[0] +print(result.content) +print(result.is_error) +``` + +Output: + +```text +hello +False +``` + +`text` is supplied through the tool call. The executor passes `Deps()` as the handler's +first argument and returns a `ToolResultPart` linked to `call-1`. diff --git a/docs/api/tracing.md b/docs/api/tracing.md index 3c7bb30..b795a51 100644 --- a/docs/api/tracing.md +++ b/docs/api/tracing.md @@ -1,7 +1,15 @@ # `agentloop.tracing` -OpenTelemetry tracing for agent conversations. Importing this module needs only the -OpenTelemetry API dependency. Sending traces requires the optional `otel` extra. +Tracing records the steps of an agent conversation as timed OpenTelemetry spans. One span +covers the whole conversation; child spans cover each model request and local tool call. +The span hierarchy lets you see which work belongs to a conversation and where time was +spent, including retries and tool execution. + +`OTelObserver` creates these spans from agent loop callbacks. Pass it as `observer=` when +you want to configure tracing yourself, or use `Agent(trace=True)` to attach one with +default settings. Your application also installs a tracer provider to collect or export +the spans. Importing this module needs only the OpenTelemetry API; the optional `otel` +extra supplies the SDK and an OTLP exporter for the provider setup. ## Constants @@ -34,26 +42,79 @@ span for each local tool call. It implements all observer callbacks: (never the queries). - `on_tool_start()` starts a tool span keyed by call ID. - `on_tool_end()` records result metadata and ends the matching span. +- `on_conversation_start()` adds conversation IDs to the root span. +- `on_conversation_end()` adds the final status, counts, timing, and usage to the root span. +- `on_budget_warning()` and `on_stall()` add events to the root span. - `on_text_delta()` is currently a no-op reserved for streaming. With `capture_content=False`, prompts, system instructions, arguments, results, and error messages are excluded. Enabling it requires `redact` (a `str -> str` function) or -`allow_unredacted=True`, otherwise `ConfigurationError`. Every captured string is redacted -and then clipped; a `redact` that raises drops that attribute. `Agent(trace=True)` never -captures content: build an `OTelObserver` yourself to opt in. +`allow_unredacted=True`, otherwise `ConfigurationError`. A redactor replaces sensitive text +before export; see the [content-capture example](../guides/event-trace.md#capturing-content) +for a simple one. Every captured string passes through `redact`, and error details are then +clipped to `ERROR_DETAIL_CHARS`. If `redact` raises, that attribute is dropped. +`Agent(trace=True)` never captures content: build an `OTelObserver` yourself to opt in. One observer instance supports one active conversation. Overlapping `conversation()` contexts raise `RuntimeError`. Tool callbacks are synchronized because tools can run in parallel. +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses +OpenAI. Install its support and the OpenTelemetry SDK, then set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat,otel]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +An in-memory exporter lets you inspect spans without a collector: + ```python +import os + +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from agentloop import Agent, Settings +from agentloop.tracing import OTelObserver + +exporter = InMemorySpanExporter() +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(exporter)) +trace.set_tracer_provider(provider) + +settings = Settings.from_env(os.environ) observer = OTelObserver( - provider="openai", - model="gpt-5.1", - agent_name="support-agent", + provider=settings.provider, + model=settings.model, + agent_name="example-agent", ) +try: + with Agent.from_env(os.environ, tools=[], name="example-agent", observer=observer) as agent: + print("answer:", agent.run("Reply with exactly ready.")) + + operations = sorted( + span.attributes["gen_ai.operation.name"] + for span in exporter.get_finished_spans() + ) + print("operations:", operations) +finally: + provider.shutdown() +``` + +Output: + +```text +answer: ready +operations: ['chat', 'invoke_agent'] ``` +The `invoke_agent` span covers the conversation. The `chat` span covers its one model +request. The application owns the tracer provider and shuts it down when finished. + ## `configure_tracing` ```text @@ -63,7 +124,36 @@ configure_tracing(endpoint, *, service_name="agentloop") -> TracerProvider Configures the process to send batches of traces over OTLP/HTTP to `{endpoint}/v1/traces`. Returns the provider so the application can call `force_flush()` and `shutdown()`. Raises `ImportError` with installation guidance if the `otel` extra is absent. -The function does not read `OTEL_*` environment variables. +The function does not read `OTEL_*` environment variables. Configure the process-global +tracer provider once: a later call cannot replace an existing provider and the returned +provider will receive no spans. + +Set `OTLP_ENDPOINT` to the base URL of your OTLP/HTTP collector, such as +`http://localhost:4318`. With OpenAI configured as above: + +```python +import os + +from agentloop import Agent +from agentloop.tracing import configure_tracing + +provider = configure_tracing(os.environ["OTLP_ENDPOINT"], service_name="example-agent") +try: + with Agent.from_env(os.environ, tools=[], trace=True) as agent: + print(agent.run("Reply with exactly ready.")) + provider.force_flush() +finally: + provider.shutdown() +``` + +Output: + +```text +ready +``` + +`configure_tracing()` sends spans to the endpoint's `/v1/traces` path. `force_flush()` +waits for the current batch to be exported; `shutdown()` releases the provider's resources. Metrics are documented separately in the [`agentloop.metrics` reference](metrics.md) and the [tracing and metrics guide](../guides/tracing.md#metrics). diff --git a/docs/api/transcript.md b/docs/api/transcript.md index f61c878..01c4f2f 100644 --- a/docs/api/transcript.md +++ b/docs/api/transcript.md @@ -1,6 +1,9 @@ # `agentloop.transcript` -Provider-independent conversation history. +Provider-independent conversation history. A `Transcript` is an ordered list of user and +assistant `Message` objects. Each message contains one or more content parts, such as text, +a local tool call, or its result. Provider adapters convert their wire formats to these +types so the loop can use one transcript shape across providers. ## Type aliases @@ -18,11 +21,7 @@ Base model carrying the shared immutable/strict configuration. It has no data fi ### `TextPart` -Fields: `type="text"`, required `text: str`, and optional `citations`. - -```python -part = TextPart(text="Hello") -``` +Fields: `type="text"`, required `text: str`, and `citations: tuple[Citation, ...] = ()`. ### `ToolCallPart` @@ -45,21 +44,32 @@ required `content: str`, and `is_error: bool = False`. ### `ThinkingPart` -Fields: `type="thinking"`, required `text`, and optional `signature`, `provider`, `model`, -and `raw`. `raw` preserves a vendor block for byte-faithful replay. Reasoning text may be -empty even when a signed block is present. +Fields: `type="thinking"`, required `text: str`, and optional `signature: str`, +`provider: str`, `model: str`, and `raw`. `raw` preserves a provider block for exact replay. +Reasoning text may be empty even when a signed block is present. ### `HostedToolPart` -Describes a provider-run tool call. Fields include `kind`, optional `call_id`, optional -`query`, `sources`, and optional `error`. The loop records this part for inspection but does -not execute it as a local Python tool. +Describes a provider-run tool call. Fields are `type="hosted_tool"`, required non-empty +`kind: str`, optional `call_id: str`, `query: str`, and `error: str`, and +`sources: tuple[Source, ...] = ()`. The loop records this part for inspection but does not +execute it as a local Python tool. ### `Source` and `Citation` -`Source` records the URL and optional title of a page consulted by a provider-hosted tool. -`Citation` connects assistant text to a URL and may include a title, quoted text, or text -offsets when the provider reports them. +`Source` has a required `url: str` and optional `title: str`. It records a page consulted by +a provider-hosted tool. `Citation` has a required `url: str` and optional `title: str`, +`cited_text: str`, `start: int`, and `end: int`. It connects assistant text to a URL and +records quoted text or text offsets when the provider reports them. + +## `Origin` + +Provider-specific information attached to an assistant message. Its fields are +`provider: str`, `api: str`, `model: str`, and `raw: tuple[dict[str, Any], ...]`. An adapter +for the same provider and API can replay `raw` to continue the conversation faithfully. +Other adapters rebuild the message from its provider-independent content parts. +`matches(provider, api) -> bool` reports whether an adapter can use this origin; it returns +`False` when `provider` is `None`. ## `Message` @@ -82,16 +92,9 @@ Mutable Pydantic model with public `messages: list[Message]`, defaulting to an e `TranscriptError` on mismatch. - `pending_tool_calls() -> tuple[ToolCallPart, ...]`: calls in the final assistant message, or an empty tuple. -- `check_invariants() -> None`: validates call/result adjacency, completeness, and unique - call IDs, and that hosted-tool parts appear only in assistant messages; raises - `TranscriptError` if the transcript is unsafe to send. - -```python -transcript = Transcript() -transcript.add_user_message("hello") -assert transcript.pending_tool_calls() == () -transcript.check_invariants() -``` +- `check_invariants() -> None`: validates call/result adjacency, completeness, unique call + IDs within each message, and that hosted-tool parts appear only in assistant messages. + It raises `TranscriptError` if the transcript is unsafe to send. ## Serialization and extraction @@ -101,7 +104,8 @@ transcript.check_invariants() to_jsonl(messages) -> str ``` -Returns one JSON-serialized message per line with a trailing newline. It performs no I/O. +Returns one JSON-serialized message per line with a trailing newline when there are +messages. It performs no I/O; an empty input returns an empty string. ### `from_jsonl` @@ -112,13 +116,50 @@ from_jsonl(lines) -> Transcript Parses any iterable of lines, skipping blanks. Invalid JSON or models raise validation errors. It intentionally does not call `check_invariants()`. +### `join_text` + +```text +join_text(parts) -> str +``` + +Joins text parts from one message. It adds a space between separate blocks when needed, +while preserving mid-sentence citation splits and punctuation. + ### `final_text` ```text final_text(transcript) -> str ``` -Joins the text parts of the last assistant message with `join_text`: a space goes between -separate blocks ("I'll search." then "The answer is..."), but not across Anthropic's mid-sentence -citation splits or before closing punctuation. Returns `""` when there is no assistant +Uses `join_text` on the last assistant message. Returns `""` when there is no assistant message or it contains no text. + +Create a transcript, serialize it, and restore it: + +```python +from agentloop import Transcript, final_text +from agentloop.transcript import TextPart, from_jsonl, to_jsonl + +transcript = Transcript() +transcript.add_user_message("Hello") +transcript.add_assistant_message((TextPart(text="Hi!"),)) +transcript.check_invariants() + +jsonl = to_jsonl(transcript.messages) +restored = from_jsonl(jsonl.splitlines()) +restored.check_invariants() +print(final_text(restored)) +print(len(restored.messages)) +print(jsonl.count("\n")) +``` + +Output: + +```text +Hi! +2 +2 +``` + +The two JSONL lines represent the user and assistant messages. `from_jsonl()` recreates +both; `final_text()` reads the assistant reply. diff --git a/docs/examples/custom-observer.md b/docs/examples/custom-observer.md index 80b01e6..d216924 100644 --- a/docs/examples/custom-observer.md +++ b/docs/examples/custom-observer.md @@ -1,48 +1,88 @@ # Custom observer +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. These examples use +OpenAI. Install its extra and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +## Count tokens with a callback + +An observer receives events while the agent runs. Subclass `NullObserver` and override only +the callback you need. `on_response()` runs after each successful model response: + ```python import os -from agentloop import Agent, MultiObserver, NullObserver +from agentloop import Agent, NullObserver from agentloop.provider import ModelResponse class TokenCounter(NullObserver): def __init__(self) -> None: - self.total = 0 + self.input_tokens = 0 + self.output_tokens = 0 def on_response(self, response: ModelResponse) -> None: - self.total += response.usage.input_tokens - self.total += response.usage.output_tokens + self.input_tokens += response.usage.input_tokens + self.output_tokens += response.usage.output_tokens counter = TokenCounter() -observer = MultiObserver(counter) +with Agent.from_env(os.environ, tools=[], observer=counter) as agent: + answer = agent.run("How many days are in a week? Reply with one digit.") + +print("answer:", answer) +print("input tokens:", counter.input_tokens) +print("output tokens:", counter.output_tokens) +``` -with Agent.from_env(os.environ, tools=[], observer=observer) as agent: - print(agent.run("Hello")) +Output: -print(f"reported tokens: {counter.total}") +```text +answer: 7 +input tokens: 19 +output tokens: 11 ``` -Subclassing `NullObserver` keeps the implementation compatible with callbacks it does not -need. +Passing `counter` as `observer` registers it with the agent. After each successful model +call, the agent loop calls `counter.on_response(response)` automatically; your code does +not call the method directly. -## A JSONL trace of every conversation +`input_tokens` and `output_tokens` are integers in `ModelResponse`. Other usage fields, +such as `cache_read_tokens`, may be `None` when the provider does not report them. -`EventTraceObserver` is a ready-made observer that writes one JSON line per event through a -callable you choose. It is content-free unless you ask, and then it needs a redactor: +## Record JSONL events + +For a ready-made event log, use `EventTraceObserver`. Its writer receives one JSONL line per +event. This example keeps those lines in memory and prints their sequence numbers and names: ```python -import re +import json +import os + from agentloop import Agent, EventTraceObserver -def redact(text: str) -> str: - return re.sub(r"sk-[A-Za-z0-9]{20,}", "[KEY]", text) +lines = [] +trace = EventTraceObserver(lines.append) +with Agent.from_env(os.environ, tools=[], observer=trace) as agent: + agent.run("Reply with exactly ready.") + +for line in lines: + record = json.loads(line) + print(json.dumps({"seq": record["seq"], "event": record["event"]})) +``` + +Output: -with open("trace.jsonl", "a") as log: - trace = EventTraceObserver(log.write, capture_content=True, redact=redact) - agent = Agent(tools, client=client, observer=trace) - agent.run("find the lead investor", correlation_id="round-42") +```text +{"seq": 1, "event": "conversation_start"} +{"seq": 2, "event": "request"} +{"seq": 3, "event": "response"} +{"seq": 4, "event": "conversation_end"} ``` -The last line of each conversation is `conversation_end`, carrying the full summary. Use one -`EventTraceObserver` per concurrent conversation. See the [event trace guide](../guides/event-trace.md). +The default trace contains metadata, without prompt or response text. See the +[event trace guide](../guides/event-trace.md) to write the lines to a file or capture +content with a redactor. diff --git a/docs/examples/custom-provider.md b/docs/examples/custom-provider.md index 433bc8f..7d01da7 100644 --- a/docs/examples/custom-provider.md +++ b/docs/examples/custom-provider.md @@ -1,6 +1,7 @@ # Custom provider -This offline client makes documentation and application tests deterministic: +A custom provider implements the `LLMClient` interface: `name`, `model`, and +`complete(request)`. This fixed client returns a reply without contacting a model service: ```python from agentloop import Agent @@ -20,10 +21,19 @@ class FixedClient: provider=self.name, ) -agent = Agent([], client=FixedClient()) -assert agent.run("ignored") == "Hello from a fixed response." +client = FixedClient() +agent = Agent([], client=client) +print(agent.run("Say hello.")) ``` -No close method is required by `LLMClient`. When a custom client owns resources, provide a -`close()` method and close it in application code; a directly constructed `Agent` borrows -the client. +Output: + +```text +Hello from a fixed response. +``` + +The agent passes a `ModelRequest` to `complete()` and uses the returned `ModelResponse` to +finish the conversation. `LLMClient` does not require a `close()` method. An `Agent` created +with `client=` borrows that client; if your implementation owns resources, close it in your +application code. See [implementing a custom provider](../guides/custom-providers.md) for +request translation and optional capabilities. diff --git a/docs/examples/index.md b/docs/examples/index.md index 905befe..c315d72 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -1,11 +1,11 @@ # Examples -These pages collect complete patterns that can be adapted directly: - -- [Minimal agent](minimal-agent.md) -- [Tool-enabled agent](tool-enabled-agent.md) -- [Custom observer](custom-observer.md) -- [Traced agent](traced-agent.md) -- [Custom provider](custom-provider.md) - - +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. Start with a single +model call, then add a local tool or observe the agent loop. The first four examples use +OpenAI; the custom provider example runs locally without a model service. + +- [Minimal agent](minimal-agent.md): ask one question and print the answer. +- [Tool-enabled agent](tool-enabled-agent.md): let the model call a local Python function. +- [Custom observer](custom-observer.md): count tokens and record JSONL events. +- [Traced agent](traced-agent.md): inspect conversation and model spans in memory. +- [Custom provider](custom-provider.md): implement a client that returns a fixed response. diff --git a/docs/examples/minimal-agent.md b/docs/examples/minimal-agent.md index 9249d19..d1b84ca 100644 --- a/docs/examples/minimal-agent.md +++ b/docs/examples/minimal-agent.md @@ -1,21 +1,32 @@ # Minimal agent +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses OpenAI. + +Install OpenAI support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +Create an agent and ask one question: + ```python import os from agentloop import Agent -def main() -> None: - with Agent.from_env( - os.environ, - tools=[], - system="Answer in one sentence.", - ) as agent: - print(agent.run("Why is the sky blue?")) +with Agent.from_env(os.environ, tools=[], system="Answer briefly.") as agent: + print(agent.run("What is 2 + 2? Answer with one digit.")) +``` + +Output: -if __name__ == "__main__": - main() +```text +4 ``` -Set one of the provider configurations from [Hello World](../getting-started/hello-world.md) -before running the program. +`Agent.from_env()` uses the environment mapping you pass it. `run()` starts a new +conversation and returns the final text. The `with` block closes the client when the run +finishes. See [Agents](../guides/agents.md) to continue a conversation or add tools. diff --git a/docs/examples/tool-enabled-agent.md b/docs/examples/tool-enabled-agent.md index 75b1ab6..5e454b1 100644 --- a/docs/examples/tool-enabled-agent.md +++ b/docs/examples/tool-enabled-agent.md @@ -1,32 +1,51 @@ # Tool-enabled agent +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses OpenAI. + +Install OpenAI support and set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +Give the agent a local function it can call: + ```python import os -from agentloop import Agent, ConsoleObserver, tool +from agentloop import Agent, tool -WEATHER_SCHEMA = { +@tool(parameters={ "type": "object", - "properties": { - "city": {"type": "string", "description": "City name"}, - }, - "required": ["city"], + "properties": {"word": {"type": "string"}}, + "required": ["word"], "additionalProperties": False, -} - -@tool(parameters=WEATHER_SCHEMA) -def weather(city: str) -> str: - """Return example weather for a city.""" - return f"The example forecast for {city} is sunny." +}) +def count_letters(word: str) -> str: + """Count the letters in a word.""" + return str(len(word)) with Agent.from_env( os.environ, - tools=[weather], - observer=ConsoleObserver(), - system="Use the weather tool for forecasts.", + tools=[count_letters], + system="Use count_letters to count letters before answering.", ) as agent: - agent.run("What is the weather in Lisbon?") + result = agent.run_result( + "How many letters are in agent? Use count_letters. Reply with just the number." + ) + print(result.text) + print(result.summary.tool_calls) +``` + +Output: + +```text +5 +{'count_letters': 1} ``` -`ConsoleObserver` prints the tool call, result, and final response, so the return value from -`run()` is intentionally not printed again. +The schema tells the model what argument to pass. When it calls `count_letters`, the agent +executes the Python function and sends its result back to the model. `run_result()` returns +the final answer and a summary; `tool_calls` confirms that the tool ran once. diff --git a/docs/examples/traced-agent.md b/docs/examples/traced-agent.md index 1e67594..791a582 100644 --- a/docs/examples/traced-agent.md +++ b/docs/examples/traced-agent.md @@ -1,27 +1,66 @@ # Traced agent +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses OpenAI. +Install OpenAI support and the OpenTelemetry extra, then set the provider credentials: + +```bash +python -m pip install 'based-models-agentloop[openai-compat,otel]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +`trace=True` attaches the library's `OTelObserver`. It sets the OpenTelemetry +[`gen_ai.operation.name`](https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-spans.md) +attribute to `invoke_agent` on the conversation span and `chat` on each model-request span. +These are operation values set by `agentloop`. The full span names also include the agent or +model name. An in-memory exporter lets this example inspect the spans without a collector: + ```python import os -from agentloop import Agent, ConsoleObserver -from agentloop.tracing import configure_tracing +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from agentloop import Agent -endpoint = os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] -tracer_provider = configure_tracing(endpoint, service_name="example-agent") +exporter = InMemorySpanExporter() +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(exporter)) +trace.set_tracer_provider(provider) try: with Agent.from_env( os.environ, tools=[], - observer=ConsoleObserver(), name="example-agent", trace=True, ) as agent: - agent.run("Hello") - tracer_provider.force_flush() + print("answer:", agent.run("Reply with exactly ready.")) + + spans = { + span.attributes["gen_ai.operation.name"]: span + for span in exporter.get_finished_spans() + } + print("operations:", sorted(spans)) + chat = spans["chat"] + conversation = spans["invoke_agent"] + print("chat is child of agent:", chat.parent.span_id == conversation.context.span_id) finally: - tracer_provider.shutdown() + provider.shutdown() +``` + +Output: + +```text +answer: ready +operations: ['chat', 'invoke_agent'] +chat is child of agent: True ``` -The application explicitly reads the endpoint and owns the returned tracer provider's -lifecycle. +The example reads those operation values from the exported spans. `chat.parent.span_id` is +the ID of the model span's parent, and `conversation.context.span_id` is the conversation +span's own ID; their equality shows that the model request belongs to this conversation. +The application configures and shuts down the tracer provider. For collector export and +metrics, see the [tracing guide](../guides/tracing.md). diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 071b43e..e740cd3 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -1,21 +1,37 @@ # Configuration -Use `Settings` when you want to validate configuration separately from constructing an -agent: +Use `Settings` to validate a mapping before constructing a client. This example creates an +OpenAI client with a placeholder credential; constructing and closing it makes no model +request: ```python -import os - from agentloop import Settings, build_client -settings = Settings.from_env(os.environ) +settings = Settings.from_env({ + "LLM_PROVIDER": "openai", + "OPENAI_API_KEY": "example-key", + "OPENAI_MODEL": "example-model", + "LLM_MAX_ATTEMPTS": "2", +}) client = build_client(settings) try: print(client.name, client.model) + print(f"max attempts: {settings.max_attempts}") finally: client.close() ``` +Output: + +```text +openai example-model +max attempts: 2 +``` + +The application supplies the mapping. Pass `os.environ` when you want process environment +variables; `Settings.from_env()` does not read them on its own. `build_client()` applies +the configured retry policy, and the caller closes the client it created. + ## Environment variables | Variable | Default | Meaning | @@ -27,6 +43,7 @@ finally: | `ANTHROPIC_MODEL` | `claude-opus-5` | Anthropic model | | `OPENAI_API_KEY` | unset | OpenAI credential | | `OPENAI_MODEL` | `gpt-5.1` | OpenAI model | +| `OPENAI_API` | `responses` | `responses` or `completions` | | `OPENROUTER_API_KEY` | unset | OpenRouter credential | | `OPENROUTER_MODEL` | `openrouter/free` | OpenRouter model or router | | `MODAL_ENDPOINT_URL` | unset | Base URL of the deployed endpoint | @@ -34,8 +51,8 @@ finally: | `MODAL_PROXY_TOKEN_ID` | unset | Optional Modal proxy token ID | | `MODAL_PROXY_TOKEN_SECRET` | unset | Optional Modal proxy token secret | -You can construct `Settings` directly, which is useful when configuration comes from a -secrets service rather than environment variables: +You can also construct `Settings` directly when values come from application configuration +or a secrets service: ```python from pydantic import SecretStr @@ -44,10 +61,21 @@ from agentloop import Settings settings = Settings( provider="openai", - openai_api_key=SecretStr("secret supplied at runtime"), - openai_model="gpt-5.1", + openai_api_key=SecretStr("example-key"), + openai_model="example-model", max_attempts=2, ) +print(settings.provider, settings.model, settings.max_attempts) ``` +Output: + +```text +openai example-model 2 +``` + +The keys above are placeholders; no request is sent by either example. Replace them with +application-supplied credentials before calling a provider. `Settings` rejects unknown +fields, invalid provider names, and nonpositive attempt or timeout values. + See the [`config` API](../api/config.md) for every field. diff --git a/docs/getting-started/hello-world.md b/docs/getting-started/hello-world.md index 94b3bf5..682a413 100644 --- a/docs/getting-started/hello-world.md +++ b/docs/getting-started/hello-world.md @@ -1,7 +1,37 @@ # Hello World across providers -The program is identical for every supported provider. Configuration chooses the provider -integration. +Start with a scripted client to see the agent loop run without an API key. `FakeClient` +returns the response you give it and records the request the loop sent. + +```python +from agentloop import Agent +from agentloop.provider import ModelResponse, StopReason, Usage +from agentloop.providers.fake import FakeClient +from agentloop.transcript import TextPart + +client = FakeClient( + ModelResponse( + content=(TextPart(text="Hello, world!"),), + stop_reason=StopReason.END_TURN, + usage=Usage(input_tokens=3, output_tokens=4), + model="fake-1", + provider="fake", + ) +) +agent = Agent([], client=client) +print(agent.run("Say hello.")) +print(f"model requests: {len(client.requests)}") +``` + +Output: + +```text +Hello, world! +model requests: 1 +``` + +To call a live provider, replace the `FakeClient` and `Agent` construction above with +`Agent.from_env`. The same `run()` call works for every built-in provider: ```python import os @@ -9,56 +39,62 @@ import os from agentloop import Agent with Agent.from_env(os.environ, tools=[]) as agent: - print(agent.run("Reply with exactly: Hello, world!")) + print(agent.run("Say hello.")) ``` -Save it as `hello.py`, configure one provider, and run `python hello.py`. +Save the live example as `hello.py`, choose one configuration below, and run +`python hello.py`. The reply comes from the selected model, so its exact text can vary. +`Agent.from_env` receives `os.environ` explicitly; the library does not read the process +environment or a `.env` file on its own. ## Anthropic ```bash pip install 'based-models-agentloop[anthropic]' export LLM_PROVIDER=anthropic -export ANTHROPIC_API_KEY='...' -export ANTHROPIC_MODEL='claude-opus-5' # optional +export ANTHROPIC_API_KEY='your-key' python hello.py ``` When `ANTHROPIC_API_KEY` is omitted, the Anthropic SDK may use another credential source it -supports, such as an authenticated CLI profile. +supports, such as an authenticated CLI profile. The default model is `claude-opus-5`; set +`ANTHROPIC_MODEL` to choose another. ## OpenAI ```bash pip install 'based-models-agentloop[openai-compat]' export LLM_PROVIDER=openai -export OPENAI_API_KEY='...' -export OPENAI_MODEL='gpt-5.1' # optional +export OPENAI_API_KEY='your-key' python hello.py ``` +OpenAI uses the Responses API and model `gpt-5.1` by default. Set `OPENAI_MODEL` to choose +another model or `OPENAI_API=completions` for Chat Completions compatibility. + ## OpenRouter ```bash pip install 'based-models-agentloop[openai-compat]' export LLM_PROVIDER=openrouter -export OPENROUTER_API_KEY='...' -export OPENROUTER_MODEL='openrouter/free' # optional +export OPENROUTER_API_KEY='your-key' python hello.py ``` +The default model is `openrouter/free`; set `OPENROUTER_MODEL` to choose another. + ## Modal ```bash pip install 'based-models-agentloop[openai-compat]' export LLM_PROVIDER=modal -export MODAL_ENDPOINT_URL='https://example--model.modal.run' -export MODAL_ENDPOINT_MODEL='Qwen/Qwen3.6-35B-A3B-FP8' # optional +export MODAL_ENDPOINT_URL='https://your-deployment.modal.run' python hello.py ``` -For an authenticated Modal proxy, set both `MODAL_PROXY_TOKEN_ID` and -`MODAL_PROXY_TOKEN_SECRET`. Setting only one is a configuration error. +The default endpoint model is `Qwen/Qwen3.6-35B-A3B-FP8`; set `MODAL_ENDPOINT_MODEL` to +match your deployment. For an authenticated Modal proxy, set both `MODAL_PROXY_TOKEN_ID` +and `MODAL_PROXY_TOKEN_SECRET`. Setting only one is a configuration error. -Do not commit credentials. `Agent.from_env` receives a mapping explicitly; it does not load -`.env` files or inspect the process environment on its own. +Keep credentials out of source control. See [configuration](configuration.md) for the full +list of settings. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 70af86b..70c5349 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -1,30 +1,62 @@ # Installation -based-models-agentloop requires Python 3.12 or newer. The import name is `agentloop`. -Install the extra for the provider you use: +`based-models-agentloop` requires Python 3.12 or newer. Install the distribution named +`based-models-agentloop`; import it in Python as `agentloop`. + +For the core agent loop and its offline `FakeClient`, no provider extra is needed: + +```bash +python -m pip install based-models-agentloop +``` + +Choose an extra when you connect to a provider or export telemetry: + +| Extra | Adds | +|---|---| +| `anthropic` | Anthropic SDK | +| `openai-compat` | HTTP client for OpenAI, OpenRouter, and Modal/vLLM | +| `otel` | OpenTelemetry SDK and OTLP HTTP exporter | +| `all` | All three extras above | + +For example, install **one** provider extra with either of these commands: ```bash -pip install 'based-models-agentloop[anthropic]' -pip install 'based-models-agentloop[openai-compat]' -pip install 'based-models-agentloop[otel]' +python -m pip install 'based-models-agentloop[anthropic]' +python -m pip install 'based-models-agentloop[openai-compat]' ``` -`openai-compat` supports OpenAI, OpenRouter, and Modal. `otel` adds OpenTelemetry tracing, -metrics, and the OTLP HTTP exporter. Install everything with: +Add the `otel` extra if the application exports spans or metrics. Install every integration +with `all`: ```bash -pip install 'based-models-agentloop[all]' +python -m pip install 'based-models-agentloop[otel]' +python -m pip install 'based-models-agentloop[all]' ``` -For a checkout of this repository: +Check that the package imports in the selected Python environment: + +```bash +python -c 'import agentloop; print("agentloop ready")' +``` + +Output: + +```text +agentloop ready +``` + +## Work from this repository + +With Python 3.12 or newer available as `python`, create a virtual environment and install +the project with its development dependencies: ```bash python -m venv .venv source .venv/bin/activate -pip install -e '.[all,dev]' +python -m pip install -e '.[all,dev]' ``` -No provider SDK is imported until its client is constructed, so a basic `import agentloop` -does not require every optional dependency. +Provider SDKs and the OpenTelemetry SDK are imported only when their integrations are +used. A basic `import agentloop` works without those optional extras. Next: [Hello World](hello-world.md). diff --git a/docs/guides/agents.md b/docs/guides/agents.md index 7b1fb85..f840b62 100644 --- a/docs/guides/agents.md +++ b/docs/guides/agents.md @@ -1,7 +1,21 @@ # Agents -`Agent` keeps a model client, tools, application dependencies, and loop settings together so -they can be reused across conversations. +`Agent` keeps a model client, tools, application dependencies, and loop settings together +so they can be reused across conversations. These examples use OpenAI. Set up the provider +before running them: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +export LLM_PROVIDER=openai +export OPENAI_API_KEY='your-key' +``` + +`Agent.from_env()` receives `os.environ` explicitly. The library does not read environment +variables on its own. + +## One prompt + +`run()` starts a new conversation and returns its final text: ```python import os @@ -11,15 +25,27 @@ from agentloop import Agent with Agent.from_env( os.environ, tools=[], - system="Be concise.", + system="Answer briefly.", name="demo", ) as agent: - answer = agent.run("What is 2 + 2?") - print(answer) + print(agent.run("What is 2 + 2?")) ``` -`run()` creates a transcript internally and returns only the final text. Use `converse()` -when you want to keep the messages and continue the same conversation: +Output: + +```text +2 + 2 = 4. +``` + +Each `run()` call creates its own transcript. Use `run_result()` when you also need that +transcript and a `ConversationSummary`. A `ConversationSummary` records how the run ended, +including status, timing, model and tool calls, retries, and token usage, without storing +the conversation text. + +## Continue a conversation + +`converse()` advances a `Transcript` you supply. It adds messages to that same object and +returns it, so the next call includes the earlier messages: ```python import os @@ -31,46 +57,62 @@ transcript = Transcript() with Agent.from_env(os.environ, tools=[]) as agent: transcript.add_user_message("My name is Ada.") returned = agent.converse(transcript) - - print(returned is transcript) # True - print(final_text(transcript)) # For example: "Nice to meet you, Ada." - print(len(transcript.messages)) # 2: one user message and one reply + print(returned is transcript) + print(final_text(transcript)) + print(f"messages: {len(transcript.messages)}") transcript.add_user_message("What is my name?") agent.converse(transcript) + print(final_text(transcript)) + print(f"messages: {len(transcript.messages)}") +``` - print(final_text(transcript)) # For example: "Your name is Ada." - print(len(transcript.messages)) # 4: both turns are still present +Output: + +```text +True +Nice to meet you, Ada. How can I help you today? +messages: 2 +Your name is Ada. +messages: 4 ``` -`converse()` appends the model reply—and any intermediate tool calls and results—to the -provided `Transcript`. It returns that exact object for convenience; it does not create a -copy. This lets the next call include everything the model has already seen. +The first printed value is `True`: `converse()` returns the updated transcript. With no +tools, the transcript has two messages after the first turn and four after the second. +`converse()` also runs registered local tools and appends their calls and results when the +model requests them. Use `converse_result()` for the updated transcript and its summary. -## Who closes the provider client? +## Client ownership -`Agent.from_env()` constructs a provider client for you, so the returned agent owns that -client. Use a context manager, as above, or call `agent.close()` when finished. Leaving the -context closes the provider's connection pool even if the conversation raises an exception. +`Agent.from_env()` constructs a provider client from a mapping you supply. The returned +agent owns that client: a `with` block or `agent.close()` closes it, including when a run +raises an exception. -When you construct an agent with an existing client, ownership stays with your application: +An agent constructed with an existing client borrows it. Close that client yourself: ```python +import os + from agentloop import Agent, Settings, build_client -client = build_client(Settings(provider="anthropic")) +client = build_client(Settings.from_env(os.environ)) try: - agent = Agent([], client=client) - print(agent.run("Hello")) + with Agent([], client=client) as agent: + print(agent.run("Say hello.")) finally: - client.close() # The caller created the client, so the caller closes it. + client.close() +``` + +Output: + +```text +Hello. ``` -Calling `agent.close()` in this second example would deliberately leave `client` open. That -makes it possible to reuse one client across several agents without one agent unexpectedly -closing it for the others. +The agent's context manager leaves this borrowed client open. This allows several agents +to reuse one client without one agent closing it for the others. -`trace=True` attaches an `OTelObserver`. A traced agent supports only one conversation at a -time; create one traced agent per concurrent conversation. +`trace=True` attaches an `OTelObserver`. A traced agent supports one active conversation at +a time; create separate traced agents for concurrent conversations. See the [`agent` API](../api/agent.md). diff --git a/docs/guides/conversations.md b/docs/guides/conversations.md index cdff144..9b0e239 100644 --- a/docs/guides/conversations.md +++ b/docs/guides/conversations.md @@ -1,100 +1,109 @@ # Conversations and transcripts -The transcript is the canonical, provider-independent conversation history. +A `Transcript` holds a conversation as provider-independent messages. Use `converse()` when +you want to keep those messages and continue later. With OpenAI configured as in +[Hello World](../getting-started/hello-world.md): ```python -from agentloop import Transcript, final_text +import os + +from agentloop import Agent, Transcript, final_text transcript = Transcript() transcript.add_user_message("Explain recursion in one sentence.") -agent.converse(transcript) + +with Agent.from_env(os.environ, tools=[]) as agent: + agent.converse(transcript) print(final_text(transcript)) +print(f"messages: {len(transcript.messages)}") transcript.check_invariants() ``` -Messages contain typed parts: text, model thinking, tool calls, and tool results. Tool -results must immediately answer every call in the preceding assistant message. Use -`add_tool_results()` rather than appending result messages manually; it verifies identifiers -and restores call order. +Output: + +```text +Recursion is a technique where a function solves a problem by calling a smaller version of itself until it reaches a simple base case that it can solve directly. +messages: 2 +``` + +Messages contain typed parts such as text, model thinking, tool calls, and tool results. +`Agent.converse()` runs registered tools and adds their results to the transcript for you. -`Agent.converse()` executes registered tools and calls `add_tool_results()` for you. Call it -yourself when you execute a pending tool call outside the agent loop or construct a transcript -manually: +## Add tool results yourself + +Call `add_tool_results()` when you execute a pending tool call outside the agent loop or +construct a transcript manually. Results may arrive out of order; the method places them +in the model's original call order: ```python from agentloop import Transcript from agentloop.transcript import ToolCallPart, ToolResultPart transcript = Transcript() -transcript.add_user_message("Read a.py and b.py") -transcript.add_assistant_message( - [ - ToolCallPart( - id="call-a", - tool_name="read_file", - arguments={"path": "a.py"}, - ), - ToolCallPart( - id="call-b", - tool_name="read_file", - arguments={"path": "b.py"}, - ), - ] -) - -# Results may finish out of order. add_tool_results() puts them back into the -# model's original call order before appending the user-role result message. -transcript.add_tool_results( - [ - ToolResultPart( - call_id="call-b", - tool_name="read_file", - content="contents of b.py", - ), - ToolResultPart( - call_id="call-a", - tool_name="read_file", - content="contents of a.py", - ), - ] -) - -assert [part.call_id for part in transcript.messages[-1].content] == [ - "call-a", - "call-b", -] +transcript.add_user_message("Look up A and B.") +transcript.add_assistant_message([ + ToolCallPart(id="call-a", tool_name="lookup", arguments={"key": "A"}), + ToolCallPart(id="call-b", tool_name="lookup", arguments={"key": "B"}), +]) +transcript.add_tool_results([ + ToolResultPart(call_id="call-b", tool_name="lookup", content="Result B"), + ToolResultPart(call_id="call-a", tool_name="lookup", content="Result A"), +]) + +print([part.call_id for part in transcript.messages[-1].content]) +transcript.check_invariants() ``` -Every pending call must have exactly one result. A missing, unexpected, or duplicate call ID -raises `TranscriptError` instead of appending an invalid result message. +Output: -## Checking transcript invariants +```text +['call-a', 'call-b'] +``` -`check_invariants()` validates the relationships that individual Pydantic models cannot -validate on their own. It checks that: +Every pending call needs exactly one result. A missing, unexpected, or duplicate call ID +raises `TranscriptError` before an invalid result message is added. -- tool call IDs are unique within a message; -- tool results immediately follow the assistant message that requested them; -- every requested call has exactly one result; and -- the transcript does not end with unanswered tool calls. +## Check transcript invariants -It returns `None` when the transcript is safe to send and raises `TranscriptError` when it is -not: +`check_invariants()` verifies that tool-call IDs are unique, results immediately follow +their calls, every call has one result, and no calls remain unanswered. It returns `None` +when the transcript is safe to send and raises `TranscriptError` otherwise: ```python +from agentloop import Transcript from agentloop.errors import TranscriptError +from agentloop.transcript import ToolCallPart, ToolResultPart + +transcript = Transcript() +transcript.add_user_message("Look up A.") +transcript.add_assistant_message([ + ToolCallPart(id="call-a", tool_name="lookup", arguments={"key": "A"}), +]) try: transcript.check_invariants() -except TranscriptError as exc: - print(f"Transcript is not sendable: {exc}") +except TranscriptError as error: + print(error) + +transcript.add_tool_results([ + ToolResultPart(call_id="call-a", tool_name="lookup", content="Result A"), +]) +transcript.check_invariants() +print("transcript is sendable") +``` + +Output: + +```text +message 1 ends with unanswered tool calls ['call-a']; the transcript is not sendable +transcript is sendable ``` -Call it before sending a transcript assembled or loaded by your application. Deserialization -does not run this check automatically: a transcript captured midway through a tool turn is a -valid saved artifact even though it is not yet safe to send back to a model. Complete the -pending calls with `add_tool_results()` and then check it again. +Call this check before sending a transcript assembled or loaded by your application. +Deserialization does not run it automatically: a saved transcript may legitimately end +mid-turn, before a tool has returned. Complete pending calls with `add_tool_results()` and +then check again. See [transcript storage](transcript-storage.md) and the [`transcript` API](../api/transcript.md). diff --git a/docs/guides/custom-providers.md b/docs/guides/custom-providers.md index 916ed84..207d995 100644 --- a/docs/guides/custom-providers.md +++ b/docs/guides/custom-providers.md @@ -1,8 +1,11 @@ # Implement a custom provider -Implement the `LLMClient` protocol: `name`, `model`, and `complete()`. +Implement the `LLMClient` protocol with `name`, `model`, and +`complete(request: ModelRequest) -> ModelResponse`. This small client echoes the latest user +message to show the interface without provider-specific wire formats: ```python +from agentloop import Agent from agentloop.provider import ModelRequest, ModelResponse, StopReason, Usage from agentloop.transcript import TextPart @@ -22,34 +25,42 @@ class EchoClient: model=self.model, provider=self.name, ) -``` - -Use it directly with `Agent`: - -```python -from agentloop import Agent agent = Agent([], client=EchoClient()) print(agent.run("hello")) ``` -Custom providers should convert vendor failures into the -[`ProviderError` hierarchy](../api/errors.md), preserve unknown stop reasons as -`StopReason.UNKNOWN`, and keep vendor-specific types out of the public method signature. +Output: + +```text +Echo: hello +``` + +`Agent` calls `complete()` with a provider-independent `ModelRequest`. The client returns +canonical content, a normalized stop reason, and usage. For a real provider, translate its +wire request and response inside the client, convert vendor failures into the +[`ProviderError` hierarchy](../api/errors.md), and keep vendor types out of the public +method signature. Map unfamiliar stop reasons to `StopReason.UNKNOWN` and retain the +original value in `raw_stop_reason`. A response containing a `ToolCallPart` must use +`StopReason.TOOL_USE` unless it was cut off at `StopReason.MAX_TOKENS`. ## Optional capabilities - **Pause.** Return `StopReason.PAUSED` when the provider stopped partway through a turn it is - running itself. The loop sends the turn back unchanged, with no user message, and joins the - continuation to the same assistant message. + running itself. The loop sends the unfinished turn back without a new user message and joins + the continuation to the same assistant message. - **Hosted tools.** Expose a `hosted_tools` attribute (a `frozenset` of kinds such as `"web_search"`) to accept `HostedTool` declarations. Without it the client supports none, and `Agent` rejects them at construction. See `supported_hosted_tools`. - **Exact replay.** Set `ModelResponse.origin` to an `Origin(provider=..., api=..., model=..., - raw=...)` holding your wire form of the turn. Replay it when `origin.matches(provider, api)` - and fall back to the canonical parts otherwise. + raw=...)` holding your wire form of the turn. When a later request contains a message whose + origin matches your provider and API, replay its raw blocks; otherwise translate its + canonical parts. - **Empty turns.** `ModelResponse.content` may be empty only when `stop_reason` is - `StopReason.MAX_TOKENS`; the loop answers that with a bigger budget. Any other empty turn - should raise `ResponseInvalid`. + `StopReason.MAX_TOKENS`; the loop discards that turn and can retry with a bigger budget. + Convert other empty provider responses to `ResponseInvalid`. + +`Agent` borrows a client passed with `client=`. If your client owns a connection, close it in +your application code. See the [`provider` API](../api/provider.md). diff --git a/docs/guides/event-trace.md b/docs/guides/event-trace.md index 1a626da..5996234 100644 --- a/docs/guides/event-trace.md +++ b/docs/guides/event-trace.md @@ -5,18 +5,44 @@ requests, responses, retries, and tool calls in order with their timing. Use it or audit workflows when a [conversation summary](summaries.md) does not provide enough detail. +With OpenAI configured as in [Hello World](../getting-started/hello-world.md), this example +writes a trace to a temporary file and prints the event names from its JSONL records: + ```python +import json +import os +from pathlib import Path +from tempfile import TemporaryDirectory + from agentloop import Agent, EventTraceObserver -with open("trace.jsonl", "a") as log: - trace = EventTraceObserver(log.write) - agent = Agent(tools, client=client, observer=trace) - agent.run("summarize the open support case", correlation_id="case-42") +with TemporaryDirectory() as directory: + path = Path(directory) / "trace.jsonl" + with path.open("w") as log: + trace = EventTraceObserver(log.write) + with Agent.from_env(os.environ, tools=[], observer=trace) as agent: + agent.run("Reply with exactly ready.", correlation_id="example-1") + + for line in path.read_text().splitlines(): + record = json.loads(line) + shown = {"seq": record["seq"], "event": record["event"]} + if "summary" in record: + shown["status"] = record["summary"]["status"] + print(json.dumps(shown)) +``` + +Output: + +```text +{"seq": 1, "event": "conversation_start"} +{"seq": 2, "event": "request"} +{"seq": 3, "event": "response"} +{"seq": 4, "event": "conversation_end", "status": "completed"} ``` The library never opens a file or selects a logging service. You provide the function that -receives each line, such as `file.write`, a queue's `put`, or a logger's `info` with -`terminator=""`. +receives each line, such as `file.write` or a queue's `put`. For a logger, pass its `info` +method and set `terminator=""` on `EventTraceObserver`. ## What a line contains @@ -25,38 +51,60 @@ event name, conversation identifiers, and relevant counts, names, sizes, or stat It does not contain prompts, model responses, tool arguments, tool results, or error text. The event names are `conversation_start`, `request`, `response`, `retry`, `model_error`, -`tool_start`, `tool_end`, `stall`, `budget_warning`, and `conversation_end`. `retry` reports an attempt that will be retried; `model_error` reports a call that failed for -good, with its `error_type`, `status_code`, `phase`, `request_sent`, `request_id`, `attempts` -and `attempt_log`. A field that is unknown is left out of the line, so a missing -`request_sent` means unknown, not `false`. The final event carries the -complete `ConversationSummary`, so the trace also serves as a JSONL log of summaries. +`tool_start`, `tool_end`, `stall`, `budget_warning`, and `conversation_end`. `retry` reports an +attempt that will be retried. `model_error` reports a call that failed for good, with fields +such as `error_type`, `status_code`, `phase`, `request_sent`, `request_id`, `attempts`, and +`attempt_log` when available. Unknown top-level event fields are omitted, so a missing +`request_sent` means unknown, not `false`. The final event embeds the complete +`ConversationSummary`, whose optional fields may be `null`. ## Capturing content +Pass a redaction function to capture prompts and responses. The example uses a simple tag +to show where your application's redactor belongs: + ```python -import re +import json +import os + +from agentloop import Agent, EventTraceObserver + +lines = [] def redact(text: str) -> str: - return re.sub(r"sk-[A-Za-z0-9]{20,}", "[KEY]", text) + return text.replace("private-tag", "[REDACTED]") + +trace = EventTraceObserver(lines.append, capture_content=True, redact=redact) +with Agent.from_env(os.environ, tools=[], observer=trace) as agent: + agent.run("Reply with exactly ready. My tag is private-tag.") + +request = next(json.loads(line) for line in lines if '"event": "request"' in line) +print(request["new_messages"][0]["parts"][0]["content"]["text"]) +print("private tag in trace:", "private-tag" in "".join(lines)) +``` + +Output: -trace = EventTraceObserver(log.write, capture_content=True, redact=redact) +```text +Reply with exactly ready. My tag is [REDACTED]. +private tag in trace: False ``` With `capture_content=True`, the trace also carries the system prompt once, messages added since the previous request, response content, tool arguments and results, and error messages. -- **Every captured string passes through `redact`**, and is then limited to - `max_string_chars` (default 2000 characters). Redaction runs first, so a cut cannot expose - part of a secret. +- **Every captured string passes through `redact`**, then is clipped after + `max_string_chars` (default 2000 characters), with a truncation marker appended. Redaction + runs first, so a cut cannot expose part of a secret. - **A redactor is required.** `capture_content=True` without one raises `ConfigurationError` unless you pass `allow_unredacted=True`. - **The library ships no patterns.** A built-in list of things that look like secrets would promise more than it can keep; plug in your organization's scrubber. - **It fails closed.** If `redact` raises an exception, that event is dropped and logged. It is never written without redaction. -- Structure you chose (tool names, ids, roles, part types) is not redacted; everything the - model, a tool or a provider produced is. +- Structural fields such as tool names, call IDs, roles, and part types are not passed through + `redact`. Review those fields before retaining a trace. ## Using it with others diff --git a/docs/guides/hosted-tools.md b/docs/guides/hosted-tools.md index 89b8ac8..99255aa 100644 --- a/docs/guides/hosted-tools.md +++ b/docs/guides/hosted-tools.md @@ -1,80 +1,90 @@ # Hosted tools and web search -A `HostedTool` is a capability the provider runs for you, such as web search. Declare it with -`Agent(tools=[...])` beside your local `ToolSpec`s. `native_web_search(...)` in agentloop-tools -builds one. Native and client-side search are separate, explicit choices: a local `ToolSpec` -named like a hosted kind raises `ConfigurationError`. +`HostedTool` is the declaration that `agentloop` passes to a model provider. The provider +runs it; there is no Python handler in your process. OpenAI's Responses API and Anthropic's +Messages API support hosted `web_search`; OpenAI Chat Completions, OpenRouter, and Modal do +not. + +The separate `agentloop-tools` library builds these same `HostedTool` objects through +validated helpers such as `native_web_search()`. It also reads sources and citations from +the transcript and offers domain-policy checks. `agentloop` checks support, translates +requests for each provider, and records search activity in the transcript. The example below +uses the core type directly, so it needs only `based-models-agentloop` and the OpenAI extra. + +With OpenAI configured as in [Hello World](../getting-started/hello-world.md), declare +web search alongside any local tools: ```python -from agentloop import Agent, HostedTool +import os + +from agentloop import Agent, HostedTool, HostedToolPart +from agentloop.transcript import TextPart -agent = Agent([HostedTool(kind="web_search")], client=client) +search = HostedTool( + kind="web_search", + options={ + "allowed_domains": ["docs.python.org"], + "search_context_size": "low", + }, +) + +with Agent.from_env( + os.environ, + tools=[search], + system="Search the web before answering. Answer in one sentence.", +) as agent: + result = agent.run_result("Find the official Python documentation homepage URL.") + +print(result.text) +parts = [part for message in result.transcript.messages for part in message.content] +searches = [part for part in parts if isinstance(part, HostedToolPart)] +print(f"search calls: {len(searches)}") +print(f"sources: {sum(len(part.sources) for part in searches)}") +print(f"citations: {sum(len(part.citations) for part in parts if isinstance(part, TextPart))}") ``` -Support depends on the client (`supported_hosted_tools(client)`). Anthropic and OpenAI over -Responses support `web_search`; Chat Completions providers do not. Approval is per request, not -per search: `before_tool` never sees a hosted call. +Output: -**Options.** Unknown options and restriction options a provider cannot honour raise; tuning -options it cannot honour are logged and ignored. OpenAI takes `search_context_size`. +```text +The official Python documentation homepage URL is `https://docs.python.org/3/`. +search calls: 1 +sources: 12 +citations: 0 +``` -## Sources and citations +The provider runs the search and returns its activity in the assistant message. A +`HostedToolPart` records the search query and consulted `Source` entries; citations attached +to `TextPart` connect answer text to a URL. A source shows what the provider consulted, while +a citation shows what the answer cites. Neither contains the searched page's full text. -The two evidence readers mean different things. +## Support and options -| | `sources(transcript)` | `citations(transcript)` | +`Agent` checks hosted-tool support when it is constructed. Asking an unsupported provider or +API to run `web_search` raises `ConfigurationError`. A local `ToolSpec` named `web_search` +also raises `ConfigurationError` when combined with hosted `web_search`; give the local tool +a different name. Hosted searches do not pass through the local tool registry, so +`Deps.before_tool` does not see individual searches. + +With `OPENAI_API=completions`, `Agent` rejects hosted search at construction with +`ConfigurationError`. Calling the Chat Completions client directly with hosted tools in a +`ModelRequest` raises `BadRequestError`. + +| Option | OpenAI Responses | Anthropic Messages | |---|---|---| -| What it is | Every page the provider's search returned (`Source(url, title)`), deduplicated | A claim in the model's text tied to a page (`Citation`) | -| Comes from | `HostedToolPart.sources` | `TextPart.citations` | -| Anthropic, basic search (default) | Filled | Filled, with `cited_text` | -| Anthropic, `dynamic_filtering=True` | Filled | **Always empty** | -| OpenAI Responses | Filled | Filled, with `start`/`end` (a span of the answer), no `cited_text` | -| Use it for | "What did it look at?" A reading list, an audit trail | "Where did this sentence come from?" Footnotes, fact-checking | - -A source is not evidence that the answer used the page; a citation is. Neither is readable page -text: Anthropic encrypts result content, and OpenAI returns only URLs. - -## `dynamic_filtering` (Anthropic only, default `False`) - -`True` sends `web_search_20260209`. Claude writes and runs a Python program in a code-execution -sandbox that calls the search, filters the results and prints what it keeps; Claude reads only -that output. `False` sends `web_search_20250305` and results go straight into the model's -context. - -- **Citations disappear** because a citation must point at a search-result block in the model's - context, and with filtering the results go to the program. Measured live in September 2026 - (`claude-opus-5`, 2 questions x 3 variants): filtering on gave 0 citations every time, basic - search 2-3, and `web_search_20260209` with `allowed_callers: ["direct"]` 2-5. The cause is the - code path, not the tool version. Frozen captures are in - `agentloop-tools/experiments/examples/citations/` (`shannon_A_dynamic.json` beside - `shannon_B_basic.json`); `experiments/` is gitignored, so rerun - `experiments/citations_experiment.py` on a fresh clone (6 requests, a few cents). -- **Cost:** it did not save tokens on simple searches (13,300 vs 12,827 input tokens). Use it for - search-heavy requests that do not need citations. -- **Constraints:** Claude 4.6+ only; not ZDR-eligible; the response carries extra - `server_tool_use` / `code_execution_tool_result` blocks, surfaced as a `HostedToolPart` of - kind `code_execution`. -- **OpenAI** has no equivalent: `dynamic_filtering=True` logs a warning and is ignored. - -## Domain policy (`agentloop_tools.web.DomainPolicy`) - -It prevents and reports, and does nothing else. - -- **Modes:** `DomainPolicy.block([...])` or `DomainPolicy.allow([...])`, not both. -- **Matching:** a rule covers its host and every subdomain, not a longer label (`europa.eu` - covers `eur-lex.europa.eu`, not `noteuropa.eu`). A rule may add a path prefix where `*` - matches within the path. -- **Prevent:** `policy.native_web_search(**options)` sends the list to the provider; passing a - domain option there as well is an error. -- **Report:** `policy.audit(transcript, since=n)` returns `Violation(url, where, message_index, - rule)`, with `where` of `"source"`, `"citation"` or `"text"`. To audit one turn, pass the - `len(transcript.messages)` taken before `converse`. -- **Text scanning:** full URLs in both modes, balanced parentheses kept; bare host mentions in - block mode only. -- **The hard limit:** with native search no library can stop the model reading a page. The - policy prevents what the provider can and reports the rest. Only client-side search filters - before the model sees anything. -- **Live evidence (September 2026):** `block(["wikipedia.org"])` held on both providers (0 leaked - sources out of 10 and 31). Asked for Wikipedia's URL, Claude wrote 4 Wikipedia links from - memory, which the audit reported as `"text"` violations. `allow(["python.org"])` on OpenAI - returned 27 sources, all on the list. +| `allowed_domains` or `blocked_domains` | Supported | Supported | +| `user_location` | Supported | Supported | +| `search_context_size` | Supported | Logged and ignored | +| `max_uses` | Logged and ignored | Supported | +| `dynamic_filtering` | Logged and ignored | Supported | + +Pass either `allowed_domains` or `blocked_domains`; passing both raises +`ConfigurationError`. Unknown options also raise `ConfigurationError`. Anthropic's +`dynamic_filtering=True` requires Claude 4.6+; it changes how search results reach the +model and may leave no citations even when sources are present. + +`agentloop` exposes `HostedToolPart`, `Source`, and `Citation`. The companion package's +`native_web_search()` returns a core `HostedTool`; its `sources()`, `citations()`, and +`DomainPolicy` helpers work on core transcripts. It does not run a second search engine. + +See the [`HostedTool` reference](../api/provider.md#hostedtool) and +[`transcript` reference](../api/transcript.md#hostedtoolpart). diff --git a/docs/guides/installing.md b/docs/guides/installing.md index 2dfdd68..b941390 100644 --- a/docs/guides/installing.md +++ b/docs/guides/installing.md @@ -1,32 +1,57 @@ # Installing +`based-models-agentloop` requires Python 3.12 or newer. Install the distribution named +`based-models-agentloop` and import it as `agentloop`. + +For OpenAI, install the `openai-compat` extra: + +```bash +python -m pip install 'based-models-agentloop[openai-compat]' +``` + +Check the import in the same Python environment: + ```bash -pip install 'based-models-agentloop[anthropic]' +python -c 'import agentloop; print("agentloop ready")' ``` -The distribution name is `based-models-agentloop`; the import name is `agentloop`. (The -bare name `agentloop` on PyPI is an unrelated project. Do not install it by mistake.) +Output: + +```text +agentloop ready +``` ## Extras -`anthropic`, `openai-compat` (OpenAI, OpenRouter, Modal), `otel` (OpenTelemetry tracing and -metrics), and `all`. These dependencies are optional. If one is missing, the error names the -extra to install. +| Extra | Adds | +|---|---| +| `anthropic` | Anthropic SDK | +| `openai-compat` | HTTP client for OpenAI, OpenRouter, and Modal/vLLM | +| `otel` | OpenTelemetry SDK and OTLP HTTP exporter | +| `all` | All three integrations above | + +The core package does not require these integrations. If you use a provider or telemetry +feature without its dependency, the error names the extra to install. See the +[getting-started installation guide](../getting-started/installation.md) for more install +commands and local development setup. ## Pinning -Pin a compatible range in your own project. While the version is `0.x`, a minor release can -change the public API: +Pin a compatible range in your project. While the version is `0.x`, a minor release can +change the public API. For example, to stay on the `0.5` series with OpenAI support, use +this requirement in your project's dependency file: ```text -based-models-agentloop[anthropic]>=0.4,<0.5 +based-models-agentloop[openai-compat]>=0.5,<0.6 ``` -For reproducible and tamper-evident installs, compile a hashed lockfile -(`pip-compile --generate-hashes`) and install with `pip install --require-hashes`. +For repeatable deployments, lock the selected version and its dependencies. If your project +uses `pip-tools`, `pip-compile --generate-hashes` can produce a hashed requirements file for +`pip install --require-hashes`. ## Verifying a release -Each PyPI release is published by GitHub Actions through Trusted Publishing and carries -build attestations. The same wheel, sdist and `SHA256SUMS` are attached to the matching -GitHub release. +The [PyPI files page](https://pypi.org/project/based-models-agentloop/#files) shows +the wheel, source distribution, checksums, and provenance for the current version. The +matching [GitHub release](https://github.com/based-models/agent-loop/releases/latest) +carries the wheel, source distribution, and `SHA256SUMS`. diff --git a/docs/guides/observers.md b/docs/guides/observers.md index 276af27..9b3d1b6 100644 --- a/docs/guides/observers.md +++ b/docs/guides/observers.md @@ -14,38 +14,92 @@ Subclass `NullObserver` so callbacks you do not implement safely do nothing. | Counters and distributions for dashboards | [OpenTelemetry metrics](tracing.md#metrics) | | Application-specific reporting | A custom observer | +## Write a callback + +`NullObserver` implements every callback as a no-op. Subclass it and override only the events +you need. Here, `on_response()` receives a `ModelResponse` after each successful model call. +With OpenAI configured as in [Hello World](../getting-started/hello-world.md): + ```python -from agentloop import NullObserver +import os + +from agentloop import Agent, NullObserver from agentloop.provider import ModelResponse + class UsageObserver(NullObserver): def __init__(self) -> None: + self.responses = 0 self.input_tokens = 0 self.output_tokens = 0 def on_response(self, response: ModelResponse) -> None: + self.responses += 1 self.input_tokens += response.usage.input_tokens self.output_tokens += response.usage.output_tokens + print("response.usage:", response.usage) + + +prompt = "How many days are in a week? Reply with one digit." +usage = UsageObserver() +with Agent.from_env(os.environ, tools=[], observer=usage) as agent: + answer = agent.run(prompt) + +print("input:", prompt) +print("output:", answer) +print("responses:", usage.responses) +print("input tokens:", usage.input_tokens) +print("output tokens:", usage.output_tokens) +``` + +Output: + +```text +response.usage: input_tokens=19 output_tokens=11 cache_read_tokens=0 cache_write_tokens=None reasoning_tokens=0 cost_usd=None input_includes_cached=True +input: How many days are in a week? Reply with one digit. +output: 7 +responses: 1 +input tokens: 19 +output tokens: 11 ``` -Combine several observers: +## Combine observers + +`ConsoleObserver` prints model text and tool activity. `ListObserver` keeps request, response, +retry, and tool events in memory. `MultiObserver` sends each callback to its children in +order, so one run can use both: ```python -from agentloop import ConsoleObserver, MultiObserver +import os -usage = UsageObserver() -observer = MultiObserver(ConsoleObserver(), usage) +from agentloop import Agent, ConsoleObserver, ListObserver, MultiObserver + +recorded = ListObserver() +observer = MultiObserver(ConsoleObserver(), recorded) +with Agent.from_env(os.environ, tools=[], observer=observer) as agent: + agent.run("Reply with exactly ready.") + +print("recorded events:", recorded.kinds()) +``` + +Output: + +```text +ready +recorded events: ['request', 'response'] ``` -`ConsoleObserver` prints text, tool activity, and retries. `ListObserver` records events for -tests and debugging. If a callback fails, the library logs the failure and continues the -conversation. An exception raised while starting an observer is returned to the caller -because agent work has not begun. +`ConsoleObserver` also prints retry errors when they occur. `ListObserver` is useful for +tests and debugging. If a callback raises, the library logs the failure and continues the +conversation. An exception raised while entering an observer's `conversation()` context +propagates before agent work begins. -Five callbacks are optional, so older observers continue to work: -`on_conversation_start`, `on_conversation_end`, `on_budget_warning`, `on_stall` and -`on_model_error`. They carry the run's identifiers, its `ConversationSummary`, the optional -warnings described in [Conversation summaries](summaries.md), and a failed call's error. To keep a JSONL record of everything, see the +Five callbacks are optional for older observer implementations: +`on_conversation_start(context)`, `on_conversation_end(summary)`, +`on_budget_warning(kind, used, limit)`, `on_stall(tool_name, count)`, and +`on_model_error(error)`. The context carries run identifiers; the summary describes how the +run ended. Budget and stall events require the options described in +[Conversation summaries](summaries.md). To keep a JSONL record of every event, see the [event trace](event-trace.md). See the [`observer` API](../api/observer.md). diff --git a/docs/guides/providers.md b/docs/guides/providers.md index 3d9f69e..d8bbba8 100644 --- a/docs/guides/providers.md +++ b/docs/guides/providers.md @@ -1,90 +1,109 @@ # Providers -The high-level path is `Settings` plus `build_client`, or `Agent.from_env`: +`Agent.from_env()` selects and builds a provider client from the mapping you give it. With +`LLM_PROVIDER=openai` and `OPENAI_API_KEY` configured as in +[Hello World](../getting-started/hello-world.md): ```python +import os + from agentloop import Agent -env = { - "LLM_PROVIDER": "openrouter", - "OPENROUTER_API_KEY": "supplied securely at runtime", - "OPENROUTER_MODEL": "openrouter/free", -} +with Agent.from_env(os.environ, tools=[]) as agent: + print(agent.run("Say hello in one word.")) +``` + +Output: -with Agent.from_env(env, tools=[]) as agent: - print(agent.run("Hello")) +```text +Hello ``` -Every provider integration implements `LLMClient` and converts provider data into the same -request, response, content, usage, stop-reason, and error types. `build_client` also applies -the configured retry policy. +Each built-in provider implements the same `LLMClient` interface and converts requests, +responses, usage, stop reasons, and errors into shared types. `Agent.from_env()` uses +`Settings.from_env()` and `build_client()` internally; `build_client()` also applies the +configured retry policy. See [configuration](../getting-started/configuration.md) if you +need to validate settings separately. -Concrete constructors are available for applications that need provider-specific options: +## Provider-specific clients + +Use a provider builder when you need options that `Settings` does not expose. For example, +the OpenAI builder accepts an organization or project header and lets you choose an API: ```python -from agentloop.providers.openrouter import build_openrouter_client - -with build_openrouter_client( - api_key="supplied securely at runtime", - app_url="https://example.com", - app_name="Example Agent", -) as client: - print(client.name, client.model) +import os + +from agentloop.providers.openai import build_openai_client + +with build_openai_client(api_key=os.environ["OPENAI_API_KEY"]) as client: + print(client.name, client.model, client.api) ``` -## Corporate networks: trust, proxy and transport +Output: + +```text +openai gpt-5.1 responses +``` + +The builder returns a direct provider client. Wrap it in `RetryingClient` if you need +retries; `build_client()` and `Agent.from_env()` do that for you. + +## HTTP client, trust, and proxy settings -The library never reads certificates or builds a trust store. Pass an HTTP client you -configured, and it carries your settings for every request. It stays yours to close: closing -the agent or the provider client leaves it open. +Pass an HTTP client you configured when you need a custom trust store, proxy, transport, or +timeout. OpenAI, OpenRouter, and Modal use `httpx.Client`. Anthropic uses the HTTP client +type required by the installed Anthropic SDK (`httpx2.Client` in the version used here). -OpenAI, OpenRouter and Modal take an `httpx.Client`. Anthropic takes an `httpx2.Client`, -because the Anthropic SDK is built on httpx2, whose classes are not httpx's. Both packages -take the same `verify`, `proxy` and `transport` arguments. Passing the wrong one raises -`ConfigurationError` naming the package the provider needs. +The supplied HTTP client stays yours to close. This example gives OpenAI a 30-second HTTP +timeout and checks that leaving the agent's context did not close the shared client: ```python -import ssl +import os import httpx -import truststore from agentloop import Agent -context = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) # the operating system's trust store -http = httpx.Client(verify=context, proxy="http://proxy.corp.example:8080") - -with Agent.from_env(env, tools=[], http_client=http) as agent: - print(agent.run("Hello")) -http.close() +with httpx.Client(timeout=30.0) as http: + with Agent.from_env(os.environ, tools=[], http_client=http) as agent: + print(agent.run("Reply with the word ready.")) + print(f"HTTP client closed by agent: {http.is_closed}") ``` -For `LLM_PROVIDER=anthropic`, build the same client with `httpx2.Client(...)` instead. +Output: -The same `http_client=` is accepted by `build_client`, the provider builders and -`AnthropicClient`. Without it, the HTTP package's environment variables (`HTTPS_PROXY`, -`SSL_CERT_FILE`) still apply. The adapter sends its URL and credentials per request, so your client needs no -`base_url` or default headers; its own timeout wins if you set one. +```text +ready +HTTP client closed by agent: False +``` -Redirects are not followed on the clients the library builds, because that would resend your -credentials to another host: a 3xx raises `UnexpectedRedirect`. The Anthropic SDK follows -redirects by default; pass a client with `follow_redirects=False` if you want one reported. -A TLS, DNS or proxy failure raises `ProviderUnreachable` immediately, without retries. +For a corporate trust store or proxy, configure `httpx.Client(verify=..., proxy=...)` in +the same place. The adapter supplies its own URL and credentials for each request; your +client needs no `base_url` or default authorization header. If `LLM_TIMEOUT_SECONDS` is set, +that explicit per-call timeout overrides the HTTP client's timeout. -See the [provider reference](../api/providers/index.md) or -[implement a custom provider](custom-providers.md). +The same `http_client=` parameter is available on `build_client()` and provider builders. +Passing a client of the wrong HTTP package raises `ConfigurationError`. A TLS, DNS, or proxy +failure raises non-retryable `ProviderUnreachable`; a transient connection error may be +retried. OpenAI-compatible clients created by the library do not follow redirects and +report a 3xx as `UnexpectedRedirect`. Anthropic's SDK follows redirects by default; supply +a client with `follow_redirects=False` if you want it to report them instead. ## Chat Completions or Responses -OpenAI can be reached over two APIs, chosen with `OPENAI_API` (`responses`, the default, or -`completions`; `Settings.openai_api`). +OpenAI uses the Responses API by default. Set `OPENAI_API=completions` (or +`Settings.openai_api="completions"`) only when you need Chat Completions compatibility. +OpenRouter and Modal always use Chat Completions. | | Chat Completions | Responses | |---|---|---| -| Spoken by | OpenAI, OpenRouter, Modal/vLLM | OpenAI | -| Hosted tools (web search) | Not supported | Supported | +| Providers | OpenAI, OpenRouter, Modal/vLLM | OpenAI | +| Hosted web search | Not supported | Supported | | Reasoning state | Provider-specific (OpenRouter `reasoning_details`) | Reasoning items replayed | -Responses runs **stateless**: every request sends `store: false` and the full transcript, and -item ids are stripped on replay, so nothing depends on the provider remembering a conversation. -OpenRouter and Modal always use Chat Completions. +Responses requests send `store: false` and the full transcript. The adapter strips output +item IDs on replay, so a continued conversation does not depend on the provider retaining +server-side state. + +See the [provider reference](../api/providers/index.md) or +[implement a custom provider](custom-providers.md). diff --git a/docs/guides/releasing.md b/docs/guides/releasing.md deleted file mode 100644 index 9779d7e..0000000 --- a/docs/guides/releasing.md +++ /dev/null @@ -1,80 +0,0 @@ -# Releasing - -Releases are cut from `main` by pushing a `vX.Y.Z` tag. The pipeline publishes to PyPI with -Trusted Publishing (OIDC); there are no API tokens to store or rotate. There is no approval -gate (GitHub Free private repos cannot require reviewers), so **pushing the tag publishes**: -run the local checklist below first, and treat tag creation as the release decision. - -## Checklist - -1. Move the `[Unreleased]` entries in `CHANGELOG.md` under a new `## [X.Y.Z]` heading. -2. Set the version in **both** `pyproject.toml` and `src/agentloop/__init__.py`. -3. Locally, from a fresh clone (your working tree hides missing files): - - ```bash - git clone . /tmp/agentloop-release && cd /tmp/agentloop-release - pip install -e '.[all,dev]' - ruff check . && mypy && pytest -q --cov --cov-fail-under=88 - bash scripts/test-wheel.sh - ``` - -4. Tag and push: - - ```bash - git tag vX.Y.Z && git push origin vX.Y.Z - ``` - -The `Release` workflow then: - -1. **build**: refuses to continue unless the tag, `pyproject.toml` and `__version__` agree - and the changelog has an entry; runs ruff, mypy, pytest and `scripts/test-wheel.sh`; - builds the wheel and sdist once and uploads them as an artifact. -2. **publish-pypi**: uploads those exact files to PyPI via Trusted Publishing, with - attestations. -3. **github-release**: attaches the same files plus `SHA256SUMS` to the GitHub release. - -## One-time setup - -- PyPI (and TestPyPI) account with 2FA. -- A **pending publisher** for the project on each: owner `based-models`, repository - `agent-loop`, workflow `release.yml`, environment `pypi` (TestPyPI: `testpypi`). The first - successful publish creates the project, so the name is reserved only at that moment: - register the pending publisher promptly. -- GitHub environments named `pypi` and `testpypi`. They must exist with exactly those names - because the PyPI trusted publisher matches on the environment name. (Required reviewers - are unavailable on private Free-plan repos, so they are not used.) - -## Dry run - -Actions -> Release -> **Run workflow** (on any branch) builds, tests and publishes to -TestPyPI only. Run it from the version-bump branch: TestPyPI refuses to re-upload a version, -and the previous version is usually already there, so a run from `main` fails until the -version is bumped. Check `https://test.pypi.org/pypi/based-models-agentloop/X.Y.Z/json` is a -404 first; to retry after an upload, bump the version. Verify in a clean venv: - -```bash -pip install --index-url https://test.pypi.org/simple/ \ - --extra-index-url https://pypi.org/simple/ 'based-models-agentloop[anthropic]==X.Y.Z' -``` - -Use `--extra-index-url` only for this check: TestPyPI does not carry pydantic and friends. - -## What `scripts/test-wheel.sh` proves - -It builds the wheel, installs it into a bare virtualenv with a single extra, and checks -what only an installed wheel can show: optional extras really are optional (`httpx`, the -OTel SDK are absent), the library cannot read the environment (`pydantic_settings` absent), -missing extras fail with a message naming the fix, `import agentloop` stays cheap (no -OpenTelemetry, no vendor SDK), every provider module imports under `[all]`, `py.typed` -ships, and no tests/docs/examples leak into the wheel or sdist. - -## Rolling back - -PyPI files are immutable and a version number can never be reused, even after deletion. -If a release is bad, **yank** it on PyPI (installs of a pinned version still work, fresh -resolves skip it), fix forward, and release `X.Y.(Z+1)`. Never move a published tag. - -## Versioning - -SemVer. While `0.x`, a minor bump may break `agentloop.__all__`. The layering tests -(`tests/test_layering.py`) are the guardrail on what the library may import. diff --git a/docs/guides/retries-and-errors.md b/docs/guides/retries-and-errors.md index cee0d47..c4752db 100644 --- a/docs/guides/retries-and-errors.md +++ b/docs/guides/retries-and-errors.md @@ -1,62 +1,119 @@ # Retries and errors -Every provider integration raises the same set of error types. Catch a specific error when your -application can recover, or catch `AgentLoopError` at its outer boundary. +When a model call fails, its provider adapter raises a `ProviderError`. The error type tells +the retry wrapper whether another attempt might help. If a later attempt succeeds, the agent +returns an answer and records the retry in its summary. If every attempt fails, the final +error reaches your application. -```python -from agentloop.errors import AgentLoopError, AuthError, RateLimited - -try: - answer = agent.run("Hello") -except AuthError: - print("Check provider credentials") -except RateLimited as error: - print("Provider remained rate limited after retries", error.retry_after) -except AgentLoopError as error: - print(f"Agent failed: {type(error).__name__}: {error}") -``` +`Agent.from_env()` installs a `RetryingClient` automatically. If you build a provider client +yourself, you can wrap it with `RetryingClient` and choose a `RetryPolicy`. -`RateLimited`, `ProviderUnavailable`, `ProviderTimeout`, and `ResponseInvalid` are retryable. -Authentication, bad-request, context-length, missing-model, and credit errors are not. That -includes an OpenAI 429 whose `type`/`code` is `insufficient_quota` or -`credit_balance_exhausted`: it raises `InsufficientCredits` and is not retried. +## Which failures get another attempt? -`ProviderUnreachable` (TLS verification, DNS, a refusing proxy) and `UnexpectedRedirect` are -not retried, because retrying cannot fix them. Both are faults in the environment. A refused -or reset connection and a read error are retried. +- **Temporary problems are retried:** `RateLimited`, `ProviderUnavailable`, + `ProviderTimeout`, and `ResponseInvalid`. These cover rate limits, temporary server or + connection failures, timeouts, and responses the adapter could not normalize. +- **Problems that need a change are not retried:** `AuthError`, `BadRequestError`, + `ContextLengthExceeded`, `ModelNotFound`, and `InsufficientCredits`. Retrying the same + credentials or request will not fix them. +- **Environment failures are not retried:** `ProviderUnreachable` covers DNS, TLS + verification, and proxy failures. `UnexpectedRedirect` covers an unexpected HTTP redirect. -## Was the request sent? +`ProviderUnreachable` is a non-retryable subclass of `ProviderUnavailable`. A refused or +reset connection can still be a temporary `ProviderUnavailable` and be retried. OpenAI 429 +responses with `insufficient_quota` or `credit_balance_exhausted` are classified as +`InsufficientCredits`, so they are not retried. -For spend and safety, ask the error rather than inspecting its cause: +## How many times will it try? -```python -from agentloop.errors import ProviderError - -try: - agent.run("Hello") -except ProviderError as error: - if error.request_sent is False: - print("nothing left this machine; nothing was billed") - elif error.request_sent is None: - print("unknown; assume it may have been billed") - print(error.phase, [r.phase for r in error.attempt_log]) -``` +`RetryPolicy.max_attempts` counts the first call. The default is three attempts. Between +attempts, the wrapper uses exponential backoff with jitter, capped by `max_delay`. If the +provider supplies `Retry-After`, that wait is also capped by `max_delay`. -`ConversationSummary.error_attempts_sent` gives the same answer across every attempt: `0` -means no attempt reached the provider. +## A retry, then an answer -Wrap a custom client with an explicit policy: +To make a retry visible, `FailOnce` wraps a client that implements `LLMClient`. It can wrap +any such client; this example passes it an OpenAI client. Its first call raises a retryable +connection error before sending anything, and its second call delegates to the wrapped +client. The `RetryPolicy` allows two attempts and keeps the wait short. Configure OpenAI as +in [Hello World](../getting-started/hello-world.md): ```python +import os + +from agentloop import Agent +from agentloop.errors import FailurePhase, ProviderUnavailable +from agentloop.providers.openai import build_openai_client from agentloop.retry import RetryingClient, RetryPolicy -client = RetryingClient( - raw_client, - policy=RetryPolicy(max_attempts=4, base_delay=0.5, max_delay=10.0), -) + +class FailOnce: + def __init__(self, inner): + self.inner = inner + self.name = inner.name + self.model = inner.model + self.calls = 0 + + def complete(self, request): + self.calls += 1 + if self.calls == 1: + raise ProviderUnavailable( + "temporary connection failure", + phase=FailurePhase.CONNECT, + request_sent=False, + ) + return self.inner.complete(request) + + +with build_openai_client(api_key=os.environ["OPENAI_API_KEY"]) as openai: + inner = FailOnce(openai) + client = RetryingClient( + inner, + policy=RetryPolicy(max_attempts=2, base_delay=0.01, max_delay=0.02), + ) + result = Agent([], client=client).run_result("Reply with exactly ready.") + +print("answer:", result.text) +print("attempts:", inner.calls) +print("provider retries:", result.summary.provider_retries) +print("first request sent:", result.summary.failed_attempts[0].request_sent) ``` -`max_attempts` includes the first request. Server `Retry-After` values are honored but -capped by `max_delay`; otherwise exponential backoff with jitter is used. +Output: + +```text +temporary connection failure (attempt 1/2); retrying in 0.0s +answer: ready +attempts: 2 +provider retries: 1 +first request sent: False +``` + +The first call fails with `request_sent=False`. `RetryingClient` makes one more call, which +returns OpenAI's answer. `result.summary.provider_retries` counts the extra attempt, while +`failed_attempts` keeps metadata about the first one. + +## If all attempts fail + +`RetryingClient` raises the final `ProviderError`. It carries `attempts` and an `attempt_log` +with one record per attempt. An error raised from an agent run also carries a +`ConversationSummary` in `error.summary`, including the retry count. Catch a specific error, +such as `RateLimited`, when your application can handle it; catch `AgentLoopError` at an outer +boundary for any library error. See the [error reference](../api/errors.md) for the hierarchy. + +## Was the request sent? + +For a `ProviderError`, inspect `request_sent` and `phase` rather than parsing its message. +`phase` names the stage that failed, such as `connect`, `read`, or `status`. + +| `request_sent` | What you know | +|---|---| +| `False` | The request did not reach the provider. | +| `True` | It may have reached the provider. | +| `None` | The adapter could not determine whether it reached the provider. | + +The example's first attempt failed during `connect`, so `request_sent` is `False`. On a +failed run, `error.summary.error_attempts_sent` counts attempts that may have reached the +provider. A value of `0` means none did. See the [`retry` API](../api/retry.md) and [`errors` API](../api/errors.md). diff --git a/docs/guides/summaries.md b/docs/guides/summaries.md index a5ba9e7..f9d3bd4 100644 --- a/docs/guides/summaries.md +++ b/docs/guides/summaries.md @@ -6,30 +6,35 @@ contain messages, tool arguments, tool results, or exception text. This metadata is suitable for operational logs, metrics, and checkpoints, subject to your organization's rules for identifiers and tool names. +Start with `Agent.run_result()`. It returns a `ConversationResult` whose `.summary` is the +`ConversationSummary`. With OpenAI configured as in +[Hello World](../getting-started/hello-world.md): + ```python -result = agent.run_result("summarize the open support case", correlation_id="case-42") +import os + +from agentloop import Agent + +with Agent.from_env(os.environ, tools=[]) as agent: + result = agent.run_result("Reply with exactly ready.", correlation_id="example-1") + print(result.text) -print(result.summary.model_dump_json()) +print(result.summary.model_dump_json(include={ + "correlation_id", "provider", "model", "status", "iterations", + "tool_calls", "provider_retries", "usage", +})) ``` -```json -{ - "conversation_id": "9f1c...", - "correlation_id": "case-42", - "agent_name": "support-case-review", - "provider": "openrouter", - "model": "gpt-5.1", - "status": "iteration_limit", - "iterations": 30, - "iteration_limit": 30, - "elapsed_seconds": 412.8, - "tool_calls": {"search_knowledge_base": 4, "fetch_article": 8}, - "tool_errors": {"fetch_article": 2}, - "provider_retries": 1, - "usage": {"input_tokens": 12000, "output_tokens": 3500, "cost_usd": 0.08} -} +Output: + +```text +ready +{"correlation_id":"example-1","provider":"openai","model":"gpt-5.1-2025-11-13","status":"completed","iterations":1,"tool_calls":{},"provider_retries":0,"usage":{"input_tokens":11,"output_tokens":11,"cache_read_tokens":0,"cache_write_tokens":null,"reasoning_tokens":0,"cost_usd":null,"input_includes_cached":true}} ``` +`model_dump_json(include=...)` selects fields for display. The full summary also includes +the generated `conversation_id`, elapsed time, limits, and other counters. + ## Status Branch on `summary.status` (a `ConversationStatus`), never on an exception message. @@ -46,7 +51,10 @@ Branch on `summary.status` (a `ConversationStatus`), never on an exception messa | `error` | Anything else raised | raised | | `cancelled` | `KeyboardInterrupt` and similar | raised | -`status.succeeded` and `status.budget_exhausted` cover the common branches. +`succeeded` and `budget_exhausted` are Boolean properties of `ConversationStatus`, not +additional statuses. `status.succeeded` is true for `completed`, `refused`, and `truncated`; +`status.budget_exhausted` is true for `iteration_limit`, `deadline_exceeded`, and +`token_limit`. Both are false for the other statuses. `LoopBudgetExceeded` also has structured `.kind`, `.limit` and `.actual`. ## Get the summary @@ -61,12 +69,42 @@ Branch on `summary.status` (a `ConversationStatus`), never on an exception messa conversation, on success and failure. If the callback raises, the library logs that callback failure without hiding the conversation's result or error. +An observer receives a summary even when a run raises. `NullObserver` supplies no-op methods, +so a subclass can implement only the callback it needs. This `SummaryObserver` keeps the +latest summary: + ```python -class JsonlSink(NullObserver): +import os + +from agentloop import Agent, ConversationSummary, NullObserver + + +class SummaryObserver(NullObserver): + def __init__(self) -> None: + self.latest: ConversationSummary | None = None + def on_conversation_end(self, summary: ConversationSummary) -> None: - log.info(summary.model_dump_json()) + self.latest = summary + + +observer = SummaryObserver() +with Agent.from_env(os.environ, tools=[], observer=observer) as agent: + result = agent.run_result("Reply with exactly ready.") + +print("observer received summary:", observer.latest == result.summary) +print("status:", result.summary.status) +``` + +Output: + +```text +observer received summary: True +status: completed ``` +The callback runs before `run_result()` returns. You can log +`summary.model_dump_json()` there to record every run, including failures. + ## Grouping conversations Your application supplies `correlation_id` and `parent_id`. For example, pass the same @@ -80,8 +118,8 @@ unless you supply one. Your application decides how to group, store, and aggrega truncated turn and each paused continuation. - `None` means unknown, never zero. `usage.cost_usd` is `None` unless a provider turn reported a cost; if only some turns did, it is the sum of those. -- `request_sent` and `error_request_sent` keep three states. `None` means the adapter could - not tell, which is not the same as `False`. +- `error_request_sent` and each `failed_attempts` entry's `request_sent` keep three states. + `None` means the adapter could not tell, which is not the same as `False`. - `provider_retries` counts attempts after the first, including attempts for a request that ultimately failed. It works with any `RetryingClient` and requires no observer setup. @@ -106,17 +144,33 @@ provider reported no cache tokens. It accounts for providers that count cached t ## Optional detail and warnings +`SummaryOptions` enables extra reporting. Pass it to `Agent` when you want a timeline, stall +detection, or budget warnings: + ```python +import os + from agentloop import Agent, SummaryOptions -agent = Agent( - tools, - client=client, +with Agent.from_env( + os.environ, + tools=[], summary_options=SummaryOptions(timeline=True, stall_threshold=3, warn_at=(0.8,)), -) +) as agent: + result = agent.run_result("Reply with exactly ready.") + +print("timeline entries:", len(result.summary.timeline or ())) +print("first stop reason:", result.summary.timeline[0].stop_reason) +``` + +Output: + +```text +timeline entries: 1 +first stop reason: end_turn ``` -These features are off by default. Configure them together with `SummaryOptions`. +These options are off by default. - **`stall_threshold`**: flag a run in which the same tool was called with identical arguments that many times (3 is a good start). `max_repeated_tool_calls` is reported either way; @@ -125,12 +179,12 @@ These features are off by default. Configure them together with `SummaryOptions` explaining why. Calls are compared by hash and only counts are kept, so neither the arguments nor the hash reach a summary, event or metric. - **`warn_at`**: fractions of the iteration limit and the deadline (for example `(0.5, 0.8)`) - at which `on_budget_warning(kind, used, limit)` fires, once each, so something can - intervene before the failure. + at which `on_budget_warning(kind, used, limit)` fires, once each. - **`timeline`**: `summary.timeline` becomes a tuple of `TurnRecord`s, one per model call: duration, stop reason, tokens, cost, attempts, the tools it led to and which of them errored, and whether a truncated turn was discarded. A call that raised appears with its - `error_type`, and with `None` tokens: it was never answered, so nothing was measured. The timeline also excludes conversation content. + `error_type` and `None` tokens because the call never answered. The timeline also excludes + conversation content. ## How reporting stays separate from execution diff --git a/docs/guides/tools.md b/docs/guides/tools.md index fb23a62..5beddde 100644 --- a/docs/guides/tools.md +++ b/docs/guides/tools.md @@ -1,10 +1,18 @@ # Tools and approval -A tool combines a handwritten JSON Schema with a Python handler. The schema is part of the -prompt contract and is never inferred from the function signature. +A local tool combines a handwritten JSON Schema with a Python handler. The schema is what +the model sees; `@tool` does not infer it from the function signature. These examples use +OpenAI, configured with `LLM_PROVIDER=openai` and `OPENAI_API_KEY` as in +[Hello World](../getting-started/hello-world.md). + +## Run a local tool + +Define the schema, decorate the function, and pass the resulting tool to `Agent`: ```python -from agentloop import ToolError, tool +import os + +from agentloop import Agent, tool ADD_SCHEMA = { "type": "object", @@ -19,39 +27,95 @@ ADD_SCHEMA = { @tool(parameters=ADD_SCHEMA) def add(left: int, right: int) -> str: """Add two integers.""" - if abs(left) > 1_000_000 or abs(right) > 1_000_000: - raise ToolError("numbers must be at most one million") return str(left + right) + +with Agent.from_env( + os.environ, + tools=[add], + system="Use the add tool for arithmetic before answering.", +) as agent: + result = agent.run_result("What is 27 plus 15? Call add to calculate it.") + print(result.text) + print(result.summary.tool_calls) ``` -Pass `[add]` to `Agent`. `ToolError` is safe feedback for the model. Unexpected exceptions -are logged and converted to an error result so every call still receives a result. +Output: + +```text +27 plus 15 is 42. +{'add': 1} +``` + +`result.summary.tool_calls` counts calls by tool name; the full call and result are in +`result.transcript`. ## Dependencies and approval -Declare a keyword-only `deps` parameter when a tool needs application state: +Declare a keyword-only `deps` parameter when a tool needs trusted application state. A +`before_tool` hook can deny a call before its handler runs: ```python -from agentloop import Deps, ToolCallPart, tool +import os + +from agentloop import Agent, Deps, tool +from agentloop.transcript import ToolCallPart -class AppDeps(Deps): - database_url: str +class NoteDeps(Deps): + notes: dict[str, str] def approve(call: ToolCallPart) -> str | None: - if call.tool_name == "delete_record": - return "destructive tools are disabled" + if call.arguments.get("name") == "private": + return "private notes are unavailable" return None -@tool(parameters={"type": "object", "properties": {}, "additionalProperties": False}) -def database_name(*, deps: AppDeps) -> str: - """Return the configured database URL.""" - return deps.database_url +@tool(parameters={ + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + "additionalProperties": False, +}) +def read_note(name: str, *, deps: NoteDeps) -> str: + """Read a note by name.""" + return deps.notes.get(name, "Note not found") -deps = AppDeps(database_url="sqlite:///app.db", before_tool=approve) +deps = NoteDeps( + notes={"public": "Blue is a color.", "private": "Hidden text."}, + before_tool=approve, +) +with Agent.from_env( + os.environ, + tools=[read_note], + deps=deps, + system=( + "Call read_note once for each request. Use the name between backticks exactly " + "as written. If denied, explain the denial." + ), +) as agent: + public = agent.run_result("Read the note named `public`.") + print(public.text) + print(public.summary.tool_calls, public.summary.tool_errors) + + private = agent.run_result("Read the note named `private`.") + print(private.text) + print(private.summary.tool_calls, private.summary.tool_errors) ``` -Multiple tool calls in one response execute concurrently, up to -`MAX_PARALLEL_TOOLS`, while results retain call order. Approval hooks may therefore run on -worker threads and should be thread-safe. +Output: + +```text +Blue is a color. +{'read_note': 1} {} +I’m not allowed to access notes marked as private, so I can’t read or display the contents of the note named `private`. +{'read_note': 1} {'read_note': 1} +``` + +The `notes` dictionary belongs to the application; the model sees only the `name` schema. +For the private request, the approval hook returns a reason and `read_note` does not run. +The denied call still appears in `tool_calls` and `tool_errors`. + +Independent tool calls in one model response run concurrently, up to eight at a time, and +their results retain the model's call order. Set `parallel_safe=False` on `@tool` if a +handler cannot run alongside others. Approval hooks may run on worker threads, so shared +state they access must be thread-safe. See the [`tooling` API](../api/tooling.md). diff --git a/docs/guides/tracing.md b/docs/guides/tracing.md index 8ddfafe..078c715 100644 --- a/docs/guides/tracing.md +++ b/docs/guides/tracing.md @@ -1,47 +1,64 @@ # OpenTelemetry tracing and metrics -Install the tracing extra, configure an exporter, and enable tracing on the agent: +`agentloop` supports Anthropic, OpenAI, OpenRouter, and Modal/vLLM. This example uses OpenAI. + +Install OpenAI support and the OpenTelemetry extra: ```bash -pip install 'based-models-agentloop[otel]' +python -m pip install 'based-models-agentloop[openai-compat,otel]' ``` +`trace=True` adds an `OTelObserver` to the agent. Install a tracer provider before running +it. This example uses an in-memory exporter so you can inspect the spans without a collector. +Configure OpenAI as in [Hello World](../getting-started/hello-world.md): + ```python import os +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from agentloop import Agent -from agentloop.tracing import configure_tracing -provider = configure_tracing( - "http://localhost:4318", - service_name="hello-agent", -) +exporter = InMemorySpanExporter() +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(exporter)) +trace.set_tracer_provider(provider) + try: - with Agent.from_env( - os.environ, - tools=[], - name="hello-agent", - trace=True, - ) as agent: - print(agent.run("Hello")) - provider.force_flush() + with Agent.from_env(os.environ, tools=[], name="demo", trace=True) as agent: + print("answer:", agent.run("Reply with exactly ready.")) + spans = exporter.get_finished_spans() + print("span operations:", [s.attributes["gen_ai.operation.name"] for s in spans]) finally: provider.shutdown() ``` +Output: + +```text +answer: ready +span operations: ['chat', 'invoke_agent'] +``` + Tracing creates one top-level span for the conversation, one child span for each model request, and one span for each local tool call. A span is one timed operation in an OpenTelemetry trace. Retries appear as events on the related model-request span. -Tracing needs a provider installed first. A traced conversation with none logs one warning -and discards its spans; calling `configure_tracing()` a second time logs a warning too, because -OpenTelemetry keeps the first provider and the returned one receives nothing. +For an OTLP collector, `configure_tracing(endpoint, service_name=...)` installs a provider +that exports spans to the endpoint's `/v1/traces` path. Call it once per process, then flush +and shut down the returned provider when your application exits. A traced conversation with +no provider logs one warning and discards its spans. Calling `configure_tracing()` after a +provider is already installed logs a warning because OpenTelemetry keeps the first one. Prompts, tool arguments, tool results, and provider error messages are not captured by -default. Construct an -`OTelObserver(..., capture_content=True, redact=scrub)` (a redactor is required, as for the event -trace) only when your data-handling policy permits that -content to leave the process. One `OTelObserver` can trace only one conversation at a time. +default. To capture them, construct an `OTelObserver(..., capture_content=True, +redact=scrub)`. A redactor is a function that replaces sensitive text before it is exported; +see [capturing content in the event trace guide](event-trace.md#capturing-content) for a +simple example. A redactor is required unless you explicitly pass +`allow_unredacted=True`. One `OTelObserver` can trace only one conversation at a time. The conversation span records its status, identifiers, iterations, limits, timing, retries, usage, and stall flag. A failed call also gets `error.type`, `agentloop.error.phase` and `agentloop.error.request_sent` @@ -51,34 +68,46 @@ same whether content capture is on or off. ## Metrics -For dashboards and alerts, `agentloop.metrics.MetricsObserver`—or the shorter -`Agent(metrics=True)` option—records counters and distributions through the OpenTelemetry -metrics API. Like tracing, it records nothing until your application configures an +For dashboards and alerts, `agentloop.metrics.MetricsObserver` records counters and +distributions through the OpenTelemetry metrics API. `Agent(metrics=True)` provides a shorter +way to enable it. Like tracing, it records nothing until your application configures an OpenTelemetry `MeterProvider`. ```python import os from opentelemetry import metrics -from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.export import InMemoryMetricReader from agentloop import Agent -exporter = OTLPMetricExporter(endpoint="http://localhost:4318/v1/metrics") -reader = PeriodicExportingMetricReader(exporter) +reader = InMemoryMetricReader() meter_provider = MeterProvider(metric_readers=[reader]) metrics.set_meter_provider(meter_provider) try: with Agent.from_env(os.environ, tools=[], metrics=True) as agent: - agent.run("Hello") - meter_provider.force_flush() + print("answer:", agent.run("Reply with exactly ready.")) + data = reader.get_metrics_data() + names = sorted({ + metric.name + for resource in data.resource_metrics + for scope in resource.scope_metrics + for metric in scope.metrics + }) + print("metric names:", names) finally: meter_provider.shutdown() ``` +Output: + +```text +answer: ready +metric names: ['agentloop.conversation.duration', 'agentloop.conversation.iterations', 'agentloop.conversation.model_duration', 'agentloop.conversations', 'agentloop.tokens'] +``` + | Instrument | Kind | Attributes | |---|---|---| | `agentloop.conversations` | counter | status, agent_name, provider, model | diff --git a/docs/guides/transcript-storage.md b/docs/guides/transcript-storage.md index 8d55364..5cf3648 100644 --- a/docs/guides/transcript-storage.md +++ b/docs/guides/transcript-storage.md @@ -1,35 +1,50 @@ # Store transcripts -Serialization is deliberately separate from filesystem policy. `to_jsonl()` returns text, -and `from_jsonl()` accepts any iterable of lines. +Use `to_jsonl()` to serialize a transcript and `from_jsonl()` to load it. These functions +work with text and lines; your application chooses where to store them. Each message +occupies one JSONL line. ```python from pathlib import Path +from tempfile import TemporaryDirectory -from agentloop.transcript import from_jsonl, to_jsonl +from agentloop.transcript import TextPart, Transcript, from_jsonl, to_jsonl -path = Path("conversation.jsonl") -path.write_text(to_jsonl(transcript.messages), encoding="utf-8") +transcript = Transcript() +transcript.add_user_message("Hello") +transcript.add_assistant_message((TextPart(text="Hi!"),)) + +with TemporaryDirectory() as directory: + path = Path(directory) / "conversation.jsonl" + path.write_text(to_jsonl(transcript.messages), encoding="utf-8") + loaded = from_jsonl(path.read_text(encoding="utf-8").splitlines()) + print("JSONL:") + print(path.read_text(encoding="utf-8"), end="") -loaded = from_jsonl(path.read_text(encoding="utf-8").splitlines()) loaded.check_invariants() +print("Same transcript:", loaded == transcript) ``` -The serializer writes one `Message` per line and includes a trailing newline. Blank input -lines are ignored. Invalid or truncated JSON raises instead of silently discarding data. +Output: + +```text +JSONL: +{"role":"user","content":[{"type":"text","text":"Hello","citations":[]}],"origin":null} +{"role":"assistant","content":[{"type":"text","text":"Hi!","citations":[]}],"origin":null} +Same transcript: True +``` -`from_jsonl()` does not check conversation invariants because a legitimate capture may end -mid-turn. Call `check_invariants()` before sending a loaded transcript to a provider. +`to_jsonl()` includes a trailing newline. `from_jsonl()` skips blank lines but raises on +invalid or truncated JSON. It does not check conversation invariants, since a transcript +captured mid-turn may be valid to store but incomplete. Call `check_invariants()` before +sending a loaded transcript to a provider. See the [`transcript` API](../api/transcript.md). -## `Message.origin` - -An assistant `Message` may carry an `Origin`: the provider's own wire form of that turn -(`provider`, `api`, `model`, `raw`). Adapters replay it instead of rebuilding the turn from the -canonical parts, which keeps things the canonical form cannot express, such as Anthropic's -encrypted search results and signed thinking, or OpenRouter's `reasoning_details`. +## Provider origin data -It can be large. JSONL keeps it so a resumed conversation replays exactly. Drop it -(`message.model_copy(update={"origin": None})`) if you accept a replay rebuilt from canonical -parts, or switch providers: an origin from a different provider or API is ignored on replay. +An assistant message may contain an `Origin`, which stores the provider, API, model, and +original response blocks. JSONL preserves this data so the same provider and API can replay +the turn in their original format, including details that the common message parts cannot +represent. Origin data can make the file large. When a different provider or API loads the +transcript, its adapter uses the common message parts instead. diff --git a/docs/index.md b/docs/index.md index a8c955f..2bc38e8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,14 +1,16 @@ # agentloop documentation -`agentloop` is a synchronous agent loop that works with several model providers. It supplies -shared conversation types, tool execution, retries, limits, and operational reporting while -leaving application policy in your code. +`agentloop` is a synchronous Python agent loop for Anthropic, OpenAI, OpenRouter, Modal/vLLM, +and custom clients. It supplies shared conversation types, tool execution, retries, limits, +and operational reporting. Your application chooses prompts, tools, approvals, and secrets. ## Start here -- [Install agentloop](getting-started/installation.md) -- [Run Hello World with any provider](getting-started/hello-world.md) -- [Configure providers](getting-started/configuration.md) +1. [Install agentloop](getting-started/installation.md). +2. [Run Hello World](getting-started/hello-world.md) with the included `FakeClient` to see + the loop work offline, then use the same `Agent.run()` call with a live provider. +3. [Configure a provider](getting-started/configuration.md) with an explicit mapping or + `Settings` object. ## Guides @@ -23,7 +25,6 @@ leaving application policy in your code. - [Observers](guides/observers.md) - [OpenTelemetry tracing and metrics](guides/tracing.md) - [Installing](guides/installing.md) -- [Releasing](guides/releasing.md) - [Retries and errors](guides/retries-and-errors.md) - [Store transcripts](guides/transcript-storage.md) @@ -32,5 +33,5 @@ leaving application policy in your code. - [API reference](api/index.md) - [Example index](examples/index.md) -The API reference covers supported names that do not begin with `_`. Names beginning with -`_` are internal and can change without notice. +The API reference describes the main public interfaces. Names beginning with `_` are +internal and can change without notice. diff --git a/src/agentloop/providers/_openai_compat.py b/src/agentloop/providers/_openai_compat.py index 6a645a4..bb2b022 100644 --- a/src/agentloop/providers/_openai_compat.py +++ b/src/agentloop/providers/_openai_compat.py @@ -230,7 +230,7 @@ def request_to_wire( payload["temperature"] = request.temperature if request.stop_sequences: payload["stop"] = list(request.stop_sequences)[:MAX_STOP_SEQUENCES] - if quirks.supports_parallel_tool_calls_param: + if request.tools and quirks.supports_parallel_tool_calls_param: payload["parallel_tool_calls"] = ( request.parallel_tool_calls and quirks.default_parallel_tool_calls ) diff --git a/tests/test_openai_compat.py b/tests/test_openai_compat.py index 1ad3690..9852f96 100644 --- a/tests/test_openai_compat.py +++ b/tests/test_openai_compat.py @@ -142,6 +142,14 @@ def test_openai_provider_requests_max_completion_tokens() -> None: assert client._quirks.max_tokens_field == "max_completion_tokens" +def test_parallel_tool_calls_is_sent_only_with_tools() -> None: + assert "parallel_tool_calls" not in wire() + + tool = ToolSchema(name="read", description="Read", parameters=EMPTY_SCHEMA) + assert wire(tools=(tool,))["parallel_tool_calls"] is True + assert wire(tools=(tool,), parallel_tool_calls=False)["parallel_tool_calls"] is False + + # --- parsing arguments (Part 4) ---