From 12403cd9867316669a8a24e9035247a89631b0f4 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 10:00:29 -0700 Subject: [PATCH 1/8] Let another plugin scan memories and add bootstrap sections and search annotations scanMemories() walks stored memory rows in createdAt, id order for trusted in-process builders, paged and resumable. registerBootstrapSection() adds a section to GET /Memory/bootstrap after pinned and needsReview, within the budget they left, as id-less memories entries clients frame as stored data. registerSearchAnnotator() attaches per-hit data to q searches (REST and MCP recall) without being able to change the hits. A throwing contributor is logged and left out. All three reach the running plugin from another app through the per-worker runtime registry. Refs #36. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: agent-memory-summary-extension-points --- CHANGELOG.md | 14 + CLAUDE.md | 3 +- DESIGN.md | 73 ++- README.md | 43 +- integrationTests/cross-app.test.ts | 42 ++ .../fixtures/memory-consumer/resources.js | 41 +- src/extensions.ts | 194 ++++++ src/index.ts | 25 + src/mcp.ts | 5 +- src/resource.ts | 2 + src/runtime.ts | 19 + src/search.ts | 125 +++- test/cross-copy.test.js | 77 +++ test/extensions.test.js | 615 ++++++++++++++++++ 14 files changed, 1232 insertions(+), 46 deletions(-) create mode 100644 src/extensions.ts create mode 100644 test/extensions.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 190ffdb..93f5b9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ All notable changes to `@harperfast/agent-memory` are documented here. The forma ## [Unreleased] +### Added + +- `scanMemories({ after?, includeInactive? })` reads every memory as stored, embedding included, oldest `createdAt` + first, for a trusted plugin building derived data (a first build, or a rebuild after it was not loaded). It + resumes after a given row without skipping or repeating rows that share a timestamp. +- `registerBootstrapSection({ name, render })` adds a section to `GET /Memory/bootstrap`, after `pinned` and + `needsReview` and before `recent`, rendered for each caller within the budget those sections left. Its items come + back as `memories` entries with a `section` and no `id`, so existing clients show them as stored data without a + change. +- `registerSearchAnnotator({ name, annotate })` adds per-hit data under its name to search results with `q` + (`GET /Memory/?q=`, MCP `recall`). It cannot change which hits come back or their order. +- A section or annotator that throws is left out and logged, and the request still succeeds. All three reach the + running plugin from another app on the same Harper, like `registerMemorySink`. + ## [0.1.4] - 2026-11-11 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 903072b..33fc256 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,5 +5,6 @@ update it in the same PR when a rule changes. Follow the harper-engineering-guid Harper DLC for every PR. - Plugin shape mirrors `@harperfast/oauth`: `config.yaml` declares `pluginModule` + `graphqlSchema`. -- The `Memory` table is never `@export`ed; the `Memory` resource is the only reader/writer. +- The `Memory` table is never `@export`ed; the `Memory` resource is the only reader/writer, except `scanMemories`, + the unguarded read for trusted in-process plugins that no route reaches. - Scope enforcement is pushed into queries as conditions, never applied as a post-filter on top-k. diff --git a/DESIGN.md b/DESIGN.md index b778182..bb46ce0 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -38,6 +38,7 @@ allowlists, idempotent `PUT` create/revise with `If-Match` revision precondition | Customer-content flag and quarantine (`customerContentCheck` option) | implemented (`src/quarantine.ts`) | | Supersession backfill: candidates for an existing memory, links between existing memories, created-order cursor | implemented (`src/lifecycle.ts`) | | Package exports reaching the plugin from another app (Extension points, Across apps) | implemented (`src/runtime.ts`) | +| `scanMemories`, bootstrap sections and search annotators for derived plugins (Extension points) | implemented (`src/extensions.ts`) | ## Data model @@ -492,7 +493,8 @@ lease, since a batch writes up to 100 rows and flags their citers inside one tra `contentFlagged = true`, the review queue of a cleared reviewer (an uncleared caller sees only its own). Results include `supersededBy` when `includeSuperseded` is set, and `needsReview` under the Provenance visibility rule; withdrawn memories never appear (except to a `super_user` passing `includeWithdrawn=true`, below). The vector lane has no distance cutoff, so it contributes the nearest readable memories whatever their distance. Passing `watch=true` records a `MemoryWatch` (with - the default 1 h TTL) for every memory id returned in the result. + the default 1 h TTL) for every memory id returned in the result. A search with `q` (REST, or MCP `recall`) also + carries each registered search annotator's value for a hit (Extension points); a listing without `q` carries none. - **Full-text query boundary** (observed locally on Harper 5.3.2, native fulltext 0.5.0): `Array.from({ length: 65 }, (_, i) => 'w' + i).join(' ')` is only 249 characters but is rejected; 64 distinct analyzed terms succeed. Native `any` reports @@ -562,10 +564,10 @@ true` puts `@embed`'s HNSW index on the native plane). Harper 5.3 makes a query `principal.id`, newest pin first, whose target memory is active, unexpired and readable by the caller), then the caller's own memories that need review (section `needsReview`: up to 20 the caller may see, newest flag first, through the indexed `needsReviewSince`, from at most the 100 newest flags, so hidden flags cannot make a - bootstrap scan without bound), then the caller's recent + bootstrap scan without bound), then the sections other plugins registered (Extension points), then the caller's recent memories, then memories relevant to `q`, deduplicated and packed into a character budget (title + content; default 8000). Expired memories (`validUntil <= now`) are excluded from every section. - A memory that does not fit is skipped rather than ending the pack. Passing `watch=true` records a + A memory or contributed item that does not fit is skipped rather than ending the pack. Passing `watch=true` records a `MemoryWatch` (default 1 h TTL) for every memory id returned across all sections. Used at session start. - Ranking may add recency decay and retrieval/citation counts later; decay is a ranking function only, never deletion. @@ -673,6 +675,53 @@ as REST: sink registered or unregistered during a dispatch takes effect from the next one. An idempotent `PUT` retry calls no sink. harper-eye uses it for the interim Reflex source-row handoff until memory is a Reflex lane. +- `scanMemories({ after?, includeInactive? })` — an async iterable of `Memory` rows as stored (the view's fields with + their stored types, plus `embedding` and `embeddingModel`), oldest `createdAt` first, then id. By default only + `active` rows unexpired when the scan starts; `includeInactive: true` yields every row, superseded, withdrawn and + expired ones included. `after: { createdAt, id }`, the last row a caller saw, resumes strictly after it with the + `?order=created` tie-break, so rows sharing a `createdAt` are neither skipped nor repeated. It takes no principal + and applies no scope rule: it is the one read outside the guarded routes, for trusted in-process plugin code + building derived data, and no route reaches it. It reads in pages of 100 (`SCAN_PAGE`) with a context of its own + (`authorize: false`), so a consumer that pauses between rows holds no read open. It is a walk, not a snapshot + or a change feed: a row revised, withdrawn or expiring behind the cursor is not revisited, a row created with an + earlier `createdAt` (an import keeping its source time) lands behind it, and expiry makes no sink call. A derived + store therefore stays current from a sink registered before its walk starts, compares `revision` before applying + a scanned row over sink-driven state, and after a gap in which its sink did not run walks again from the start + with `includeInactive`. +- `registerBootstrapSection({ name, render })` returns an unregister function. `render({ principal, q, budgetChars, +context })` (sync or async) returns `{ text }`, an array of `{ title?, content }` items in priority order, or + nothing. Sections render after `pinned` and `needsReview` have packed, so `budgetChars` is the budget they left; + sections render concurrently, so each is told the same remainder. Their items pack, in registration order, + between `needsReview` and `recent`, each counted like a memory (title + content) and skipped when it does not + fit, never truncated; `{ text }` is one item, and agent-memory measures it. A packed item is an element of + `memories` that agent-memory builds as exactly `{ title, content, section: }`. Clients frame it as stored + data like any memory. It carries no `id`, so clients never watch, cache or deduplicate it as a memory and + `watch=true` watches memory ids only; and no `author`, `scopes` or `revision`, so contributed text cannot borrow + a principal's attribution. A client that charges framing per entry (the Claude Code plugin does) can omit an + item that filled `budgetChars`; several smaller items in priority order lose only their tail. A section that + throws, rejects, or returns anything else is omitted and logged at most once a minute per name; bootstrap still + answers. `name` matches `^[A-Za-z][A-Za-z0-9_-]{0,63}$`, is not a built-in section, and is unique in the domain; + otherwise registration throws a `TypeError`. Bootstrap has no MCP tool. +- `registerSearchAnnotator({ name, annotate })` returns an unregister function. `annotate({ principal, hits, +context })` (sync or async) returns a `Map` from memory id to a JSON value, or nothing; each hit whose id the + map holds with a value other than `undefined` gains it as `hit[name]`. It runs for searches with `q` (REST and + MCP `recall`), not for listings (the `order=created` walk feeds Copy between instances, whose import rejects + unknown fields) or bootstrap's `relevant` search. `hits` is one deep-frozen copy of the results shared by every + annotator, and agent-memory merges by id in its own order, so an annotator cannot add, remove, reorder or edit a + hit. The map is read through `Map.prototype`, so a `Map` from another app's realm works, and each value is copied + through JSON inside the annotator's error boundary, so a value the response could not serialize (a cycle, a + `BigInt`, a throwing `toJSON`) skips that annotator like a throw or a non-`Map` result: logged at most once a + minute per name, and the search still answers. `name` follows the section rule and is not a field a hit carries + (the view's fields and `score`); a later version that adds a hit field of the same name rejects that + registration. With no annotator registered, results go out untouched. +- **Contributed output is the contributor's to authorize.** Sections and annotators get the resolved principal, + and agent-memory does not filter what they return. `canReadLabels` is the label test only: ownership, withdrawal + and the customer-content quarantine (`contentFlag`; clearance is `superUser` or the `customer:*` read wildcard in + `principal.read`) are rule 1's other parts, and `scanMemories` yields flagged and private rows. Text derived from + several memories is shown only to a reader of all of them, checked against the sources' current state: a + rescope, flag or withdrawal reaches the contributor through its sink before its derived text is rebuilt, so it + must stop serving that text then, not after the rebuild. +- Sections and annotators are awaited, like sinks, with no time bound: a slow one delays the response. - `setPrincipalResolver(fn)` — see Identity. - `resolvePrincipal(context)` — the caller behind a request context, resolved exactly as the plugin's own routes resolve it (custom resolver, else the Harper user and its `ScopeGrant` rows, read on every call). It resolves @@ -700,19 +749,23 @@ as REST: loaders. All live apps on a worker share that pair. A deploy-validation candidate loads into its own `Resources`, and an app with branched databases has its own `databases`, so neither reaches the live plugin. Within a domain: - - Sinks and the principal resolver live in the domain, so every copy registers where the plugin dispatches, - whatever order the apps load in. The last `setPrincipalResolver` from any app wins. - - `handleApplication` publishes `{ version, resolvePrincipal, canReadLabels }` (the plugin's own functions) - once `configure()` succeeds, and never for an `isTransientValidation` scope. The exported `resolvePrincipal` - calls it when present, else resolves locally. `canReadLabels` is answered by the copy that compiled the + - Sinks, the principal resolver, bootstrap sections and search annotators live in the domain, so every copy + registers where the plugin dispatches, whatever order the apps load in. The last `setPrincipalResolver` from + any app wins. The section and annotator lists are added on first use (`??=`), since an older copy may have + created the domain object without them. + - `handleApplication` publishes `{ version, resolvePrincipal, canReadLabels, scanMemories }` (the plugin's own + functions) once `configure()` succeeds, and never for an `isTransientValidation` scope. The exported + `resolvePrincipal` and `scanMemories` call it when present, else run locally. A runtime without `scanMemories` + is 0.1.4 or earlier, which never renders sections or runs annotators: registering one beside it warns once + (`noteExtensionRegistration`), and a scan runs locally. `canReadLabels` is answered by the copy that compiled the principal (`compiledHere`), so a principal resolved before the plugin published keeps working. A different major version warns once and still delegates. The plugin's own routes call their functions directly; only `src/index.ts` forwards. - A second plugin instance in a domain throws before mounting: one resolver and one sink list cannot serve two hosts' registrations. Each worker decides independently which instance loaded first. The check, `configure()` and the publish run with no `await` between them. - - The anchor, the domain object, `MemoryRuntime` and the sink and resolver call shapes are a protocol between - package versions. They only gain members; an incompatible change needs a new anchor key. + - The anchor, the domain object, `MemoryRuntime` and the sink, resolver, section and annotator call shapes are a + protocol between package versions. They only gain members; an incompatible change needs a new anchor key. - A copy whose `harper` has no `resources` was loaded without an app scope (`moduleLoader` `native` or `compartment`, or `dependencyLoader: native`). It keeps a private domain, as every copy did before, so two such copies in different apps never meet. Mixed with a scoped plugin, whichever side comes second warns diff --git a/README.md b/README.md index 8577f03..57e3e3e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Shared, permissioned, live memory for agents and people, as a Harper plugin. - Customer-content flag: a `global` write that names a configured customer, or matches a configured host pattern, is committed but readable only by its owner and `customer:*` readers until a reviewer rescopes or confirms it; an optional model check runs after commit - Live updates: Level 1 real-time subscriptions on `/Memory` and `/Memory/` (SSE, WebSocket) with delivery-time read checks, and Level 2 watches (`POST /Memory/watch`, `watch=true` on search/bootstrap) delivered with own `needsReview` flags on `/MemoryEvents` - MCP surface (`mcp: true`): `remember`, `revise`, `recall`, `get`, `withdraw`, `pin`, `unpin`, `confirm`, `confirm-scope`, `watch` tools and `memory:///{id}` resource -- Extension points: `registerMemorySink` runs inside the lifecycle transaction for apps that mirror memories elsewhere, `setPrincipalResolver` maps app-defined identities, and `resolvePrincipal` / `canReadLabels` let other components authorize with the same identity and grants +- Extension points: `registerMemorySink` runs inside the lifecycle transaction for apps that mirror memories elsewhere, `setPrincipalResolver` maps app-defined identities, `resolvePrincipal` / `canReadLabels` let other components authorize with the same identity and grants, and `scanMemories`, `registerBootstrapSection` and `registerSearchAnnotator` let a plugin build derived data over memories and add it to bootstrap and search results Status: v0 (0.x releases may change contracts; see [CHANGELOG.md](CHANGELOG.md)). Design and contracts: [DESIGN.md](DESIGN.md). @@ -172,6 +172,42 @@ const principal = await resolvePrincipal(context); const visible = records.filter((record) => canReadLabels(principal, record.scopes)); ``` +### Derived data: scan, bootstrap sections, search annotations + +A plugin that builds something over memories (topic summaries, say) reads them with `scanMemories`, stays current +with a sink, and serves its output beside memories: + +```js +import { + canReadLabels, + registerBootstrapSection, + registerSearchAnnotator, + scanMemories, +} from '@harperfast/agent-memory'; + +// Every active, unexpired memory as stored (embedding included), oldest first. Pass the last row's +// { createdAt, id } as `after` to resume; `includeInactive: true` also yields superseded, withdrawn and expired rows. +for await (const memory of scanMemories({ after: savedCursor })) await index(memory); + +// Packed into GET /Memory/bootstrap after pinned and needsReview, before recent, within the caller's budget. +registerBootstrapSection({ + name: 'orientation', + render: async ({ principal, q, budgetChars, context }) => + (await topicsFor(principal, budgetChars)).map((topic) => ({ title: topic.title, content: topic.summary })), +}); + +// Adds hit.summaries to GET /Memory/?q= and MCP recall results; cannot add, drop or reorder hits. +registerSearchAnnotator({ + name: 'summaries', + annotate: async ({ principal, hits, context }) => new Map(await summariesFor(principal, hits)), +}); +``` + +The scan takes no principal: it is for trusted plugin code. Sections and annotators receive the caller's principal, +and agent-memory shows what they return as is, so they must show a caller only what that caller may read: all of +the memories a summary draws on, judged by their current scopes and content flags. A section or annotator that +throws is left out and logged; the request still succeeds. Contract: [DESIGN.md](DESIGN.md#extension-points). + `read` and `write` list the effective grant patterns: `user:`, exact labels and namespace wildcards, with role and user grants merged and patterns that grant nothing dropped. A super_user's lists do not enumerate its access; check `superUser`, or let `canReadLabels` do it. `canReadLabels` is the label rule only (any label @@ -179,7 +215,7 @@ matches), and accepts only a principal `resolvePrincipal` returned. Contract: [D ### From another app -These four functions also work in an app that imports the package without listing the plugin, while +These functions also work in an app that imports the package without listing the plugin, while agent-memory runs as its own app on the same Harper, for example so it can be upgraded separately. That app's copy of the package registers with the running plugin and asks it to resolve principals. Its sinks run inside the plugin's lifecycle transaction, and its authorization matches the plugin's routes even when the two versions @@ -193,7 +229,8 @@ differ. This needs: - Exactly one app listing the plugin in its `config.yaml`. A second one fails to load. Which of the two loads first can differ from worker to worker, so remove one. - A version of this package newer than 0.1.3 in the importing app. 0.1.3 and earlier keep registrations to - themselves. + themselves. Bootstrap sections and search annotators also need a running plugin newer than 0.1.4; beside 0.1.4 + they are never used, and registering one logs a warning. To move the plugin out of a host app: diff --git a/integrationTests/cross-app.test.ts b/integrationTests/cross-app.test.ts index fc19225..fd10a07 100644 --- a/integrationTests/cross-app.test.ts +++ b/integrationTests/cross-app.test.ts @@ -191,6 +191,48 @@ for (const moduleLoader of ['vm-current-context', 'vm']) { ['alice', ['user:alice', 'global'], false], ); }); + + test("the consumer's bootstrap section reaches the plugin's bootstrap, rendered for the caller", async () => { + const id = memId(`boot-${moduleLoader}`); + strictEqual((await call('alice', 'PUT', `Memory/${id}`, { title: 'x', content: 'plover' })).status, 201); + const booted = await call('alice', 'GET', 'Memory/bootstrap?budget=8000'); + strictEqual(booted.status, 200, JSON.stringify(booted.body)); + const sections = booted.body.memories.map((entry: any) => entry.section); + const at = sections.indexOf('consumerOrientation'); + ok(at !== -1 && at < sections.indexOf('recent'), JSON.stringify(sections)); + deepStrictEqual(booted.body.memories[at], { + title: 'Orientation', + content: 'for alice within 8000', + section: 'consumerOrientation', + }); + }); + + test("the consumer's annotator adds per-hit data to the plugin's search", async () => { + const id = memId(`note-${moduleLoader}`); + const created = await call('alice', 'PUT', `Memory/${id}`, { + title: 'x', + content: 'sandpiper probing', + visibility: 'shared', + }); + strictEqual(created.status, 201, JSON.stringify(created.body)); + const searched = await call('alice', 'GET', 'Memory/?q=sandpiper'); + strictEqual(searched.status, 200, JSON.stringify(searched.body)); + const hit = searched.body.memories.find((memory: any) => memory.id === id); + deepStrictEqual(hit?.consumerNote, { by: 'alice', readable: true }); + const listed = await call('alice', 'GET', 'Memory/?limit=50'); + ok( + listed.body.memories.every((memory: any) => !('consumerNote' in memory)), + 'a listing is not annotated', + ); + }); + + test("the consumer's scanMemories reads every memory through the plugin and resumes after a row", async () => { + const probe = await call('alice', 'GET', 'ScanProbe/'); + strictEqual(probe.status, 200, JSON.stringify(probe.body)); + ok(probe.body.ids.length >= 2, JSON.stringify(probe.body)); + deepStrictEqual(probe.body.resumed, probe.body.ids.slice(1)); + strictEqual(probe.body.embedded, true); + }); }, ); } diff --git a/integrationTests/fixtures/memory-consumer/resources.js b/integrationTests/fixtures/memory-consumer/resources.js index f222f1e..81cebba 100644 --- a/integrationTests/fixtures/memory-consumer/resources.js +++ b/integrationTests/fixtures/memory-consumer/resources.js @@ -1,4 +1,12 @@ -import { canReadLabels, registerMemorySink, resolvePrincipal, setPrincipalResolver } from '@harperfast/agent-memory'; +import { + canReadLabels, + registerBootstrapSection, + registerMemorySink, + registerSearchAnnotator, + resolvePrincipal, + scanMemories, + setPrincipalResolver, +} from '@harperfast/agent-memory'; // Writes before it can throw, so a failure must roll this row back along with the memory. const record = (kind) => async (memory, context) => { @@ -22,6 +30,37 @@ setPrincipalResolver((context, next) => : next(), ); +registerBootstrapSection({ + name: 'consumerOrientation', + render: ({ principal, budgetChars }) => [ + { title: 'Orientation', content: `for ${principal.id} within ${budgetChars}` }, + ], +}); + +// The Map is made in this app's realm; under the vm loader that is not the plugin's realm. +registerSearchAnnotator({ + name: 'consumerNote', + annotate: ({ principal, hits }) => + new Map(hits.map((hit) => [hit.id, { by: principal.id, readable: canReadLabels(principal, hit.scopes) }])), +}); + +export class ScanProbe extends Resource { + static async get() { + const rows = []; + for await (const row of scanMemories()) rows.push(row); + // The cursor carries the plugin's Date back into the plugin. + const resumed = []; + for await (const row of scanMemories({ after: { createdAt: rows[0].createdAt, id: rows[0].id } })) { + resumed.push(row.id); + } + return { + ids: rows.map((row) => row.id), + resumed, + embedded: rows.every((row) => row.embedding?.length > 0 && row.embeddingModel === 'default'), + }; + } +} + export class PrincipalProbe extends Resource { static async get(target, context) { const principal = await resolvePrincipal(context); diff --git a/src/extensions.ts b/src/extensions.ts new file mode 100644 index 0000000..b60fb7d --- /dev/null +++ b/src/extensions.ts @@ -0,0 +1,194 @@ +import { logger } from 'harper'; +import { domain, noteExtensionRegistration } from './runtime.ts'; +import type { Principal } from './scopes.ts'; +import { VIEW_FIELDS } from './search.ts'; + +export interface BootstrapItem { + title?: string; + content: string; +} + +export interface BootstrapRenderInput { + readonly principal: Principal; + readonly q: string | undefined; + readonly budgetChars: number; + readonly context: object; +} + +export type BootstrapSectionOutput = { text: string } | readonly BootstrapItem[] | null | undefined; + +export interface BootstrapSection { + readonly name: string; + render(input: BootstrapRenderInput): BootstrapSectionOutput | Promise; +} + +export interface SearchAnnotateInput { + readonly principal: Principal; + readonly hits: readonly Readonly>[]; + readonly context: object; +} + +export type SearchAnnotations = ReadonlyMap | null | undefined; + +export interface SearchAnnotator { + readonly name: string; + annotate(input: SearchAnnotateInput): SearchAnnotations | Promise; +} + +const NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; +const BUILT_IN_SECTIONS = ['pinned', 'needsReview', 'recent', 'relevant']; +const FAILURE_LOG_INTERVAL_MS = 60_000; +const lastFailureLog = new Map(); + +function register( + list: T[], + entry: T, + kind: string, + method: 'render' | 'annotate', + reserved: readonly string[], +): () => void { + const name = entry?.name; + if (typeof name !== 'string' || !NAME_PATTERN.test(name)) { + throw new TypeError(`@harperfast/agent-memory: a ${kind} name must match ${NAME_PATTERN}`); + } + if (typeof (entry as any)[method] !== 'function') { + throw new TypeError(`@harperfast/agent-memory: ${kind} "${name}" needs a ${method} function`); + } + if (reserved.includes(name)) throw new TypeError(`@harperfast/agent-memory: ${kind} name "${name}" is reserved`); + if (list.some((other) => other.name === name)) { + throw new TypeError(`@harperfast/agent-memory: ${kind} "${name}" is already registered`); + } + noteExtensionRegistration(); + list.push(entry); + return () => { + const index = list.indexOf(entry); + if (index !== -1) list.splice(index, 1); + }; +} + +/** + * Adds a section to `GET /Memory/bootstrap` after `pinned` and `needsReview` and before `recent`, rendered per + * caller. Its items pack into the caller's budget like memories; a throwing section is omitted and logged. Returns + * an unregister function. Contract: DESIGN.md, Extension points. + */ +export function registerBootstrapSection(section: BootstrapSection): () => void { + const target = domain(); + return register((target.bootstrapSections ??= []), section, 'bootstrap section', 'render', BUILT_IN_SECTIONS); +} + +/** + * Attaches per-hit data to hybrid search results (REST `q` searches and MCP `recall` with `q`) under the + * annotator's name. It cannot add, remove, reorder or change hits; a throwing annotator is skipped and logged. + * Returns an unregister function. Contract: DESIGN.md, Extension points. + */ +export function registerSearchAnnotator(annotator: SearchAnnotator): () => void { + const target = domain(); + return register((target.searchAnnotators ??= []), annotator, 'search annotator', 'annotate', [ + ...VIEW_FIELDS, + 'score', + ]); +} + +function reportFailure(kind: string, name: string, error: unknown): void { + const key = `${kind}:${name}`; + const now = Date.now(); + if (now - (lastFailureLog.get(key) ?? -Infinity) < FAILURE_LOG_INTERVAL_MS) return; + lastFailureLog.set(key, now); + try { + logger.error?.(`[agent-memory] ${kind} "${name}" failed and was left out of the response`, error); + } catch {} +} + +/** The sections registered now; one registered or unregistered later takes effect from the next bootstrap. */ +export function registeredBootstrapSections(): BootstrapSection[] { + return domain().bootstrapSections?.slice() ?? []; +} + +function bootstrapItems(output: unknown): BootstrapItem[] { + if (output === null || output === undefined) return []; + let list: readonly any[]; + if (Array.isArray(output)) list = output; + else if (typeof (output as any).text === 'string') list = [{ content: (output as any).text }]; + else throw new TypeError('render must return { text }, an array of { title?, content }, or nothing'); + return list.map((item) => { + const { title, content } = item ?? {}; + if (typeof content !== 'string' || (title !== undefined && typeof title !== 'string')) { + throw new TypeError('a bootstrap item must be { title?: string, content: string }'); + } + return title === undefined ? { content } : { title, content }; + }); +} + +/** Each section's items, in registration order. */ +export function renderBootstrapSections( + sections: readonly BootstrapSection[], + input: BootstrapRenderInput, +): Promise<{ name: string; items: BootstrapItem[] }[]> { + const shared = Object.freeze({ ...input }); + return Promise.all( + sections.map(async (section) => { + const name = section.name; + try { + return { name, items: bootstrapItems(await section.render(shared)) }; + } catch (error) { + reportFailure('bootstrap section', name, error); + return { name, items: [] }; + } + }), + ); +} + +function deepFreeze(value: T): T { + if (typeof value === 'object' && value !== null && !Object.isFrozen(value)) { + Object.freeze(value); + for (const child of Object.values(value)) deepFreeze(child); + } + return value; +} + +async function runAnnotators( + annotators: readonly SearchAnnotator[], + hits: Record[], + principal: Principal, + context: object, +): Promise { + const input = Object.freeze({ principal, hits: deepFreeze(structuredClone(hits)), context }); + const results = await Promise.all( + annotators.map(async (annotator) => { + const name = annotator.name; + try { + const annotations = await annotator.annotate(input); + if (annotations === null || annotations === undefined) return undefined; + // Called through Map.prototype so a Map from another app's realm works, and anything else throws. The JSON + // copy keeps a value the response could not serialize inside this catch. + const values = new Map(); + for (const { id } of hits) { + if (!Map.prototype.has.call(annotations, id)) continue; + const value = Map.prototype.get.call(annotations, id); + if (value !== undefined) values.set(id, JSON.parse(JSON.stringify(value))); + } + return { name, values }; + } catch (error) { + reportFailure('search annotator', name, error); + return undefined; + } + }), + ); + for (const result of results) { + if (result === undefined) continue; + for (const hit of hits) { + if (result.values.has(hit.id)) hit[result.name] = result.values.get(hit.id); + } + } +} + +/** Adds each annotator's value for a hit to that hit, in place. */ +export function annotateSearchHits( + hits: Record[], + principal: Principal, + context: object, +): void | Promise { + const annotators = domain().searchAnnotators; + if (!annotators?.length || hits.length === 0) return; + return runAnnotators(annotators.slice(), hits, principal, context); +} diff --git a/src/index.ts b/src/index.ts index 41bac86..9f5904c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,10 +4,23 @@ import { resolvePrincipal as resolveHere } from './principal.ts'; import { MEMORY_RESOURCES, MemoryMcpResource } from './resource.ts'; import { domain, foreignRuntime, publishRuntime, type MemoryRuntime } from './runtime.ts'; import { canReadLabels as canReadLabelsHere, compiledHere, type Principal } from './scopes.ts'; +import { scanMemories as scanHere, type ScanOptions } from './search.ts'; import { applySettings, parseSettings } from './settings.ts'; import { PACKAGE_VERSION } from './version.ts'; export { registerMemorySink, type MemorySink } from './sinks.ts'; +export { + registerBootstrapSection, + registerSearchAnnotator, + type BootstrapItem, + type BootstrapRenderInput, + type BootstrapSection, + type BootstrapSectionOutput, + type SearchAnnotateInput, + type SearchAnnotations, + type SearchAnnotator, +} from './extensions.ts'; +export type { ScanOptions } from './search.ts'; export { setPrincipalResolver, type PrincipalResolver, type PrincipalResolution } from './principal.ts'; export type { Principal } from './scopes.ts'; @@ -15,6 +28,7 @@ const RUNTIME: MemoryRuntime = Object.freeze({ version: PACKAGE_VERSION, resolvePrincipal: resolveHere, canReadLabels: canReadLabelsHere, + scanMemories: scanHere, }); /** @@ -44,6 +58,17 @@ export function canReadLabels(principal: Principal, labels: readonly string[] | : canReadLabelsHere(principal, labels); } +/** + * Every memory row as stored, oldest `createdAt` first, then by id: active and unexpired ones, or every row with + * `includeInactive`. `after` (the last row's `createdAt` and `id`) resumes strictly after that row. For trusted + * in-process code building derived data: it takes no principal and applies no scope rule. Read by the running + * plugin, even one in another app (DESIGN.md, Extension points). + */ +export function scanMemories(options?: ScanOptions): AsyncIterable> { + const runtime = foreignRuntime(RUNTIME); + return runtime?.scanMemories ? runtime.scanMemories(options) : scanHere(options); +} + function routePathFor(basePath: string, resource: { path: string }): string { return resource.path.replace(/^\/Memory(?=\/|$|(?:Events|Mcp)\b)/, basePath); } diff --git a/src/mcp.ts b/src/mcp.ts index e893472..066dac5 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto'; import { badRequest, isCallerSafe, memoryError, normalizeError, notFound } from './errors.ts'; import { WATCH_MAX_IDS, WATCH_MAX_TTL_SECONDS, watchMemories } from './events.ts'; +import { annotateSearchHits } from './extensions.ts'; import { confirmMemory, confirmScopeMemory, @@ -407,7 +408,9 @@ export const MEMORY_MCP_METHODS = { mcpRecall(params: unknown, context: unknown) { return runMcpTool('recall', context, async (principal, ctx) => { const searchParams = parseRecallParams(params); - const memories = await searchMemories(searchParams, principal, mcpLifecycleContext(ctx)); + const searchContext = mcpLifecycleContext(ctx); + const memories = await searchMemories(searchParams, principal, searchContext); + if (searchParams.q) await annotateSearchHits(memories, principal, searchContext); return { memories }; }); }, diff --git a/src/resource.ts b/src/resource.ts index 1f53a83..8a8d589 100644 --- a/src/resource.ts +++ b/src/resource.ts @@ -2,6 +2,7 @@ import { Resource } from 'harper'; import { badRequest, memoryError, normalizeError, notFound } from './errors.ts'; import { subscribeMemory, subscribeMemoryEvents, watchMemories, type SubscriptionStream } from './events.ts'; +import { annotateSearchHits } from './extensions.ts'; import { importMemories } from './import.ts'; import { confirmMemory, @@ -211,6 +212,7 @@ export class MemoryResource extends MemoryEndpoint { if (target?.rest) throw notFound(); const params = parseSearchParams(target); const memories = await searchMemories(params, principal, ctx); + if (params.q) await annotateSearchHits(memories, principal, ctx); // An older server ignores the parameter, so a full export needs this to know withdrawn rows were included. return json(params.includeWithdrawn ? { memories, includeWithdrawn: true } : { memories }); }); diff --git a/src/runtime.ts b/src/runtime.ts index 983bf52..5706250 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -1,6 +1,8 @@ import * as harper from 'harper'; +import type { BootstrapSection, SearchAnnotator } from './extensions.ts'; import type { PrincipalResolver } from './principal.ts'; import type { Principal } from './scopes.ts'; +import type { ScanOptions } from './search.ts'; import type { MemorySink } from './sinks.ts'; import { PACKAGE_VERSION } from './version.ts'; @@ -12,12 +14,17 @@ export interface MemoryRuntime { readonly version: string; resolvePrincipal(context: any): Promise; canReadLabels(principal: Principal, labels: readonly string[] | null | undefined): boolean; + // Absent from a runtime published by 0.1.4 or earlier, which also never renders sections or runs annotators. + scanMemories?(options?: ScanOptions): AsyncIterable>; } export interface MemoryDomain { readonly sinks: MemorySink[]; resolver: PrincipalResolver | undefined; runtime: MemoryRuntime | undefined; + // Absent from a domain an older copy created; added on first use. + bootstrapSections?: BootstrapSection[]; + searchAnnotators?: SearchAnnotator[]; } interface Anchor { @@ -34,6 +41,7 @@ const UNSCOPED_COPY = let memo: MemoryDomain | undefined; let scoped = false; let warnedUnscoped = false; +let warnedOldRuntime = false; let checkedRuntime: MemoryRuntime | undefined; function isObject(value: unknown): value is object { @@ -97,6 +105,17 @@ export function noteRegistration(): void { if (shared.scopedRuntime) warnUnscopedOnce(); } +/** For a section or annotator registration: an older running plugin never reads them, so say so once. */ +export function noteExtensionRegistration(): void { + noteRegistration(); + const runtime = domain().runtime; + if (warnedOldRuntime || runtime === undefined || typeof runtime.scanMemories === 'function') return; + warnedOldRuntime = true; + warn( + `@harperfast/agent-memory: the running plugin is ${runtime.version}, which never renders bootstrap sections or runs search annotators; upgrade it to use them`, + ); +} + export function publishRuntime(runtime: MemoryRuntime): void { domain().runtime = runtime; if (!scoped) return; diff --git a/src/search.ts b/src/search.ts index 3b02a85..7943d28 100644 --- a/src/search.ts +++ b/src/search.ts @@ -1,6 +1,7 @@ import { logger } from 'harper'; import { badRequest, memoryError } from './errors.ts'; import { recordWatches, WATCH_DEFAULT_TTL_SECONDS } from './events.ts'; +import { registeredBootstrapSections, renderBootstrapSections } from './extensions.ts'; import { BOOTSTRAP_PINNED_LIMIT, embedTexts, @@ -38,7 +39,7 @@ const BOOTSTRAP_REVIEW_LIMIT = 20; // Bounds the rows a bootstrap inspects when most of a caller's flags name sources it can no longer read. const BOOTSTRAP_REVIEW_SCAN = 5 * BOOTSTRAP_REVIEW_LIMIT; -const VIEW_FIELDS = [ +export const VIEW_FIELDS = [ 'id', 'title', 'content', @@ -146,6 +147,9 @@ export function parseSearchParams(params: URLSearchParams): SearchParams { } const EARLIEST = new Date(-8.64e15); +const SCAN_PAGE = 100; +// Vectors are only comparable within one model's space, so a builder needs the model beside the vector. +const SCAN_FIELDS = [...VIEW_FIELDS, 'embedding', 'embeddingModel']; export function createdOrderConditions( comparator: 'less_than' | 'greater_than', @@ -172,6 +176,54 @@ export function unexpiredCondition(now: Date): Condition { ]); } +export interface ScanOptions { + after?: { createdAt: Date | string | number; id: string }; + includeInactive?: boolean; +} + +/** + * Every stored memory row in `createdAt`, then id, order, for trusted in-process builders: no principal, no scope + * conditions. Paged, so a slow consumer never holds a read open across the walk; a walk, not a snapshot. + */ +export async function* scanMemories(options: ScanOptions = {}): AsyncGenerator> { + let after: CreatedPosition | undefined; + if (options.after !== undefined) { + // A Date from another app's realm is not a Date to Harper; this one is. + const createdAt = new Date(options.after?.createdAt as any); + if (Number.isNaN(createdAt.getTime()) || typeof options.after?.id !== 'string') { + throw new TypeError('scanMemories: after must be { createdAt, id } from a scanned row'); + } + after = { createdAt, id: options.after.id }; + } + const filter: Condition[] = options.includeInactive + ? [] + : [{ attribute: 'status', comparator: 'equals', value: STATUS_ACTIVE }, unexpiredCondition(new Date())]; + const Memory = memoryTable(); + for (;;) { + const page = await collect( + Memory.search( + { + conditions: [ + ...(after + ? createdOrderConditions('greater_than', after) + : [{ attribute: 'createdAt', comparator: 'greater_than_equal', value: EARLIEST }]), + ...filter, + ], + enforceExecutionOrder: true, + sort: { attribute: 'createdAt', next: { attribute: 'id' } }, + select: SCAN_FIELDS, + limit: SCAN_PAGE, + }, + { authorize: false }, + ), + ); + yield* page; + if (page.length < SCAN_PAGE) return; + const last = page[page.length - 1]; + after = { createdAt: last.createdAt, id: last.id }; + } +} + function filterConditions(params: SearchParams, principal: Principal, now = new Date()): Condition[] { const statuses = params.includeSuperseded ? [STATUS_ACTIVE, STATUS_SUPERSEDED] : [STATUS_ACTIVE]; if (params.includeWithdrawn) statuses.push(STATUS_WITHDRAWN); @@ -524,9 +576,45 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin if (chunk.length > 0) await admit(); return visible; }; - const [pinned, needsReview, recent, relevant] = await Promise.all([ - loadPinned(), - loadNeedsReview(), + const seen = new Set(); + const memories: Record[] = []; + let used = 0; + const fits = (title: unknown, content: unknown) => { + const size = String(title ?? '').length + String(content ?? '').length; + if (used + size > params.budget) return false; + used += size; + return true; + }; + const pack = (section: string, records: readonly MemoryView[]) => { + for (const memory of records) { + if (seen.has(memory.id) || !fits(memory.title, memory.content)) continue; + seen.add(memory.id); + memories.push({ ...memory, section }); + } + }; + const sections = registeredBootstrapSections(); + // Contributed sections render once pinned and needsReview have packed, so they know the budget left, while recent + // and relevant still load. One Promise.all observes every stage's rejection. + const head = Promise.all([loadPinned(), loadNeedsReview()]).then(async ([pinned, needsReview]) => { + pack('pinned', pinned); + pack('needsReview', needsReview); + if (sections.length === 0) return; + const rendered = await renderBootstrapSections(sections, { + principal, + q: params.q, + budgetChars: params.budget - used, + context, + }); + for (const { name, items } of rendered) { + // No id: clients must not watch, cache or deduplicate contributed text as a memory. + for (const item of items) { + if (fits(item.title, item.content)) + memories.push({ title: item.title ?? null, content: item.content, section: name }); + } + } + }); + const [, recent, relevant] = await Promise.all([ + head, collect( Memory.search( { @@ -555,31 +643,8 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin ).then((hits) => hits.map(({ score: _score, ...memory }) => memory)) : [], ]); - const seen = new Set(); - const memories = []; - let used = 0; - for (const [section, records] of [ - ['pinned', pinned], - ['needsReview', needsReview], - ['recent', recent], - ['relevant', relevant], - ] as const) { - for (const memory of records) { - if (seen.has(memory.id)) continue; - const size = String(memory.title ?? '').length + String(memory.content ?? '').length; - if (used + size > params.budget) continue; - seen.add(memory.id); - used += size; - memories.push({ ...memory, section }); - } - } - if (params.watch) { - await recordWatches( - memories.map((memory) => memory.id), - principal, - WATCH_DEFAULT_TTL_SECONDS, - context, - ); - } + pack('recent', recent); + pack('relevant', relevant); + if (params.watch) await recordWatches([...seen], principal, WATCH_DEFAULT_TTL_SECONDS, context); return { memories, budget: params.budget, used }; } diff --git a/test/cross-copy.test.js b/test/cross-copy.test.js index fca6ac8..40ebcf7 100644 --- a/test/cross-copy.test.js +++ b/test/cross-copy.test.js @@ -13,7 +13,9 @@ import { __test, databases, logger, resources } from 'harper'; import * as plugin from '../dist/index.js'; import { putMemory, withdrawMemory } from '../dist/lifecycle.js'; import { resolvePrincipal as pluginResolvePrincipal } from '../dist/principal.js'; +import { MemoryMcpResource, MemoryResource } from '../dist/resource.js'; import { canReadLabels as pluginCanReadLabels } from '../dist/scopes.js'; +import { bootstrapMemories, scanMemories as pluginScanMemories } from '../dist/search.js'; const ANCHOR = Symbol.for('@harperfast/agent-memory'); const stubUrl = new URL('./helpers/harper-stub.mjs', import.meta.url).href; @@ -133,6 +135,21 @@ describe('a second copy of the package', () => { assert.throws(() => pluginCanReadLabels(earlyPrincipal, ['global']), TypeError); }); + it('scans with its own code, and holds a section and an annotator registered before any plugin loads', async () => { + const scan = copy.scanMemories(); + assert.notEqual(Object.getPrototypeOf(scan), pluginScanMemories.prototype); + for await (const _row of scan); + copy.registerBootstrapSection({ + name: 'early', + render: ({ principal }) => [{ content: `early for ${principal.id}` }], + }); + copy.registerSearchAnnotator({ + name: 'earlyNote', + annotate: ({ hits }) => new Map(hits.map((hit) => [hit.id, 'early'])), + }); + assert.equal(warnings.length, 0); + }); + it('reaches the plugin with a sink and a resolver registered before the plugin loads', async () => { const calls = []; copy.registerMemorySink({ @@ -197,6 +214,57 @@ describe('a second copy of the package', () => { ); }); + it("uses the copy's sections and annotators in the plugin's bootstrap and search, and scans through the plugin", async () => { + const unregisterSection = copy.registerBootstrapSection({ + name: 'late', + render: ({ budgetChars }) => ({ text: `late with ${budgetChars}` }), + }); + const unregisterAnnotator = copy.registerSearchAnnotator({ + name: 'lateNote', + annotate: ({ principal, hits }) => + new Map(hits.map((hit) => [hit.id, { by: principal.id, readable: copy.canReadLabels(principal, hit.scopes) }])), + }); + const alice = await pluginResolvePrincipal(member('alice')); + const id = idFor('annotated'); + await putMemory(id, input('kestrel hovering'), alice, {}); + + const booted = await bootstrapMemories({ budget: 8000 }, alice, {}); + assert.deepEqual( + booted.memories.filter((entry) => entry.id === undefined).map((entry) => [entry.section, entry.content]), + [ + ['early', 'early for alice'], + ['late', 'late with 8000'], + ], + ); + + const response = await MemoryResource.get(new URLSearchParams('q=kestrel'), member('alice')); + const [hit] = (await response.json()).memories; + assert.equal(hit.id, id); + assert.equal(hit.earlyNote, 'early'); + assert.deepEqual(hit.lateNote, { by: 'alice', readable: true }); + const recalled = await new MemoryMcpResource().mcpRecall({ q: 'kestrel' }, member('alice')); + assert.deepEqual(recalled.memories[0].lateNote, { by: 'alice', readable: true }); + + const scan = copy.scanMemories(); + assert.equal(Object.getPrototypeOf(scan), pluginScanMemories.prototype); + const rows = []; + for await (const row of scan) rows.push(row); + assert.ok(rows.some((row) => row.id === id && Array.isArray(row.embedding))); + const resumed = []; + for await (const row of copy.scanMemories({ after: { createdAt: rows[0].createdAt, id: rows[0].id } })) { + resumed.push(row.id); + } + assert.deepEqual( + resumed, + rows.slice(1).map((row) => row.id), + ); + + unregisterSection(); + unregisterAnnotator(); + const after = await bootstrapMemories({ budget: 8000 }, alice, {}); + assert.ok(!after.memories.some((entry) => entry.section === 'late')); + }); + it('stops a sink unregistered through the copy', async () => { const calls = []; const unregister = copy.registerMemorySink({ write: (memory) => calls.push(memory.id) }); @@ -231,11 +299,20 @@ describe('a second copy of the package', () => { for (const other of [candidate, branch]) { other.registerMemorySink({ write: (memory) => calls.push(memory.id) }); other.setPrincipalResolver(() => ({ id: 'intruder', superUser: true })); + other.registerBootstrapSection({ name: 'intruder', render: () => [{ content: 'intruder' }] }); + other.registerSearchAnnotator({ + name: 'intruder', + annotate: ({ hits }) => new Map(hits.map((hit) => [hit.id, 1])), + }); } const erin = await pluginResolvePrincipal(member('erin')); assert.equal(erin.id, 'agent-erin'); await putMemory(idFor('apart'), input('apart'), erin, {}); assert.deepEqual(calls, []); + const booted = await bootstrapMemories({ budget: 8000 }, erin, {}); + assert.ok(!booted.memories.some((entry) => entry.section === 'intruder')); + const searched = await (await MemoryResource.get(new URLSearchParams('q=apart'), member('erin'))).json(); + assert.ok(searched.memories.length > 0 && searched.memories.every((hit) => !('intruder' in hit))); const viaBranch = await branch.resolvePrincipal(member('erin')); assert.equal(viaBranch.id, 'intruder'); diff --git a/test/extensions.test.js b/test/extensions.test.js new file mode 100644 index 0000000..afa7aa8 --- /dev/null +++ b/test/extensions.test.js @@ -0,0 +1,615 @@ +import { afterEach, beforeEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { runInNewContext } from 'node:vm'; +import { __test, databases, logger } from 'harper'; +import { loadBootstrapCache, loadSessionState } from '../claude-plugin/lib/cache.mjs'; +import { resolveConfig } from '../claude-plugin/lib/config.mjs'; +import { handleSessionStart } from '../claude-plugin/lib/hooks.mjs'; +import { annotateSearchHits } from '../dist/extensions.js'; +import { registerBootstrapSection, registerSearchAnnotator, scanMemories } from '../dist/index.js'; +import { pinMemory, putMemory, withdrawMemory } from '../dist/lifecycle.js'; +import { MemoryBootstrapResource, MemoryMcpResource, MemoryResource } from '../dist/resource.js'; +import { domain } from '../dist/runtime.js'; +import { createPrincipal } from '../dist/scopes.js'; +import { applySettings, parseSettings } from '../dist/settings.js'; +import { bootstrapMemories, parseSearchParams, searchMemories } from '../dist/search.js'; + +const alice = createPrincipal({ id: 'alice', grants: { read: ['global'], write: ['global'] } }); +const root = createPrincipal({ id: 'root', superUser: true }); +const context = {}; +const aliceUser = { username: 'alice' }; +const idFor = (key) => `mem-${key}--`.padEnd(16, '_'); +const create = async (principal, key, overrides = {}) => + (await putMemory(idFor(key), { title: key, content: `${key} content`, ...overrides }, principal, context)).memory; +const queryTarget = (query = '') => new URLSearchParams(query); +const sectionsOf = (result) => result.memories.map((entry) => [entry.id ?? null, entry.section]); + +const unregisters = []; +const register = (fn, entry) => { + const unregister = fn(entry); + unregisters.push(unregister); + return unregister; +}; +const errors = []; +const originalError = logger.error; + +beforeEach(async () => { + __test.reset(); + applySettings(parseSettings({})); + errors.length = 0; + logger.error = (...args) => errors.push(args); + await databases.memory.ScopeGrant.put({ id: 'user:alice', read: ['global'], write: ['global'] }); +}); + +afterEach(() => { + for (const unregister of unregisters.splice(0)) unregister(); + logger.error = originalError; +}); + +describe('scanMemories', () => { + // Rows written straight to the store, so createdAt can be shared the way an import preserving source time does. + const seed = (count, createdAtFor, overridesFor = () => ({})) => { + const rows = __test.stores.get('Memory'); + for (let i = 0; i < count; i++) { + const id = `mem-scan-${String(i).padStart(4, '0')}`.padEnd(16, '_'); + rows.set(id, { + id, + title: `row ${i}`, + content: `content ${i}`, + scopes: ['global'], + ownerId: 'alice', + status: 'active', + revision: 1, + embedding: [i, 1], + embeddingModel: 'default', + validUntil: null, + createdAt: createdAtFor(i), + updatedAt: createdAtFor(i), + ...overridesFor(i), + }); + } + }; + const collectScan = async (options) => { + const rows = []; + for await (const row of scanMemories(options)) rows.push(row); + return rows; + }; + const scanPages = () => + __test.searchLog.filter((entry) => entry.table === 'Memory' && entry.query.enforceExecutionOrder === true); + + it('yields active, unexpired rows as stored, oldest createdAt first then id, with the vector and its model', async () => { + const day = (n) => new Date(Date.UTC(2026, 0, n)); + seed( + 6, + (i) => day(10 - i), + (i) => + [ + {}, + { status: 'withdrawn' }, + { status: 'superseded', supersededBy: idFor('next') }, + { validUntil: new Date('2020-01-01T00:00:00Z') }, + { validUntil: new Date('2999-01-01T00:00:00Z') }, + { contentFlag: { kind: 'possible-customer-detail', since: day(1).toISOString(), source: 'pattern' } }, + ][i], + ); + const rows = await collectScan(); + assert.deepEqual( + rows.map((row) => row.title), + ['row 5', 'row 4', 'row 0'], + ); + const [flagged] = rows; + assert.ok(flagged.createdAt instanceof Date); + assert.deepEqual(flagged.embedding, [5, 1]); + assert.equal(flagged.embeddingModel, 'default'); + assert.equal(flagged.contentFlag.kind, 'possible-customer-detail'); + assert.ok(Object.isFrozen(flagged)); + assert.equal(scanPages()[0].query.sort.next.attribute, 'id'); + }); + + it('yields every row with includeInactive', async () => { + seed( + 4, + () => new Date(Date.UTC(2026, 0, 1)), + (i) => ({ + status: ['active', 'withdrawn', 'superseded', 'active'][i], + validUntil: i === 3 ? new Date('2020-01-01T00:00:00Z') : null, + }), + ); + assert.deepEqual( + (await collectScan({ includeInactive: true })).map((row) => row.status), + ['active', 'withdrawn', 'superseded', 'active'], + ); + }); + + it('pages through a same-createdAt group larger than a page, and resumes from any row without a skip or a repeat', async () => { + // 40 older rows, then 180 sharing one timestamp, so pages of 100 end inside the group. + const shared = new Date(Date.UTC(2026, 2, 1)); + seed(220, (i) => (i < 40 ? new Date(Date.UTC(2026, 1, 1) + i * 1000) : shared)); + const all = await collectScan(); + assert.equal(all.length, 220); + assert.equal(new Set(all.map((row) => row.id)).size, 220); + assert.deepEqual( + all.map((row) => row.id), + [...__test.stores.get('Memory').keys()].sort(), + ); + assert.equal(scanPages().length, 3); + assert.ok(scanPages().every((entry) => entry.query.limit === 100)); + + for (const index of [39, 40, 99, 100, 150, 218]) { + const last = all[index]; + const rest = await collectScan({ after: { createdAt: last.createdAt, id: last.id } }); + assert.deepEqual( + rest.map((row) => row.id), + all.slice(index + 1).map((row) => row.id), + `after row ${index}`, + ); + } + const last = all.at(-1); + assert.deepEqual(await collectScan({ after: { createdAt: last.createdAt.toISOString(), id: last.id } }), []); + }); + + it('accepts a cursor Date from another realm and rejects a malformed cursor', async () => { + seed(3, (i) => new Date(Date.UTC(2026, 0, 1 + i))); + const [first] = await collectScan(); + const foreign = runInNewContext(`new Date(${first.createdAt.getTime()})`); + assert.equal(foreign instanceof Date, false); + assert.equal((await collectScan({ after: { createdAt: foreign, id: first.id } })).length, 2); + for (const after of [{ createdAt: 'not a date', id: first.id }, { createdAt: first.createdAt }, {}]) { + await assert.rejects(collectScan({ after }), TypeError); + } + }); +}); + +describe('registerBootstrapSection', () => { + const setUp = async () => { + const guide = await create(root, 'guide', { content: 'pinned guide', visibility: 'shared' }); + await pinMemory(guide.id, {}, alice, context); + const source = await create(root, 'source', { content: 'kernel guidance', visibility: 'shared' }); + const citer = await create(alice, 'citer', { + content: 'my conclusion', + evidence: [{ kind: 'memory', ref: source.id }], + }); + await putMemory( + idFor('next'), + { title: 'next', content: 'tapir kernel guidance, corrected', visibility: 'shared', supersedes: [source.id] }, + root, + context, + ); + const mine = await create(alice, 'mine', { content: 'my recent note' }); + return { guide, citer, mine }; + }; + + it('packs a section after pinned and needsReview and before recent and relevant, told the budget they left', async () => { + const { guide, citer, mine } = await setUp(); + const calls = []; + register(registerBootstrapSection, { + name: 'orientation', + render(input) { + calls.push(input); + return [ + { title: 'Topics', content: 'kernels, tapirs' }, + { content: 'x'.repeat(10_000) }, + { content: 'storage' }, + ]; + }, + }); + register(registerBootstrapSection, { name: 'second', render: async () => ({ text: 'second section' }) }); + + const result = await bootstrapMemories({ q: 'tapir', budget: 8000 }, alice, context); + assert.deepEqual(sectionsOf(result), [ + [guide.id, 'pinned'], + [citer.id, 'needsReview'], + [null, 'orientation'], + [null, 'orientation'], + [null, 'second'], + [mine.id, 'recent'], + [idFor('next'), 'relevant'], + ]); + const contributed = result.memories.filter((entry) => entry.id === undefined); + assert.deepEqual(contributed, [ + { title: 'Topics', content: 'kernels, tapirs', section: 'orientation' }, + { title: null, content: 'storage', section: 'orientation' }, + { title: null, content: 'second section', section: 'second' }, + ]); + const headChars = ['guide', 'pinned guide', 'citer', 'my conclusion'].join('').length; + assert.equal(calls.length, 1); + assert.equal(calls[0].budgetChars, 8000 - headChars); + assert.equal(calls[0].principal, alice); + assert.equal(calls[0].q, 'tapir'); + assert.equal(calls[0].context, context); + assert.ok(Object.isFrozen(calls[0])); + assert.ok(result.used <= 8000); + }); + + it('never lets contributed output exceed the budget, and leaves less for recent and relevant', async () => { + await setUp(); + register(registerBootstrapSection, { + name: 'orientation', + render: ({ budgetChars }) => ({ text: 'o'.repeat(budgetChars) }), + }); + const result = await bootstrapMemories({ q: 'tapir', budget: 100 }, alice, context); + assert.deepEqual( + result.memories.map((entry) => entry.section), + ['pinned', 'needsReview', 'orientation'], + ); + assert.equal(result.used, 100); + + unregisters.pop()(); + register(registerBootstrapSection, { + name: 'orientation', + render: ({ budgetChars }) => ({ text: 'o'.repeat(budgetChars + 1) }), + }); + const over = await bootstrapMemories({ q: 'tapir', budget: 100 }, alice, context); + assert.ok(!over.memories.some((entry) => entry.section === 'orientation')); + assert.ok(over.used <= 100); + }); + + it('omits a section that throws, rejects or returns another shape, logging each at most once a minute', async () => { + await setUp(); + const broken = { + throws: () => { + throw new Error('boom'); + }, + rejects: async () => { + throw new Error('async boom'); + }, + wrongShape: () => 'just a string', + badItem: () => [{ content: 42 }], + badTitle: () => [{ title: 7, content: 'x' }], + }; + for (const [name, render] of Object.entries(broken)) register(registerBootstrapSection, { name, render }); + register(registerBootstrapSection, { name: 'healthy', render: () => [{ content: 'fine' }] }); + + for (let i = 0; i < 2; i++) { + const result = await bootstrapMemories({ budget: 8000 }, alice, context); + assert.deepEqual( + result.memories.filter((entry) => entry.id === undefined).map((entry) => entry.section), + ['healthy'], + ); + } + assert.deepEqual( + errors.map(([message]) => message.match(/"(\w+)"/)[1]), + Object.keys(broken), + ); + }); + + it('still answers when the logger itself throws while reporting a failed section', async () => { + logger.error = () => { + throw new Error('logger down'); + }; + register(registerBootstrapSection, { + name: 'loggerProbe', + render: () => { + throw new Error('boom'); + }, + }); + assert.deepEqual((await bootstrapMemories({ budget: 8000 }, alice, context)).memories, []); + }); + + it('rejects with the failing load while a section still renders, leaving no rejection unobserved', async () => { + await setUp(); + const unhandled = []; + const onUnhandled = (reason) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + const Memory = databases.memory.Memory; + const search = Memory.search; + Memory.search = function (query, ctx) { + // The recent load: the caller's own memories, newest first. + if (query.sort?.attribute === 'updatedAt' && query.limit === 50) throw new Error('recent load failed'); + return search.call(this, query, ctx); + }; + let rendered = false; + try { + register(registerBootstrapSection, { + name: 'slow', + render: async () => { + await new Promise((resolve) => setTimeout(resolve, 30)); + rendered = true; + return [{ content: 'late' }]; + }, + }); + await assert.rejects(bootstrapMemories({ budget: 8000 }, alice, context), /recent load failed/); + await new Promise((resolve) => setTimeout(resolve, 60)); + assert.equal(rendered, true); + assert.deepEqual(unhandled, []); + } finally { + Memory.search = search; + process.off('unhandledRejection', onUnhandled); + } + }); + + it('warns once when the running plugin predates sections and annotators', () => { + const warnings = []; + const originalWarn = logger.warn; + logger.warn = (message) => warnings.push(message); + const target = domain(); + target.runtime = { version: '0.1.4', resolvePrincipal() {}, canReadLabels() {} }; + try { + register(registerBootstrapSection, { name: 'orientation', render: () => [] }); + register(registerSearchAnnotator, { name: 'summaries', annotate: () => undefined }); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /running plugin is 0\.1\.4, which never renders bootstrap sections/); + } finally { + target.runtime = undefined; + logger.warn = originalWarn; + } + }); + + it('gives contributed entries no id, so watch=true watches memories only and the Claude client neither watches nor caches them', async () => { + const { guide, citer, mine } = await setUp(); + register(registerBootstrapSection, { + name: 'orientation', + render: () => [{ title: 'Topics', content: 'Main areas: kernels and tapirs.' }], + }); + const response = await MemoryBootstrapResource.get(queryTarget('budget=4000&watch=true'), { user: aliceUser }); + assert.equal(response.status, 200); + const body = await response.json(); + assert.deepEqual( + [...__test.stores.get('MemoryWatch').values()].map((row) => row.memoryId).sort(), + [guide.id, citer.id, mine.id].sort(), + ); + + const cacheDir = mkdtempSync(join(tmpdir(), 'agent-memory-sections-')); + try { + const env = { + AGENT_MEMORY_URL: 'https://memory.example.com', + AGENT_MEMORY_TOKEN: 'secret-token', + AGENT_MEMORY_USER: 'alice', + AGENT_MEMORY_CACHE_DIR: cacheDir, + AGENT_MEMORY_FOLLOWER: 'off', + }; + const config = resolveConfig(env); + const started = await handleSessionStart( + { session_id: 'sess-sections', source: 'startup' }, + { + config, + env, + spawnFollower: false, + nonce: 'aaaa1111bbbb2222', + fetchImpl: async () => new Response(JSON.stringify(body), { status: 200 }), + }, + ); + const text = started.hookSpecificOutput.additionalContext; + const dataBlock = text.slice(text.indexOf(' entry.section === 'orientation')); + } finally { + rmSync(cacheDir, { recursive: true, force: true }); + } + }); + + it('rejects a bad name, a built-in or taken name, or no render function, and stops rendering once unregistered', async () => { + for (const name of ['', '1abc', 'has space', '__proto__', 'x'.repeat(65), undefined]) { + assert.throws(() => registerBootstrapSection({ name, render: () => [] }), TypeError, String(name)); + } + for (const name of ['pinned', 'needsReview', 'recent', 'relevant']) { + assert.throws(() => registerBootstrapSection({ name, render: () => [] }), /reserved/); + } + assert.throws(() => registerBootstrapSection({ name: 'orientation' }), /render function/); + let renders = 0; + const unregister = register(registerBootstrapSection, { + name: 'orientation', + render: () => { + renders++; + return [{ content: 'x' }]; + }, + }); + assert.throws(() => registerBootstrapSection({ name: 'orientation', render: () => [] }), /already registered/); + await bootstrapMemories({ budget: 8000 }, alice, context); + unregister(); + await bootstrapMemories({ budget: 8000 }, alice, context); + assert.equal(renders, 1); + register(registerBootstrapSection, { name: 'orientation', render: () => [] }); + }); +}); + +describe('registerSearchAnnotator', () => { + const seedHits = async () => { + const first = await create(alice, 'heron-one', { content: 'heron wading', visibility: 'shared' }); + const second = await create(alice, 'heron-two', { content: 'heron fishing heron', visibility: 'shared' }); + const third = await create(alice, 'egret', { content: 'egret heron', visibility: 'shared' }); + return [first, second, third]; + }; + const restSearch = async (query) => { + const response = await MemoryResource.get(queryTarget(query), { user: aliceUser }); + assert.equal(response.status, 200); + return (await response.json()).memories; + }; + + it('attaches each annotator value under its name on REST and MCP searches, leaving the hits and their order alone', async () => { + await seedHits(); + const baseline = await restSearch('q=heron'); + const seen = []; + register(registerSearchAnnotator, { + name: 'summaries', + annotate({ principal, hits, context: annotateContext }) { + seen.push({ principal, hits, annotateContext }); + return new Map([ + [hits[0].id, [{ node: 'n1', size: 37 }]], + [hits[1].id, { node: 'n2' }], + ['mem-not-a-hit-0000', 'ignored'], + [hits[2].id, undefined], + ]); + }, + }); + register(registerSearchAnnotator, { + name: 'clusters', + annotate: async ({ hits }) => new Map(hits.map((hit, index) => [hit.id, index])), + }); + + const annotated = await restSearch('q=heron'); + assert.deepEqual( + annotated.map(({ summaries: _s, clusters: _c, ...hit }) => hit), + baseline, + ); + assert.deepEqual( + annotated.map((hit) => [hit.summaries, hit.clusters]), + [ + [[{ node: 'n1', size: 37 }], 0], + [{ node: 'n2' }, 1], + [undefined, 2], + ], + ); + assert.ok(!('summaries' in annotated[2])); + assert.equal(seen[0].principal.id, 'alice'); + assert.deepEqual(seen[0].hits, baseline); + + const recalled = await new MemoryMcpResource().mcpRecall({ q: 'heron' }, { user: aliceUser }); + assert.deepEqual( + recalled.memories.map((hit) => [hit.id, hit.clusters]), + baseline.map((hit, index) => [hit.id, index]), + ); + }); + + it('cannot add, remove, reorder or edit hits, even nested fields, and the healthy annotators still apply', async () => { + await seedHits(); + const baseline = await restSearch('q=heron'); + const attempts = { + sorts: ({ hits }) => hits.sort(() => -1), + pops: ({ hits }) => hits.pop(), + pushes: ({ hits }) => hits.push({ id: 'mem-injected-00000' }), + retitles: ({ hits }) => { + hits[0].title = 'changed'; + }, + rescopes: ({ hits }) => { + hits[0].scopes[0] = 'user:bob'; + }, + reattributes: ({ hits }) => { + hits[0].author.principalId = 'bob'; + }, + }; + for (const [name, annotate] of Object.entries(attempts)) register(registerSearchAnnotator, { name, annotate }); + register(registerSearchAnnotator, { name: 'healthy', annotate: ({ hits }) => new Map([[hits[0].id, true]]) }); + + const annotated = await restSearch('q=heron'); + assert.deepEqual( + annotated.map(({ healthy: _h, ...hit }) => hit), + baseline, + ); + assert.equal(annotated[0].healthy, true); + assert.deepEqual( + errors.map(([message]) => message.match(/"(\w+)"/)[1]), + Object.keys(attempts), + ); + }); + + it('skips an annotator that throws, returns a non-Map, or a value the response could not serialize', async () => { + await seedHits(); + const cycle = {}; + cycle.self = cycle; + const broken = { + throws: () => { + throw new Error('boom'); + }, + rejects: async () => { + throw new Error('async boom'); + }, + plainObject: ({ hits }) => ({ [hits[0].id]: 1 }), + cyclic: ({ hits }) => new Map([[hits[0].id, cycle]]), + bigint: ({ hits }) => new Map([[hits[0].id, 10n]]), + badToJSON: ({ hits }) => + new Map([ + [ + hits[0].id, + { + toJSON() { + throw new Error('no'); + }, + }, + ], + ]), + }; + for (const [name, annotate] of Object.entries(broken)) register(registerSearchAnnotator, { name, annotate }); + register(registerSearchAnnotator, { name: 'healthy', annotate: ({ hits }) => new Map([[hits[0].id, 'ok']]) }); + + const annotated = await restSearch('q=heron'); + assert.equal(annotated[0].healthy, 'ok'); + for (const name of Object.keys(broken)) + assert.ok( + annotated.every((hit) => !(name in hit)), + name, + ); + assert.deepEqual( + errors.map(([message]) => message.match(/"(\w+)"/)[1]), + Object.keys(broken), + ); + }); + + it("reads a Map from another app's realm and copies its values out", async () => { + const [first] = await seedHits(); + const value = { node: 'n1' }; + register(registerSearchAnnotator, { + name: 'summaries', + annotate: () => { + const map = runInNewContext('new Map()'); + map.set(first.id, value); + return map; + }, + }); + const hits = await searchMemories(parseSearchParams(new URLSearchParams('q=wading')), alice, context); + await annotateSearchHits(hits, alice, context); + assert.deepEqual(hits[0].summaries, { node: 'n1' }); + assert.notEqual(hits[0].summaries, value); + }); + + it('does no work without an annotator, and annotates neither q-less listings nor bootstrap', async () => { + await seedHits(); + await create(root, 'heron-shared', { content: 'heron shared by root', visibility: 'shared' }); + const hits = await searchMemories(parseSearchParams(new URLSearchParams('q=heron')), alice, context); + assert.equal(annotateSearchHits(hits, alice, context), undefined); + + let calls = 0; + register(registerSearchAnnotator, { + name: 'summaries', + annotate: ({ hits: given }) => { + calls++; + return new Map(given.map((hit) => [hit.id, 1])); + }, + }); + for (const query of ['', 'order=created', 'flagged=true']) { + assert.ok( + (await restSearch(query)).every((hit) => !('summaries' in hit)), + query, + ); + } + const recalled = await new MemoryMcpResource().mcpRecall({}, { user: aliceUser }); + assert.ok(recalled.memories.length > 0 && recalled.memories.every((hit) => !('summaries' in hit))); + const booted = await bootstrapMemories({ q: 'heron', budget: 8000 }, alice, context); + assert.ok(booted.memories.some((entry) => entry.section === 'relevant')); + assert.ok(booted.memories.every((entry) => !('summaries' in entry))); + assert.equal(calls, 0); + }); + + it('rejects a name a hit already carries, a taken name, or no annotate function', () => { + for (const name of ['title', 'content', 'scopes', 'author', 'score', 'updatedAt']) { + assert.throws(() => registerSearchAnnotator({ name, annotate: () => undefined }), /reserved/, name); + } + assert.throws(() => registerSearchAnnotator({ name: 'bad name', annotate: () => undefined }), TypeError); + assert.throws(() => registerSearchAnnotator({ name: 'summaries' }), /annotate function/); + register(registerSearchAnnotator, { name: 'summaries', annotate: () => undefined }); + assert.throws(() => registerSearchAnnotator({ name: 'summaries', annotate: () => undefined }), /already/); + }); +}); + +describe('withdrawn memories and annotations', () => { + it('hands annotators only the readable hits the search returned', async () => { + const kept = await create(alice, 'kept-heron', { content: 'heron kept', visibility: 'shared' }); + const gone = await create(alice, 'gone-heron', { content: 'heron gone', visibility: 'shared' }); + await withdrawMemory(gone.id, {}, alice, { headers: { 'if-match': '"1"' } }); + const given = []; + register(registerSearchAnnotator, { + name: 'summaries', + annotate: ({ hits }) => { + given.push(...hits.map((hit) => hit.id)); + return undefined; + }, + }); + await new MemoryMcpResource().mcpRecall({ q: 'heron' }, { user: aliceUser }); + assert.deepEqual(given, [kept.id]); + }); +}); From 621b3c1fbfbc76d732ad6727b9f40ba1d62e5447 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 10:28:09 -0700 Subject: [PATCH 2/8] Harden extension registrations against contributor mistakes Register a frozen copy of each section and annotator, so the name checked at registration is the one dispatch uses. Validate rendered items with Array.from, so a sparse array is caught inside the section's own boundary. Copy the scan cursor's Date before yielding the page. Skip the annotation await entirely when nothing is registered. Warn about an older running plugin also when the copy first calls it, once per runtime. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: agent-memory-summary-extension-points --- DESIGN.md | 6 ++++-- src/extensions.ts | 26 ++++++++++++++--------- src/mcp.ts | 3 ++- src/resource.ts | 3 ++- src/runtime.ts | 20 ++++++++++++------ src/search.ts | 26 ++++++++++++----------- test/extensions.test.js | 47 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 98 insertions(+), 33 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index bb46ce0..06da55b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -68,7 +68,8 @@ database). - `MemoryWatch` — a principal's time-bounded interest in specific memories (live updates). See `schema/memory.graphql`. The table is not `@export`ed: every read and write goes through the -`Memory` resource so the rules below have exactly one enforcement point. +`Memory` resource so the rules below have exactly one enforcement point. The one exception is `scanMemories` +(Extension points), an unguarded read for trusted in-process plugins that no route reaches. Server-owned fields: `ownerId`, `status`, `revision`, `author`, `lastEditor`, `supersededBy`, `autoSupersedes`, `evidenceMemoryIds`, `needsReview`, `needsReviewSince`, `confirmed`, `contentFlag`, @@ -757,7 +758,8 @@ context })` (sync or async) returns a `Map` from memory id to a JSON value, or n functions) once `configure()` succeeds, and never for an `isTransientValidation` scope. The exported `resolvePrincipal` and `scanMemories` call it when present, else run locally. A runtime without `scanMemories` is 0.1.4 or earlier, which never renders sections or runs annotators: registering one beside it warns once - (`noteExtensionRegistration`), and a scan runs locally. `canReadLabels` is answered by the copy that compiled the + (`warnIfOldRuntime`, also when this copy first calls a runtime it had not seen, so a registration made before the + old plugin published is caught by the copy's first scan), and a scan runs locally. `canReadLabels` is answered by the copy that compiled the principal (`compiledHere`), so a principal resolved before the plugin published keeps working. A different major version warns once and still delegates. The plugin's own routes call their functions directly; only `src/index.ts` forwards. diff --git a/src/extensions.ts b/src/extensions.ts index b60fb7d..3380e86 100644 --- a/src/extensions.ts +++ b/src/extensions.ts @@ -1,5 +1,5 @@ import { logger } from 'harper'; -import { domain, noteExtensionRegistration } from './runtime.ts'; +import { domain, noteExtensionRegistered, noteRegistration } from './runtime.ts'; import type { Principal } from './scopes.ts'; import { VIEW_FIELDS } from './search.ts'; @@ -40,6 +40,7 @@ const BUILT_IN_SECTIONS = ['pinned', 'needsReview', 'recent', 'relevant']; const FAILURE_LOG_INTERVAL_MS = 60_000; const lastFailureLog = new Map(); +// Stores a frozen copy, so the name checked here is the name dispatch uses. function register( list: T[], entry: T, @@ -48,20 +49,23 @@ function register( reserved: readonly string[], ): () => void { const name = entry?.name; + const fn = (entry as any)?.[method]; if (typeof name !== 'string' || !NAME_PATTERN.test(name)) { throw new TypeError(`@harperfast/agent-memory: a ${kind} name must match ${NAME_PATTERN}`); } - if (typeof (entry as any)[method] !== 'function') { + if (typeof fn !== 'function') { throw new TypeError(`@harperfast/agent-memory: ${kind} "${name}" needs a ${method} function`); } if (reserved.includes(name)) throw new TypeError(`@harperfast/agent-memory: ${kind} name "${name}" is reserved`); if (list.some((other) => other.name === name)) { throw new TypeError(`@harperfast/agent-memory: ${kind} "${name}" is already registered`); } - noteExtensionRegistration(); - list.push(entry); + noteRegistration(); + const registered = Object.freeze({ name, [method]: fn.bind(entry) }) as unknown as T; + list.push(registered); + noteExtensionRegistered(); return () => { - const index = list.indexOf(entry); + const index = list.indexOf(registered); if (index !== -1) list.splice(index, 1); }; } @@ -99,9 +103,12 @@ function reportFailure(kind: string, name: string, error: unknown): void { } catch {} } +const NO_SECTIONS: readonly BootstrapSection[] = Object.freeze([]); + /** The sections registered now; one registered or unregistered later takes effect from the next bootstrap. */ -export function registeredBootstrapSections(): BootstrapSection[] { - return domain().bootstrapSections?.slice() ?? []; +export function registeredBootstrapSections(): readonly BootstrapSection[] { + const sections = domain().bootstrapSections; + return sections?.length ? sections.slice() : NO_SECTIONS; } function bootstrapItems(output: unknown): BootstrapItem[] { @@ -110,7 +117,8 @@ function bootstrapItems(output: unknown): BootstrapItem[] { if (Array.isArray(output)) list = output; else if (typeof (output as any).text === 'string') list = [{ content: (output as any).text }]; else throw new TypeError('render must return { text }, an array of { title?, content }, or nothing'); - return list.map((item) => { + // Array.from visits holes, so a sparse array fails here, inside the section's own catch. + return Array.from(list, (item) => { const { title, content } = item ?? {}; if (typeof content !== 'string' || (title !== undefined && typeof title !== 'string')) { throw new TypeError('a bootstrap item must be { title?: string, content: string }'); @@ -119,7 +127,6 @@ function bootstrapItems(output: unknown): BootstrapItem[] { }); } -/** Each section's items, in registration order. */ export function renderBootstrapSections( sections: readonly BootstrapSection[], input: BootstrapRenderInput, @@ -182,7 +189,6 @@ async function runAnnotators( } } -/** Adds each annotator's value for a hit to that hit, in place. */ export function annotateSearchHits( hits: Record[], principal: Principal, diff --git a/src/mcp.ts b/src/mcp.ts index 066dac5..4c6ef27 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -410,7 +410,8 @@ export const MEMORY_MCP_METHODS = { const searchParams = parseRecallParams(params); const searchContext = mcpLifecycleContext(ctx); const memories = await searchMemories(searchParams, principal, searchContext); - if (searchParams.q) await annotateSearchHits(memories, principal, searchContext); + const annotating = searchParams.q ? annotateSearchHits(memories, principal, searchContext) : undefined; + if (annotating) await annotating; return { memories }; }); }, diff --git a/src/resource.ts b/src/resource.ts index 8a8d589..a8b3a1c 100644 --- a/src/resource.ts +++ b/src/resource.ts @@ -212,7 +212,8 @@ export class MemoryResource extends MemoryEndpoint { if (target?.rest) throw notFound(); const params = parseSearchParams(target); const memories = await searchMemories(params, principal, ctx); - if (params.q) await annotateSearchHits(memories, principal, ctx); + const annotating = params.q ? annotateSearchHits(memories, principal, ctx) : undefined; + if (annotating) await annotating; // An older server ignores the parameter, so a full export needs this to know withdrawn rows were included. return json(params.includeWithdrawn ? { memories, includeWithdrawn: true } : { memories }); }); diff --git a/src/runtime.ts b/src/runtime.ts index 5706250..dc0d90d 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -41,7 +41,7 @@ const UNSCOPED_COPY = let memo: MemoryDomain | undefined; let scoped = false; let warnedUnscoped = false; -let warnedOldRuntime = false; +let warnedOldRuntime: MemoryRuntime | undefined; let checkedRuntime: MemoryRuntime | undefined; function isObject(value: unknown): value is object { @@ -105,17 +105,22 @@ export function noteRegistration(): void { if (shared.scopedRuntime) warnUnscopedOnce(); } -/** For a section or annotator registration: an older running plugin never reads them, so say so once. */ -export function noteExtensionRegistration(): void { - noteRegistration(); - const runtime = domain().runtime; - if (warnedOldRuntime || runtime === undefined || typeof runtime.scanMemories === 'function') return; - warnedOldRuntime = true; +// Checked after a section or annotator registers and when this copy first meets a runtime, so a registration made +// before an older plugin published is caught on this copy's next call to it (a summary plugin's first scan). +function warnIfOldRuntime(runtime: MemoryRuntime | undefined): void { + if (runtime === undefined || runtime === warnedOldRuntime || typeof runtime.scanMemories === 'function') return; + const { bootstrapSections, searchAnnotators } = domain(); + if (!bootstrapSections?.length && !searchAnnotators?.length) return; + warnedOldRuntime = runtime; warn( `@harperfast/agent-memory: the running plugin is ${runtime.version}, which never renders bootstrap sections or runs search annotators; upgrade it to use them`, ); } +export function noteExtensionRegistered(): void { + warnIfOldRuntime(domain().runtime); +} + export function publishRuntime(runtime: MemoryRuntime): void { domain().runtime = runtime; if (!scoped) return; @@ -135,6 +140,7 @@ export function foreignRuntime(own: MemoryRuntime): MemoryRuntime | undefined { if (runtime === undefined || runtime === own) return undefined; if (runtime !== checkedRuntime) { checkedRuntime = runtime; + warnIfOldRuntime(runtime); if (major(runtime.version) !== OWN_MAJOR) { warn( `@harperfast/agent-memory: this app imports ${PACKAGE_VERSION} but the running plugin is ${runtime.version}; calls go to the running plugin`, diff --git a/src/search.ts b/src/search.ts index 7943d28..9a464c8 100644 --- a/src/search.ts +++ b/src/search.ts @@ -217,10 +217,11 @@ export async function* scanMemories(options: ScanOptions = {}): AsyncGenerator { + const head = Promise.all([loadPinned(), loadNeedsReview()]).then(([pinned, needsReview]) => { pack('pinned', pinned); pack('needsReview', needsReview); if (sections.length === 0) return; - const rendered = await renderBootstrapSections(sections, { + return renderBootstrapSections(sections, { principal, q: params.q, budgetChars: params.budget - used, context, - }); - for (const { name, items } of rendered) { - // No id: clients must not watch, cache or deduplicate contributed text as a memory. - for (const item of items) { - if (fits(item.title, item.content)) - memories.push({ title: item.title ?? null, content: item.content, section: name }); + }).then((rendered) => { + for (const { name, items } of rendered) { + // No id: clients must not watch, cache or deduplicate contributed text as a memory. + for (const item of items) { + if (fits(item.title, item.content)) + memories.push({ title: item.title ?? null, content: item.content, section: name }); + } } - } + }); }); const [, recent, relevant] = await Promise.all([ head, diff --git a/test/extensions.test.js b/test/extensions.test.js index afa7aa8..861b3cc 100644 --- a/test/extensions.test.js +++ b/test/extensions.test.js @@ -151,6 +151,16 @@ describe('scanMemories', () => { assert.deepEqual(await collectScan({ after: { createdAt: last.createdAt.toISOString(), id: last.id } }), []); }); + it('keeps its own cursor when the consumer mutates the createdAt it was handed', async () => { + seed(250, (i) => new Date(Date.UTC(2026, 0, 1) + i * 1000)); + const ids = []; + for await (const row of scanMemories()) { + ids.push(row.id); + row.createdAt.setUTCFullYear(2000); + } + assert.deepEqual(ids, [...__test.stores.get('Memory').keys()].sort()); + }); + it('accepts a cursor Date from another realm and rejects a malformed cursor', async () => { seed(3, (i) => new Date(Date.UTC(2026, 0, 1 + i))); const [first] = await collectScan(); @@ -259,6 +269,11 @@ describe('registerBootstrapSection', () => { wrongShape: () => 'just a string', badItem: () => [{ content: 42 }], badTitle: () => [{ title: 7, content: 'x' }], + sparse: () => { + const items = new Array(3); + items[0] = { content: 'first' }; + return items; + }, }; for (const [name, render] of Object.entries(broken)) register(registerBootstrapSection, { name, render }); register(registerBootstrapSection, { name: 'healthy', render: () => [{ content: 'fine' }] }); @@ -321,6 +336,25 @@ describe('registerBootstrapSection', () => { } }); + it('warns when a registration made before an older plugin published meets it on the next scan', async () => { + const warnings = []; + const originalWarn = logger.warn; + logger.warn = (message) => warnings.push(message); + const target = domain(); + try { + register(registerBootstrapSection, { name: 'orientation', render: () => [] }); + assert.deepEqual(warnings, []); + target.runtime = { version: '0.1.4', resolvePrincipal() {}, canReadLabels() {} }; + for await (const _row of scanMemories()); + register(registerSearchAnnotator, { name: 'summaries', annotate: () => undefined }); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /running plugin is 0\.1\.4/); + } finally { + target.runtime = undefined; + logger.warn = originalWarn; + } + }); + it('warns once when the running plugin predates sections and annotators', () => { const warnings = []; const originalWarn = logger.warn; @@ -585,6 +619,19 @@ describe('registerSearchAnnotator', () => { assert.equal(calls, 0); }); + it('dispatches under the name it checked, whatever the registered object later says', async () => { + await seedHits(); + const annotator = { name: 'notes', annotate: ({ hits }) => new Map(hits.map((hit) => [hit.id, 'note'])) }; + register(registerSearchAnnotator, annotator); + annotator.name = 'title'; + annotator.annotate = () => { + throw new Error('replaced'); + }; + const [hit] = await restSearch('q=wading'); + assert.equal(hit.title, 'heron-one'); + assert.equal(hit.notes, 'note'); + }); + it('rejects a name a hit already carries, a taken name, or no annotate function', () => { for (const name of ['title', 'content', 'scopes', 'author', 'score', 'updatedAt']) { assert.throws(() => registerSearchAnnotator({ name, annotate: () => undefined }), /reserved/, name); From 66a2fedebefd0dfb700824cbcc64a8d390fb5713 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 10:42:57 -0700 Subject: [PATCH 3/8] Answer a search without annotations when its hits cannot be copied for annotators Co-Authored-By: Claude Opus 5.5 Dispatch-Task: agent-memory-summary-extension-points --- src/extensions.ts | 9 ++++++++- src/search.ts | 2 +- test/extensions.test.js | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/extensions.ts b/src/extensions.ts index 3380e86..5c1aad3 100644 --- a/src/extensions.ts +++ b/src/extensions.ts @@ -159,7 +159,14 @@ async function runAnnotators( principal: Principal, context: object, ): Promise { - const input = Object.freeze({ principal, hits: deepFreeze(structuredClone(hits)), context }); + let snapshot: Record[]; + try { + snapshot = deepFreeze(structuredClone(hits)); + } catch (error) { + reportFailure('search annotation', 'snapshot', error); + return; + } + const input = Object.freeze({ principal, hits: snapshot, context }); const results = await Promise.all( annotators.map(async (annotator) => { const name = annotator.name; diff --git a/src/search.ts b/src/search.ts index 9a464c8..1414287 100644 --- a/src/search.ts +++ b/src/search.ts @@ -595,7 +595,7 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin }; const sections = registeredBootstrapSections(); // Contributed sections render once pinned and needsReview have packed, so they know the budget left, while recent - // and relevant still load. One Promise.all observes every stage's rejection. + // and relevant still load. Awaiting head on its own would leave a recent or relevant rejection unobserved. const head = Promise.all([loadPinned(), loadNeedsReview()]).then(([pinned, needsReview]) => { pack('pinned', pinned); pack('needsReview', needsReview); diff --git a/test/extensions.test.js b/test/extensions.test.js index 861b3cc..0dbfa83 100644 --- a/test/extensions.test.js +++ b/test/extensions.test.js @@ -574,6 +574,24 @@ describe('registerSearchAnnotator', () => { ); }); + it('answers without annotations when the hits cannot be copied for annotators', async () => { + await seedHits(); + let calls = 0; + register(registerSearchAnnotator, { + name: 'summaries', + annotate: () => { + calls++; + return undefined; + }, + }); + const hits = await searchMemories(parseSearchParams(new URLSearchParams('q=heron')), alice, context); + hits[0].uncloneable = () => {}; + await annotateSearchHits(hits, alice, context); + assert.equal(calls, 0); + assert.ok(hits.every((hit) => !('summaries' in hit))); + assert.match(errors[0][0], /"snapshot"/); + }); + it("reads a Map from another app's realm and copies its values out", async () => { const [first] = await seedHits(); const value = { node: 'n1' }; From 0045a8e58dc33e0bae6096af0ec619d2081ab977 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 10:56:58 -0700 Subject: [PATCH 4/8] Skip the section stage in a bootstrap with no sections, and catch an older plugin publishing late Bootstrap with no registered section keeps one four-way Promise.all. A section or annotator registered before any plugin published makes the domain's runtime an accessor, so an older plugin's plain publish runs the same old-runtime check. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: agent-memory-summary-extension-points --- DESIGN.md | 6 +- .../fixtures/memory-consumer/resources.js | 1 - src/runtime.ts | 18 ++++- src/search.ts | 67 +++++++++++-------- test/extensions.test.js | 4 +- 5 files changed, 61 insertions(+), 35 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 06da55b..4a91e8f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -758,8 +758,10 @@ context })` (sync or async) returns a `Map` from memory id to a JSON value, or n functions) once `configure()` succeeds, and never for an `isTransientValidation` scope. The exported `resolvePrincipal` and `scanMemories` call it when present, else run locally. A runtime without `scanMemories` is 0.1.4 or earlier, which never renders sections or runs annotators: registering one beside it warns once - (`warnIfOldRuntime`, also when this copy first calls a runtime it had not seen, so a registration made before the - old plugin published is caught by the copy's first scan), and a scan runs locally. `canReadLabels` is answered by the copy that compiled the + per runtime (`warnIfOldRuntime`). A registration made before any plugin published turns the domain's `runtime` + into an accessor whose setter runs the same check (`noteExtensionRegistered`), since an older plugin publishes + with a plain assignment; reads and writes of `runtime` behave as before. A scan beside an older runtime runs + locally. `canReadLabels` is answered by the copy that compiled the principal (`compiledHere`), so a principal resolved before the plugin published keeps working. A different major version warns once and still delegates. The plugin's own routes call their functions directly; only `src/index.ts` forwards. diff --git a/integrationTests/fixtures/memory-consumer/resources.js b/integrationTests/fixtures/memory-consumer/resources.js index 81cebba..33ea026 100644 --- a/integrationTests/fixtures/memory-consumer/resources.js +++ b/integrationTests/fixtures/memory-consumer/resources.js @@ -48,7 +48,6 @@ export class ScanProbe extends Resource { static async get() { const rows = []; for await (const row of scanMemories()) rows.push(row); - // The cursor carries the plugin's Date back into the plugin. const resumed = []; for await (const row of scanMemories({ after: { createdAt: rows[0].createdAt, id: rows[0].id } })) { resumed.push(row.id); diff --git a/src/runtime.ts b/src/runtime.ts index dc0d90d..bc196fd 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -105,8 +105,6 @@ export function noteRegistration(): void { if (shared.scopedRuntime) warnUnscopedOnce(); } -// Checked after a section or annotator registers and when this copy first meets a runtime, so a registration made -// before an older plugin published is caught on this copy's next call to it (a summary plugin's first scan). function warnIfOldRuntime(runtime: MemoryRuntime | undefined): void { if (runtime === undefined || runtime === warnedOldRuntime || typeof runtime.scanMemories === 'function') return; const { bootstrapSections, searchAnnotators } = domain(); @@ -118,7 +116,21 @@ function warnIfOldRuntime(runtime: MemoryRuntime | undefined): void { } export function noteExtensionRegistered(): void { - warnIfOldRuntime(domain().runtime); + const target = domain(); + warnIfOldRuntime(target.runtime); + if (target.runtime !== undefined || Object.getOwnPropertyDescriptor(target, 'runtime')?.set) return; + // No plugin has published yet, and one from 0.1.4 or earlier will not look at these lists when it does: it + // publishes with a plain assignment, which this setter observes. Reads and writes behave as before. + let runtime: MemoryRuntime | undefined; + Object.defineProperty(target, 'runtime', { + configurable: true, + enumerable: true, + get: () => runtime, + set: (value: MemoryRuntime | undefined) => { + runtime = value; + warnIfOldRuntime(value); + }, + }); } export function publishRuntime(runtime: MemoryRuntime): void { diff --git a/src/search.ts b/src/search.ts index 1414287..3562da5 100644 --- a/src/search.ts +++ b/src/search.ts @@ -593,30 +593,7 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin memories.push({ ...memory, section }); } }; - const sections = registeredBootstrapSections(); - // Contributed sections render once pinned and needsReview have packed, so they know the budget left, while recent - // and relevant still load. Awaiting head on its own would leave a recent or relevant rejection unobserved. - const head = Promise.all([loadPinned(), loadNeedsReview()]).then(([pinned, needsReview]) => { - pack('pinned', pinned); - pack('needsReview', needsReview); - if (sections.length === 0) return; - return renderBootstrapSections(sections, { - principal, - q: params.q, - budgetChars: params.budget - used, - context, - }).then((rendered) => { - for (const { name, items } of rendered) { - // No id: clients must not watch, cache or deduplicate contributed text as a memory. - for (const item of items) { - if (fits(item.title, item.content)) - memories.push({ title: item.title ?? null, content: item.content, section: name }); - } - } - }); - }); - const [, recent, relevant] = await Promise.all([ - head, + const loadRecent = () => collect( Memory.search( { @@ -627,7 +604,8 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin }, context, ), - ).then((records) => forReader(records.map(toMemoryView), principal, context, access)), + ).then((records) => forReader(records.map(toMemoryView), principal, context, access)); + const loadRelevant = () => params.q ? searchMemories( { @@ -643,8 +621,43 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin context, access, ).then((hits) => hits.map(({ score: _score, ...memory }) => memory)) - : [], - ]); + : []; + const sections = registeredBootstrapSections(); + let recent: MemoryView[]; + let relevant: MemoryView[]; + if (sections.length === 0) { + let pinned: MemoryView[]; + let needsReview: MemoryView[]; + [pinned, needsReview, recent, relevant] = await Promise.all([ + loadPinned(), + loadNeedsReview(), + loadRecent(), + loadRelevant(), + ]); + pack('pinned', pinned); + pack('needsReview', needsReview); + } else { + // Sections render once pinned and needsReview have packed, so they know the budget left, while recent and + // relevant still load. Awaiting the head on its own would leave a recent or relevant rejection unobserved. + const head = Promise.all([loadPinned(), loadNeedsReview()]).then(async ([pinned, needsReview]) => { + pack('pinned', pinned); + pack('needsReview', needsReview); + const rendered = await renderBootstrapSections(sections, { + principal, + q: params.q, + budgetChars: params.budget - used, + context, + }); + for (const { name, items } of rendered) { + // No id: clients must not watch, cache or deduplicate contributed text as a memory. + for (const item of items) { + if (fits(item.title, item.content)) + memories.push({ title: item.title ?? null, content: item.content, section: name }); + } + } + }); + [, recent, relevant] = await Promise.all([head, loadRecent(), loadRelevant()]); + } pack('recent', recent); pack('relevant', relevant); if (params.watch) await recordWatches([...seen], principal, WATCH_DEFAULT_TTL_SECONDS, context); diff --git a/test/extensions.test.js b/test/extensions.test.js index 0dbfa83..b5dc752 100644 --- a/test/extensions.test.js +++ b/test/extensions.test.js @@ -312,7 +312,6 @@ describe('registerBootstrapSection', () => { const Memory = databases.memory.Memory; const search = Memory.search; Memory.search = function (query, ctx) { - // The recent load: the caller's own memories, newest first. if (query.sort?.attribute === 'updatedAt' && query.limit === 50) throw new Error('recent load failed'); return search.call(this, query, ctx); }; @@ -336,7 +335,7 @@ describe('registerBootstrapSection', () => { } }); - it('warns when a registration made before an older plugin published meets it on the next scan', async () => { + it('warns when an older plugin publishes after a section registered, and once per runtime', async () => { const warnings = []; const originalWarn = logger.warn; logger.warn = (message) => warnings.push(message); @@ -345,6 +344,7 @@ describe('registerBootstrapSection', () => { register(registerBootstrapSection, { name: 'orientation', render: () => [] }); assert.deepEqual(warnings, []); target.runtime = { version: '0.1.4', resolvePrincipal() {}, canReadLabels() {} }; + assert.equal(warnings.length, 1); for await (const _row of scanMemories()); register(registerSearchAnnotator, { name: 'summaries', annotate: () => undefined }); assert.equal(warnings.length, 1); From cb05f0afe5c1ed2846a8eac333e12736f109bd30 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 11:15:33 -0700 Subject: [PATCH 5/8] Recheck contributor names in the running plugin, and keep bootstrap loads from escaping Promise.all The running plugin skips an annotator whose name is a hit field in its version, or a section named like a built-in, since a copy of another version validated the name against its own lists. The recent and relevant loads are async, so a synchronous query throw rejects instead of leaving the other loads' rejections unobserved. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: agent-memory-summary-extension-points --- DESIGN.md | 5 ++-- src/extensions.ts | 11 +++++--- src/runtime.ts | 3 +-- src/search.ts | 5 ++-- test/extensions.test.js | 56 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 10 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 4a91e8f..f5b7dcd 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -713,8 +713,9 @@ context })` (sync or async) returns a `Map` from memory id to a JSON value, or n through JSON inside the annotator's error boundary, so a value the response could not serialize (a cycle, a `BigInt`, a throwing `toJSON`) skips that annotator like a throw or a non-`Map` result: logged at most once a minute per name, and the search still answers. `name` follows the section rule and is not a field a hit carries - (the view's fields and `score`); a later version that adds a hit field of the same name rejects that - registration. With no annotator registered, results go out untouched. + (the view's fields and `score`). The running plugin checks that again at dispatch, since a copy of another + version checked it against its own fields: an annotator whose name is a hit field in the plugin's version is + skipped and logged, as is a section named like a built-in. With no annotator registered, results go out untouched. - **Contributed output is the contributor's to authorize.** Sections and annotators get the resolved principal, and agent-memory does not filter what they return. `canReadLabels` is the label test only: ownership, withdrawal and the customer-content quarantine (`contentFlag`; clearance is `superUser` or the `customer:*` read wildcard in diff --git a/src/extensions.ts b/src/extensions.ts index 5c1aad3..05eb9b1 100644 --- a/src/extensions.ts +++ b/src/extensions.ts @@ -37,6 +37,7 @@ export interface SearchAnnotator { const NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; const BUILT_IN_SECTIONS = ['pinned', 'needsReview', 'recent', 'relevant']; +const hitFields = () => [...VIEW_FIELDS, 'score']; const FAILURE_LOG_INTERVAL_MS = 60_000; const lastFailureLog = new Map(); @@ -87,10 +88,7 @@ export function registerBootstrapSection(section: BootstrapSection): () => void */ export function registerSearchAnnotator(annotator: SearchAnnotator): () => void { const target = domain(); - return register((target.searchAnnotators ??= []), annotator, 'search annotator', 'annotate', [ - ...VIEW_FIELDS, - 'score', - ]); + return register((target.searchAnnotators ??= []), annotator, 'search annotator', 'annotate', hitFields()); } function reportFailure(kind: string, name: string, error: unknown): void { @@ -136,6 +134,8 @@ export function renderBootstrapSections( sections.map(async (section) => { const name = section.name; try { + // A copy of another version checked the name against its own list at registration. + if (BUILT_IN_SECTIONS.includes(name)) throw new TypeError(`section name "${name}" is reserved`); return { name, items: bootstrapItems(await section.render(shared)) }; } catch (error) { reportFailure('bootstrap section', name, error); @@ -167,10 +167,13 @@ async function runAnnotators( return; } const input = Object.freeze({ principal, hits: snapshot, context }); + const reserved = hitFields(); const results = await Promise.all( annotators.map(async (annotator) => { const name = annotator.name; try { + // A copy of another version checked the name against its own hit fields at registration. + if (reserved.includes(name)) throw new TypeError(`annotator name "${name}" is a hit field`); const annotations = await annotator.annotate(input); if (annotations === null || annotations === undefined) return undefined; // Called through Map.prototype so a Map from another app's realm works, and anything else throws. The JSON diff --git a/src/runtime.ts b/src/runtime.ts index bc196fd..5500b7a 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -119,8 +119,7 @@ export function noteExtensionRegistered(): void { const target = domain(); warnIfOldRuntime(target.runtime); if (target.runtime !== undefined || Object.getOwnPropertyDescriptor(target, 'runtime')?.set) return; - // No plugin has published yet, and one from 0.1.4 or earlier will not look at these lists when it does: it - // publishes with a plain assignment, which this setter observes. Reads and writes behave as before. + // A plugin from 0.1.4 or earlier never reads these lists, and publishes with a plain assignment this setter sees. let runtime: MemoryRuntime | undefined; Object.defineProperty(target, 'runtime', { configurable: true, diff --git a/src/search.ts b/src/search.ts index 3562da5..9ecfba7 100644 --- a/src/search.ts +++ b/src/search.ts @@ -593,7 +593,8 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin memories.push({ ...memory, section }); } }; - const loadRecent = () => + // async, so a synchronous throw from a query rejects rather than escaping before Promise.all observes the others. + const loadRecent = async () => collect( Memory.search( { @@ -605,7 +606,7 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin context, ), ).then((records) => forReader(records.map(toMemoryView), principal, context, access)); - const loadRelevant = () => + const loadRelevant = async () => params.q ? searchMemories( { diff --git a/test/extensions.test.js b/test/extensions.test.js index b5dc752..4803ddf 100644 --- a/test/extensions.test.js +++ b/test/extensions.test.js @@ -335,6 +335,45 @@ describe('registerBootstrapSection', () => { } }); + it('rejects cleanly when every query throws synchronously, with or without a section', async () => { + await setUp(); + const unhandled = []; + const onUnhandled = (reason) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + const Memory = databases.memory.Memory; + const search = Memory.search; + Memory.search = () => { + throw new Error('database closed'); + }; + try { + await assert.rejects(bootstrapMemories({ q: 'tapir', budget: 8000 }, alice, context), /database closed/); + register(registerBootstrapSection, { name: 'orientation', render: () => [] }); + await assert.rejects(bootstrapMemories({ q: 'tapir', budget: 8000 }, alice, context), /database closed/); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.deepEqual(unhandled, []); + } finally { + Memory.search = search; + process.off('unhandledRejection', onUnhandled); + } + }); + + it("skips a section another version's copy registered under a built-in name", async () => { + const { mine } = await setUp(); + const sections = (domain().bootstrapSections ??= []); + const smuggled = Object.freeze({ name: 'recent', render: () => [{ content: 'not a memory' }] }); + sections.push(smuggled); + try { + const result = await bootstrapMemories({ budget: 8000 }, alice, context); + assert.deepEqual( + result.memories.filter((entry) => entry.section === 'recent').map((entry) => entry.id), + [mine.id], + ); + assert.match(errors[0][0], /"recent"/); + } finally { + sections.splice(sections.indexOf(smuggled), 1); + } + }); + it('warns when an older plugin publishes after a section registered, and once per runtime', async () => { const warnings = []; const originalWarn = logger.warn; @@ -650,6 +689,23 @@ describe('registerSearchAnnotator', () => { assert.equal(hit.notes, 'note'); }); + it("skips an annotator another version's copy registered under a name that is a hit field here", async () => { + await seedHits(); + const annotators = (domain().searchAnnotators ??= []); + const smuggled = Object.freeze({ + name: 'title', + annotate: ({ hits }) => new Map(hits.map((hit) => [hit.id, 'x'])), + }); + annotators.push(smuggled); + try { + const [hit] = await restSearch('q=wading'); + assert.equal(hit.title, 'heron-one'); + assert.match(errors[0][0], /"title"/); + } finally { + annotators.splice(annotators.indexOf(smuggled), 1); + } + }); + it('rejects a name a hit already carries, a taken name, or no annotate function', () => { for (const name of ['title', 'content', 'scopes', 'author', 'score', 'updatedAt']) { assert.throws(() => registerSearchAnnotator({ name, annotate: () => undefined }), /reserved/, name); From 795df57e0ba66853963c2817a79efd61d0632968 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 11:46:31 -0700 Subject: [PATCH 6/8] Treat contributors as trusted in-process code, as sinks are Annotators get a frozen copy of the hit list holding the result objects, so they still cannot add, drop or reorder hits, but agent-memory no longer deep-clones the hits, JSON-copies annotation values, reads Maps through Map.prototype, checks for sparse item arrays, or rechecks contributor names in the running plugin. Failures (a throw or a rejection) are still contained and logged. The recent and relevant bootstrap loads return to the shape origin/main has. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: agent-memory-summary-extension-points --- DESIGN.md | 20 +++---- src/extensions.ts | 58 +++++-------------- src/search.ts | 5 +- test/extensions.test.js | 122 ++++------------------------------------ 4 files changed, 38 insertions(+), 167 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index f5b7dcd..31d58cf 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -707,15 +707,13 @@ context })` (sync or async) returns `{ text }`, an array of `{ title?, content } context })` (sync or async) returns a `Map` from memory id to a JSON value, or nothing; each hit whose id the map holds with a value other than `undefined` gains it as `hit[name]`. It runs for searches with `q` (REST and MCP `recall`), not for listings (the `order=created` walk feeds Copy between instances, whose import rejects - unknown fields) or bootstrap's `relevant` search. `hits` is one deep-frozen copy of the results shared by every - annotator, and agent-memory merges by id in its own order, so an annotator cannot add, remove, reorder or edit a - hit. The map is read through `Map.prototype`, so a `Map` from another app's realm works, and each value is copied - through JSON inside the annotator's error boundary, so a value the response could not serialize (a cycle, a - `BigInt`, a throwing `toJSON`) skips that annotator like a throw or a non-`Map` result: logged at most once a - minute per name, and the search still answers. `name` follows the section rule and is not a field a hit carries - (the view's fields and `score`). The running plugin checks that again at dispatch, since a copy of another - version checked it against its own fields: an annotator whose name is a hit field in the plugin's version is - skipped and logged, as is a section named like a built-in. With no annotator registered, results go out untouched. + unknown fields) or bootstrap's `relevant` search. `hits` is a frozen copy of the result list holding the result + objects themselves, which annotators must not change; agent-memory merges by position in its own list, so an + annotator cannot add, remove or reorder hits. One that throws, rejects, or returns something without a `get` + method is skipped and logged at most once a minute per name, and the search still answers. `name` follows the + section rule and is not a field a hit carries (the view's fields and `score`), checked by the registering copy: + a plugin of a later version that adds a hit field of that name sees it overwritten, so pick a name specific to + the plugin. With no annotator registered, results go out untouched. - **Contributed output is the contributor's to authorize.** Sections and annotators get the resolved principal, and agent-memory does not filter what they return. `canReadLabels` is the label test only: ownership, withdrawal and the customer-content quarantine (`contentFlag`; clearance is `superUser` or the `customer:*` read wildcard in @@ -723,7 +721,9 @@ context })` (sync or async) returns a `Map` from memory id to a JSON value, or n several memories is shown only to a reader of all of them, checked against the sources' current state: a rescope, flag or withdrawal reaches the contributor through its sink before its derived text is rebuilt, so it must stop serving that text then, not after the rebuild. -- Sections and annotators are awaited, like sinks, with no time bound: a slow one delays the response. +- Contributors (sections, annotators, `scanMemories` callers) are trusted in-process code, as sinks are: + agent-memory contains their failures, a throw or a rejection, but does not defend against the values they + return or the objects they are handed. They are awaited with no time bound, so a slow one delays the response. - `setPrincipalResolver(fn)` — see Identity. - `resolvePrincipal(context)` — the caller behind a request context, resolved exactly as the plugin's own routes resolve it (custom resolver, else the Harper user and its `ScopeGrant` rows, read on every call). It resolves diff --git a/src/extensions.ts b/src/extensions.ts index 05eb9b1..c99c734 100644 --- a/src/extensions.ts +++ b/src/extensions.ts @@ -35,9 +35,10 @@ export interface SearchAnnotator { annotate(input: SearchAnnotateInput): SearchAnnotations | Promise; } +type SearchHit = Record & { id: string }; + const NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; const BUILT_IN_SECTIONS = ['pinned', 'needsReview', 'recent', 'relevant']; -const hitFields = () => [...VIEW_FIELDS, 'score']; const FAILURE_LOG_INTERVAL_MS = 60_000; const lastFailureLog = new Map(); @@ -88,7 +89,10 @@ export function registerBootstrapSection(section: BootstrapSection): () => void */ export function registerSearchAnnotator(annotator: SearchAnnotator): () => void { const target = domain(); - return register((target.searchAnnotators ??= []), annotator, 'search annotator', 'annotate', hitFields()); + return register((target.searchAnnotators ??= []), annotator, 'search annotator', 'annotate', [ + ...VIEW_FIELDS, + 'score', + ]); } function reportFailure(kind: string, name: string, error: unknown): void { @@ -115,8 +119,7 @@ function bootstrapItems(output: unknown): BootstrapItem[] { if (Array.isArray(output)) list = output; else if (typeof (output as any).text === 'string') list = [{ content: (output as any).text }]; else throw new TypeError('render must return { text }, an array of { title?, content }, or nothing'); - // Array.from visits holes, so a sparse array fails here, inside the section's own catch. - return Array.from(list, (item) => { + return list.map((item) => { const { title, content } = item ?? {}; if (typeof content !== 'string' || (title !== undefined && typeof title !== 'string')) { throw new TypeError('a bootstrap item must be { title?: string, content: string }'); @@ -134,8 +137,6 @@ export function renderBootstrapSections( sections.map(async (section) => { const name = section.name; try { - // A copy of another version checked the name against its own list at registration. - if (BUILT_IN_SECTIONS.includes(name)) throw new TypeError(`section name "${name}" is reserved`); return { name, items: bootstrapItems(await section.render(shared)) }; } catch (error) { reportFailure('bootstrap section', name, error); @@ -145,46 +146,21 @@ export function renderBootstrapSections( ); } -function deepFreeze(value: T): T { - if (typeof value === 'object' && value !== null && !Object.isFrozen(value)) { - Object.freeze(value); - for (const child of Object.values(value)) deepFreeze(child); - } - return value; -} - async function runAnnotators( annotators: readonly SearchAnnotator[], - hits: Record[], + hits: SearchHit[], principal: Principal, context: object, ): Promise { - let snapshot: Record[]; - try { - snapshot = deepFreeze(structuredClone(hits)); - } catch (error) { - reportFailure('search annotation', 'snapshot', error); - return; - } - const input = Object.freeze({ principal, hits: snapshot, context }); - const reserved = hitFields(); + // Each annotator sees a frozen copy of the list, so none can add, drop or reorder hits for the response or the others. + const input = Object.freeze({ principal, hits: Object.freeze(hits.slice()), context }); const results = await Promise.all( annotators.map(async (annotator) => { const name = annotator.name; try { - // A copy of another version checked the name against its own hit fields at registration. - if (reserved.includes(name)) throw new TypeError(`annotator name "${name}" is a hit field`); const annotations = await annotator.annotate(input); if (annotations === null || annotations === undefined) return undefined; - // Called through Map.prototype so a Map from another app's realm works, and anything else throws. The JSON - // copy keeps a value the response could not serialize inside this catch. - const values = new Map(); - for (const { id } of hits) { - if (!Map.prototype.has.call(annotations, id)) continue; - const value = Map.prototype.get.call(annotations, id); - if (value !== undefined) values.set(id, JSON.parse(JSON.stringify(value))); - } - return { name, values }; + return { name, values: hits.map((hit) => annotations.get(hit.id)) }; } catch (error) { reportFailure('search annotator', name, error); return undefined; @@ -193,17 +169,13 @@ async function runAnnotators( ); for (const result of results) { if (result === undefined) continue; - for (const hit of hits) { - if (result.values.has(hit.id)) hit[result.name] = result.values.get(hit.id); - } + hits.forEach((hit, index) => { + if (result.values[index] !== undefined) hit[result.name] = result.values[index]; + }); } } -export function annotateSearchHits( - hits: Record[], - principal: Principal, - context: object, -): void | Promise { +export function annotateSearchHits(hits: SearchHit[], principal: Principal, context: object): void | Promise { const annotators = domain().searchAnnotators; if (!annotators?.length || hits.length === 0) return; return runAnnotators(annotators.slice(), hits, principal, context); diff --git a/src/search.ts b/src/search.ts index 9ecfba7..3562da5 100644 --- a/src/search.ts +++ b/src/search.ts @@ -593,8 +593,7 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin memories.push({ ...memory, section }); } }; - // async, so a synchronous throw from a query rejects rather than escaping before Promise.all observes the others. - const loadRecent = async () => + const loadRecent = () => collect( Memory.search( { @@ -606,7 +605,7 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin context, ), ).then((records) => forReader(records.map(toMemoryView), principal, context, access)); - const loadRelevant = async () => + const loadRelevant = () => params.q ? searchMemories( { diff --git a/test/extensions.test.js b/test/extensions.test.js index 4803ddf..730a4e5 100644 --- a/test/extensions.test.js +++ b/test/extensions.test.js @@ -269,11 +269,6 @@ describe('registerBootstrapSection', () => { wrongShape: () => 'just a string', badItem: () => [{ content: 42 }], badTitle: () => [{ title: 7, content: 'x' }], - sparse: () => { - const items = new Array(3); - items[0] = { content: 'first' }; - return items; - }, }; for (const [name, render] of Object.entries(broken)) register(registerBootstrapSection, { name, render }); register(registerBootstrapSection, { name: 'healthy', render: () => [{ content: 'fine' }] }); @@ -335,45 +330,6 @@ describe('registerBootstrapSection', () => { } }); - it('rejects cleanly when every query throws synchronously, with or without a section', async () => { - await setUp(); - const unhandled = []; - const onUnhandled = (reason) => unhandled.push(reason); - process.on('unhandledRejection', onUnhandled); - const Memory = databases.memory.Memory; - const search = Memory.search; - Memory.search = () => { - throw new Error('database closed'); - }; - try { - await assert.rejects(bootstrapMemories({ q: 'tapir', budget: 8000 }, alice, context), /database closed/); - register(registerBootstrapSection, { name: 'orientation', render: () => [] }); - await assert.rejects(bootstrapMemories({ q: 'tapir', budget: 8000 }, alice, context), /database closed/); - await new Promise((resolve) => setTimeout(resolve, 20)); - assert.deepEqual(unhandled, []); - } finally { - Memory.search = search; - process.off('unhandledRejection', onUnhandled); - } - }); - - it("skips a section another version's copy registered under a built-in name", async () => { - const { mine } = await setUp(); - const sections = (domain().bootstrapSections ??= []); - const smuggled = Object.freeze({ name: 'recent', render: () => [{ content: 'not a memory' }] }); - sections.push(smuggled); - try { - const result = await bootstrapMemories({ budget: 8000 }, alice, context); - assert.deepEqual( - result.memories.filter((entry) => entry.section === 'recent').map((entry) => entry.id), - [mine.id], - ); - assert.match(errors[0][0], /"recent"/); - } finally { - sections.splice(sections.indexOf(smuggled), 1); - } - }); - it('warns when an older plugin publishes after a section registered, and once per runtime', async () => { const warnings = []; const originalWarn = logger.warn; @@ -500,8 +456,8 @@ describe('registerSearchAnnotator', () => { const seen = []; register(registerSearchAnnotator, { name: 'summaries', - annotate({ principal, hits, context: annotateContext }) { - seen.push({ principal, hits, annotateContext }); + annotate({ principal, hits }) { + seen.push({ principal, ids: hits.map((hit) => hit.id), frozen: Object.isFrozen(hits) }); return new Map([ [hits[0].id, [{ node: 'n1', size: 37 }]], [hits[1].id, { node: 'n2' }], @@ -530,7 +486,11 @@ describe('registerSearchAnnotator', () => { ); assert.ok(!('summaries' in annotated[2])); assert.equal(seen[0].principal.id, 'alice'); - assert.deepEqual(seen[0].hits, baseline); + assert.deepEqual( + seen[0].ids, + baseline.map((hit) => hit.id), + ); + assert.equal(seen[0].frozen, true); const recalled = await new MemoryMcpResource().mcpRecall({ q: 'heron' }, { user: aliceUser }); assert.deepEqual( @@ -539,22 +499,13 @@ describe('registerSearchAnnotator', () => { ); }); - it('cannot add, remove, reorder or edit hits, even nested fields, and the healthy annotators still apply', async () => { + it('cannot add, remove or reorder hits, and the healthy annotators still apply', async () => { await seedHits(); const baseline = await restSearch('q=heron'); const attempts = { sorts: ({ hits }) => hits.sort(() => -1), pops: ({ hits }) => hits.pop(), pushes: ({ hits }) => hits.push({ id: 'mem-injected-00000' }), - retitles: ({ hits }) => { - hits[0].title = 'changed'; - }, - rescopes: ({ hits }) => { - hits[0].scopes[0] = 'user:bob'; - }, - reattributes: ({ hits }) => { - hits[0].author.principalId = 'bob'; - }, }; for (const [name, annotate] of Object.entries(attempts)) register(registerSearchAnnotator, { name, annotate }); register(registerSearchAnnotator, { name: 'healthy', annotate: ({ hits }) => new Map([[hits[0].id, true]]) }); @@ -571,10 +522,8 @@ describe('registerSearchAnnotator', () => { ); }); - it('skips an annotator that throws, returns a non-Map, or a value the response could not serialize', async () => { + it('skips an annotator that throws, rejects or returns something other than a Map', async () => { await seedHits(); - const cycle = {}; - cycle.self = cycle; const broken = { throws: () => { throw new Error('boom'); @@ -583,19 +532,6 @@ describe('registerSearchAnnotator', () => { throw new Error('async boom'); }, plainObject: ({ hits }) => ({ [hits[0].id]: 1 }), - cyclic: ({ hits }) => new Map([[hits[0].id, cycle]]), - bigint: ({ hits }) => new Map([[hits[0].id, 10n]]), - badToJSON: ({ hits }) => - new Map([ - [ - hits[0].id, - { - toJSON() { - throw new Error('no'); - }, - }, - ], - ]), }; for (const [name, annotate] of Object.entries(broken)) register(registerSearchAnnotator, { name, annotate }); register(registerSearchAnnotator, { name: 'healthy', annotate: ({ hits }) => new Map([[hits[0].id, 'ok']]) }); @@ -613,25 +549,7 @@ describe('registerSearchAnnotator', () => { ); }); - it('answers without annotations when the hits cannot be copied for annotators', async () => { - await seedHits(); - let calls = 0; - register(registerSearchAnnotator, { - name: 'summaries', - annotate: () => { - calls++; - return undefined; - }, - }); - const hits = await searchMemories(parseSearchParams(new URLSearchParams('q=heron')), alice, context); - hits[0].uncloneable = () => {}; - await annotateSearchHits(hits, alice, context); - assert.equal(calls, 0); - assert.ok(hits.every((hit) => !('summaries' in hit))); - assert.match(errors[0][0], /"snapshot"/); - }); - - it("reads a Map from another app's realm and copies its values out", async () => { + it("reads a Map from another app's realm", async () => { const [first] = await seedHits(); const value = { node: 'n1' }; register(registerSearchAnnotator, { @@ -644,8 +562,7 @@ describe('registerSearchAnnotator', () => { }); const hits = await searchMemories(parseSearchParams(new URLSearchParams('q=wading')), alice, context); await annotateSearchHits(hits, alice, context); - assert.deepEqual(hits[0].summaries, { node: 'n1' }); - assert.notEqual(hits[0].summaries, value); + assert.equal(hits[0].summaries, value); }); it('does no work without an annotator, and annotates neither q-less listings nor bootstrap', async () => { @@ -689,23 +606,6 @@ describe('registerSearchAnnotator', () => { assert.equal(hit.notes, 'note'); }); - it("skips an annotator another version's copy registered under a name that is a hit field here", async () => { - await seedHits(); - const annotators = (domain().searchAnnotators ??= []); - const smuggled = Object.freeze({ - name: 'title', - annotate: ({ hits }) => new Map(hits.map((hit) => [hit.id, 'x'])), - }); - annotators.push(smuggled); - try { - const [hit] = await restSearch('q=wading'); - assert.equal(hit.title, 'heron-one'); - assert.match(errors[0][0], /"title"/); - } finally { - annotators.splice(annotators.indexOf(smuggled), 1); - } - }); - it('rejects a name a hit already carries, a taken name, or no annotate function', () => { for (const name of ['title', 'content', 'scopes', 'author', 'score', 'updatedAt']) { assert.throws(() => registerSearchAnnotator({ name, annotate: () => undefined }), /reserved/, name); From b41116dbef8eb62dc3d5b0fd29072f1fabf70343 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 12:04:21 -0700 Subject: [PATCH 7/8] Start the recent load first, so a closing database cannot strand another bootstrap load's rejection Co-Authored-By: Claude Opus 5.5 Dispatch-Task: agent-memory-summary-extension-points --- src/extensions.ts | 3 ++- src/search.ts | 12 ++++++++---- test/extensions.test.js | 28 +++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/extensions.ts b/src/extensions.ts index c99c734..aa633a6 100644 --- a/src/extensions.ts +++ b/src/extensions.ts @@ -84,7 +84,8 @@ export function registerBootstrapSection(section: BootstrapSection): () => void /** * Attaches per-hit data to hybrid search results (REST `q` searches and MCP `recall` with `q`) under the - * annotator's name. It cannot add, remove, reorder or change hits; a throwing annotator is skipped and logged. + * annotator's name. It cannot add, remove or reorder hits, and must not change them; a throwing annotator is skipped + * and logged. * Returns an unregister function. Contract: DESIGN.md, Extension points. */ export function registerSearchAnnotator(annotator: SearchAnnotator): () => void { diff --git a/src/search.ts b/src/search.ts index 3562da5..972b98d 100644 --- a/src/search.ts +++ b/src/search.ts @@ -625,18 +625,22 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin const sections = registeredBootstrapSections(); let recent: MemoryView[]; let relevant: MemoryView[]; + // loadRecent goes first: it is the only load that can throw synchronously (a closing database), and then nothing + // else has started whose rejection would go unobserved. if (sections.length === 0) { let pinned: MemoryView[]; let needsReview: MemoryView[]; - [pinned, needsReview, recent, relevant] = await Promise.all([ - loadPinned(), - loadNeedsReview(), + [recent, relevant, pinned, needsReview] = await Promise.all([ loadRecent(), loadRelevant(), + loadPinned(), + loadNeedsReview(), ]); pack('pinned', pinned); pack('needsReview', needsReview); } else { + const recentLoad = loadRecent(); + const relevantLoad = loadRelevant(); // Sections render once pinned and needsReview have packed, so they know the budget left, while recent and // relevant still load. Awaiting the head on its own would leave a recent or relevant rejection unobserved. const head = Promise.all([loadPinned(), loadNeedsReview()]).then(async ([pinned, needsReview]) => { @@ -656,7 +660,7 @@ export async function bootstrapMemories(params: BootstrapParams, principal: Prin } } }); - [, recent, relevant] = await Promise.all([head, loadRecent(), loadRelevant()]); + [, recent, relevant] = await Promise.all([head, recentLoad, relevantLoad]); } pack('recent', recent); pack('relevant', relevant); diff --git a/test/extensions.test.js b/test/extensions.test.js index 730a4e5..f6836fb 100644 --- a/test/extensions.test.js +++ b/test/extensions.test.js @@ -307,7 +307,11 @@ describe('registerBootstrapSection', () => { const Memory = databases.memory.Memory; const search = Memory.search; Memory.search = function (query, ctx) { - if (query.sort?.attribute === 'updatedAt' && query.limit === 50) throw new Error('recent load failed'); + if (query.sort?.attribute === 'updatedAt' && query.limit === 50) { + return (async function* () { + throw new Error('recent load failed'); + })(); + } return search.call(this, query, ctx); }; let rendered = false; @@ -330,6 +334,28 @@ describe('registerBootstrapSection', () => { } }); + it('rejects cleanly when every query throws synchronously, with or without a section', async () => { + await setUp(); + const unhandled = []; + const onUnhandled = (reason) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + const Memory = databases.memory.Memory; + const search = Memory.search; + Memory.search = () => { + throw new Error('database closed'); + }; + try { + await assert.rejects(bootstrapMemories({ q: 'tapir', budget: 8000 }, alice, context), /database closed/); + register(registerBootstrapSection, { name: 'orientation', render: () => [] }); + await assert.rejects(bootstrapMemories({ q: 'tapir', budget: 8000 }, alice, context), /database closed/); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.deepEqual(unhandled, []); + } finally { + Memory.search = search; + process.off('unhandledRejection', onUnhandled); + } + }); + it('warns when an older plugin publishes after a section registered, and once per runtime', async () => { const warnings = []; const originalWarn = logger.warn; From 23dc59d7363d9b8b1c5a9ec805585e5c839203c4 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 12:04:34 -0700 Subject: [PATCH 8/8] Point the scan's doc comment at the exported contract Co-Authored-By: Claude Opus 5.5 Dispatch-Task: agent-memory-summary-extension-points --- src/search.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/search.ts b/src/search.ts index 972b98d..c05857a 100644 --- a/src/search.ts +++ b/src/search.ts @@ -181,10 +181,7 @@ export interface ScanOptions { includeInactive?: boolean; } -/** - * Every stored memory row in `createdAt`, then id, order, for trusted in-process builders: no principal, no scope - * conditions. Paged, so a slow consumer never holds a read open across the walk; a walk, not a snapshot. - */ +// Paged, so a consumer that pauses between rows holds no read open. Contract: `scanMemories` in src/index.ts. export async function* scanMemories(options: ScanOptions = {}): AsyncGenerator> { let after: CreatedPosition | undefined; if (options.after !== undefined) {