diff --git a/.github/scripts/close-issues-on-dev.sh b/.github/scripts/close-issues-on-dev.sh new file mode 100755 index 000000000..cfd511a13 --- /dev/null +++ b/.github/scripts/close-issues-on-dev.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# close-issues-on-dev.sh — close the issues named by pull requests that merged into a branch other than +# dev, once their merge commit is part of dev (see .github/workflows/close-issues-on-dev.yml). +# +# DRY_RUN=1 .github/scripts/close-issues-on-dev.sh # list, close nothing +# +# Needs: gh (authenticated; GH_TOKEN in CI), a checkout with full history, and origin/dev or HEAD on dev. +# LOOKBACK: how many recently merged pull requests to consider (default 300). +set -euo pipefail + +DRY_RUN="${DRY_RUN:-0}" +LOOKBACK="${LOOKBACK:-300}" +REPO="${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner -q .nameWithOwner)}" +DEV_REF="${DEV_REF:-HEAD}" +[ "$DEV_REF" = HEAD ] && [ "$(git rev-parse --abbrev-ref HEAD)" != dev ] && DEV_REF=origin/dev + +closed=0; pending=0 +while IFS=$'\t' read -r pr base merge body64; do + [ -n "$merge" ] || continue + [ "$base" = dev ] && continue # GitHub closes these itself + if ! git merge-base --is-ancestor "$merge" "$DEV_REF" 2>/dev/null; then + pending=$((pending + 1)); continue # merged, but not on dev yet + fi + body=$(printf '%s' "$body64" | base64 --decode) + for n in $(printf '%s\n' "$body" | grep -oiE '\b(close[sd]?|fix(e[sd])?|resolve[sd]?)[[:space:]]*:?[[:space:]]+#[0-9]+' \ + | grep -oE '[0-9]+$' | sort -un); do + state=$(gh issue view "$n" --repo "$REPO" --json state -q .state 2>/dev/null || echo MISSING) + [ "$state" = OPEN ] || continue + if [ "$DRY_RUN" = 1 ]; then + echo "would close #$n (pull request #$pr into $base, ${merge:0:9} is on dev)" + else + gh issue close "$n" --repo "$REPO" \ + --comment "Closed by #$pr, which merged into \`$base\` and reached \`dev\` in ${merge:0:9}." + echo "closed #$n (pull request #$pr into $base)" + fi + closed=$((closed + 1)) + done +done < <(gh pr list --repo "$REPO" --state merged --limit "$LOOKBACK" \ + --json number,baseRefName,mergeCommit,body \ + -q '.[] | [.number, .baseRefName, (.mergeCommit.oid // ""), (.body // "" | @base64)] | @tsv') + +verb=closed; [ "$DRY_RUN" = 1 ] && verb="to close" +echo "issues $verb: $closed; merged pull requests not on dev yet: $pending" diff --git a/.github/workflows/close-issues-on-dev.yml b/.github/workflows/close-issues-on-dev.yml new file mode 100644 index 000000000..e491bed82 --- /dev/null +++ b/.github/workflows/close-issues-on-dev.yml @@ -0,0 +1,44 @@ +# ───────────────────────────────────────────────────────────────────────────── +# A pull request that merges into a release or integration branch (0.2.1, a +# perf branch) never closes the issues it names: GitHub acts on "Closes #N" only +# when the merge lands on the default branch. This closes them when that work +# reaches `dev`. +# +# On every push to dev it looks at pull requests merged into any other branch, +# keeps those whose merge commit is now part of dev, and closes each open issue +# their description closes ("Closes", "Fixes", "Resolves" #N), with a comment +# naming the pull request. "Part of #N" and "Refs #N" are not closing words, so +# umbrella issues stay open. +# ───────────────────────────────────────────────────────────────────────────── +name: close-issues-on-dev + +on: + push: + branches: [dev] + workflow_dispatch: + inputs: + dry_run: + description: list what would be closed, close nothing + type: boolean + default: true + +permissions: + contents: read + issues: write + pull-requests: read + +jobs: + close: + name: close issues whose pull request reached dev + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: close the issues + env: + GH_TOKEN: ${{ github.token }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run && '1' || '0' }} + run: .github/scripts/close-issues-on-dev.sh diff --git a/README.md b/README.md index ba0d1d453..b68e651f1 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

AxiomCode Graph

- A god's-eye view of your codebase for AI agents. Stop grepping. Ensure correctness and completeness for any task your AI agent performs. + Know what each change breaks in your codebase. Stop grepping. Ensure correctness and completeness for any task your AI agent performs. Ship less buggy code.

@@ -167,6 +167,9 @@ Any other agent that speaks MCP takes one entry in its MCP config; see [Support for agents](#support-for-agents). Start a new agent session afterward: plugins are loaded at startup. Add `.axiomcode/` to your `.gitignore`; the graph is built there on first use. +After upgrading the engine or the plugin, **restart the agent session**: a session started before the upgrade keeps the +old version's MCP server and hooks running beside the new ones, and the two rebuild the graph in turn. + ### Uninstallation ```bash diff --git a/bin/axiomcode b/bin/axiomcode index 71ae8b8cf..b5225e774 100755 --- a/bin/axiomcode +++ b/bin/axiomcode @@ -412,7 +412,7 @@ case "$cmd" in case "$which" in java|typescript|python|javascript|csharp) run_suite "$which" "$@";; web) echo "══ web: cli smoke + violation checks + include checks"; bash "$ROOT/graph/test/web/cli-smoke.sh" || rc=1; bash "$ROOT/graph/test/web/violations.sh" || rc=1; bash "$ROOT/graph/test/web/includes.sh" || rc=1; run_suite web "$@";; - parser) for t in java-extractor-tests typescript-tests python-tests javascript-tests csharp-tests gradle-tests services-tests web-tests web-torture web-plumbing-tests discovery-tests; do + parser) for t in java-extractor-tests typescript-tests deno-tests python-tests javascript-tests csharp-tests gradle-tests services-tests web-tests web-torture web-plumbing-tests discovery-tests; do [ -f "$ROOT/parser/src/test/$t.ts" ] || continue; echo "══ parser: $t"; (cd "$ROOT/parser" && npx tsx "src/test/$t.ts") || rc=1; done;; all) for l in java typescript python javascript csharp web; do [ -f "$ROOT/graph/test/$l/run-tests.sh" ] && run_suite "$l" "$@"; done; bash "$ROOT/graph/test/web/cli-smoke.sh" || rc=1; "$0" test parser || rc=1;; *) die "test: unknown suite $which (java | typescript | python | javascript | csharp | web | parser | all)";; diff --git a/gemini-extension.json b/gemini-extension.json index 4a44f6eae..dad925927 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.2.0", + "version": "0.2.1", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", "contextFileName": "plugins/axiomcode/AGENTS.md", "mcpServers": { diff --git a/graph/bundle/SCHEMA.md b/graph/bundle/SCHEMA.md index 6b33b221c..6cd2c4bd2 100644 --- a/graph/bundle/SCHEMA.md +++ b/graph/bundle/SCHEMA.md @@ -442,7 +442,7 @@ One row per place a call is written (or, for a synthesised edge, the construct t - **typescript** — end_line / end_column come from the expression row; the call-site row itself records only the start. - **javascript** — caller_id is the parser's enclosing method, or the module initializer for top-level code. end_line / end_column come from the expression row. `require()` is a module edge, not a call site. - **typescript** — PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor's name is on the callee's methods row. Filter them out with kind NOT IN (…) when counting calls. -- **python** — PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. +- **python** — PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METHOD_REFERENCE, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. - **python** — The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class's TYPE hash for METACLASS_CREATION, positioned at the class declaration. ### `call_edges` @@ -541,6 +541,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | `SUBSCRIPT_PROTOCOL` | python | `x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver's class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call. | | `BUILTIN_PROTOCOL` | python | `repr(x)`, `str(x)`, `len(x)`, `hash(x)`, `bool(x)`, `iter(x)`, `next(x)`, `abs(x)`, `format(x)` and `reversed(x)` run the matching dunder of the argument's class (str falls back to `__repr__`, bool to `__len__`). The written call is to the builtin; this edge is the dunder it runs. The site is the argument expression. Its own kind so it is never counted as a written call. | | `OPERATOR_PROTOCOL` | python | An operator runs a dunder of its operands: `a + b` the left operand's `__add__` and the right one's `__radd__`, a comparison the left slot and the right one's mirror (`a < b` -> `b.__gt__`, `!=` falls back to `__eq__`), `x in c` the right operand's `__contains__` (or `__iter__`), `-a`/`+a`/`~a` its `__neg__`/`__pos__`/`__invert__`. No written call; the site is the operator expression. | +| `METHOD_REFERENCE` | python | A method or function READ AS A VALUE, not called: an argument (`pool.submit(self.work, x)`, `map(Cls.convert, rows)`, `partial(svc.cancel, oid)`), a keyword (`Thread(target=svc.run)`), an element, a stored or returned value. The target is what the expression resolves to (a typed receiver through the MRO, a class, a scoped name, a module member); an untyped receiver gives no row. Always tier callback_registered; the site is the reference expression (call-edge-generation/method-reference.dl). | | `TRUTH_PROTOCOL` | python | A truth test runs the tested value's `__bool__`, or `__len__` when its class has none: `if x:`, `while x:`, `assert x`, `not x`, a conditional or comprehension condition, and an operand of `and`/`or` (the right one only when the whole expression is tested). No written call; the site is the tested expression. | **`call_edges.tier` values** @@ -564,6 +565,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | `dynamic_terminal` | javascript | `obj[expr]()`, `eval`, `import()`: no static target by construction; callee is NULL. | | `fan_capped` | javascript, java, csharp | More targets than --dispatch-cap: the set was refused rather than emitted. JavaScript: callee is NULL. Java and C#: callee is the declared base method the fan would have started from; dispatch-capped-sites.csv carries the refused count. | | `callback_registered` | javascript, typescript | The site HANDS the callee this function (`xs.forEach(f)`, `p.then(f)`, `emitter.on('x', h)`, `setTimeout(f)`), which may invoke it. Not the site's own callee; a reachability edge, labelled so it is never read as a resolved call. | +| `callback_registered` | python | A method or function named as a VALUE and handed over, stored or returned (`pool.submit(self.work)`, `Thread(target=svc.run)`, `sorted(xs, key=self.rank)`, `[h.on_event]`), always kind METHOD_REFERENCE. Its target is resolved the way a call on the same receiver would be; whoever receives the value calls it. A reachability edge, labelled so it is never read as a resolved call. | | `event_dispatch` | javascript | `x.emit('name')` reaching a handler registered by `x.on('name', h)` on a value x may hold — name-sensitive for literal names, every handler on that value for a computed one. | | `event_dispatch` | typescript | A publish on a project class's keyed registry reaching each function filed under the same key: `bus.publish('x', p)` where `publish` (or a method it passes the key to) calls what `this.handlers.get(name)` holds, to the function a `bus.subscribe('x', f)` stored there, or an entry `{ ['x']: f }` of a table a subscribe-all method files by key. A literal, a const or a const-object member key; a key only known at run time matches every key, and a constant the dispatch also looks up (a wildcard) matches every publish. The site is the publish call (call-edge-generation/keyed_registry.dl). | | `event_dispatch` | java | A Spring application event: `publishEvent(e)` reaching each listener (`@EventListener`, `@TransactionalEventListener`, `ApplicationListener.onApplicationEvent`) whose declared event type e's static type is, or is a subtype of. Added beside the publishEvent boundary row, never in place of it (call-edge-generation/event_dispatch.dl). Also a JPA entity write (`save`, `persist`, `merge`, `delete` on a repository or EntityManager) reaching the `@PrePersist`/`@PreUpdate`/`@PreRemove`-style callbacks of the written entity's type and of the listeners `@EntityListeners` names on it or a superclass (kind entity_callback; call-edge-generation/entity_lifecycle.dl). | @@ -586,7 +588,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h - **python** — A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row. - **java** — A `boundary_lib` edge with callee_provenance external names a method of an ancestor type no staged IR declares (callee_label `external:.`, no methods row). A site whose receiver is declared as such a type is multi_inferred even with one client override: the platform method itself, and the platform's own subclasses, are the other possible targets. Stage the library to replace the label with the real method. - **python** — The reason a site is ambiguous_unknown is exported per site in ext_call_site_unresolved (site, caller, reason, detail). -- **all** — THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge and multi_inferred in every language, and boundary_lib where the library is staged (--library) — without it boundary_lib names the target in callee_label and leaves callee_method_id NULL; ALSO ambient_terminal in TypeScript, fan_capped in Java and C# (the declared base, the fan refused), and runtime_observed in C#. HANDED OVER (callee set, but the site passes the function rather than calling it): callback_registered and event_dispatch in JavaScript and TypeScript; event_dispatch in C# too, where a mediator Send or Publish runs the handler for the request type, beside the row for the site itself. CORRECT END (callee NULL, and nothing is missing): intrinsic_terminal in TypeScript; ambient_terminal, implicit_constructor and dynamic_terminal in JavaScript; known_implicit_ctor, known_builtin_operator and boundary_generated (callee_label set) in C#. BLIND SPOT (callee NULL; exactly the tiers named `ambiguous_*`, which are what unresolved_sites holds): ambiguous_unknown everywhere, ALSO ambiguous_anon in Java and ambiguous_dynamic in C#. CAPPED (callee NULL, not in unresolved_sites): fan_capped in JavaScript. Python emits only the four shared tiers. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in every language but Python — derive the set from this note or from unresolved_sites, never from a hardcoded list. +- **all** — THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge and multi_inferred in every language, and boundary_lib where the library is staged (--library) — without it boundary_lib names the target in callee_label and leaves callee_method_id NULL; ALSO ambient_terminal in TypeScript, fan_capped in Java and C# (the declared base, the fan refused), and runtime_observed in C#. HANDED OVER (callee set, but the site passes the function rather than calling it): callback_registered and event_dispatch in JavaScript and TypeScript, callback_registered (kind METHOD_REFERENCE) in Python; event_dispatch in C# too, where a mediator Send or Publish runs the handler for the request type, beside the row for the site itself. CORRECT END (callee NULL, and nothing is missing): intrinsic_terminal in TypeScript; ambient_terminal, implicit_constructor and dynamic_terminal in JavaScript; known_implicit_ctor, known_builtin_operator and boundary_generated (callee_label set) in C#. BLIND SPOT (callee NULL; exactly the tiers named `ambiguous_*`, which are what unresolved_sites holds): ambiguous_unknown everywhere, ALSO ambiguous_anon in Java and ambiguous_dynamic in C#. CAPPED (callee NULL, not in unresolved_sites): fan_capped in JavaScript. Python emits the four shared tiers and callback_registered. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in every language but Python — derive the set from this note or from unresolved_sites, never from a hardcoded list. - **java** — A multi_inferred fan is CHA-wide: it is every override the hierarchy admits, bounded only by the dispatch cap. type_instantiated is computed and exported but NOT read by any rule, so the fan is not narrowed to types the program constructs. Narrow it yourself by joining dispatch_candidates to type_instantiated — see the dispatch_envelope_of query. A receiver is ALSO typed by what flows into it (a local's initializer, the arguments callers pass to a parameter, the receivers callers invoke a method on for its `this`), and each flow-in type resolves its member directly, outside the fan: that is why a `fan_capped` site still carries edges, and why they are the types the program was seen to hand over, not the whole hierarchy. - **typescript** — A multi_inferred fan is CHA-wide, as in Java: type_instantiated is computed and exported but NOT read by any rule. The fan also has sources that are not virtual dispatch at all — an overload set or a union-typed receiver produces one too. - **python** — A multi_inferred fan IS narrowed by the instantiation set: type_instantiated_reachable (the constructed classes and their bases) bounds dispatch in resolution/dispatch.dl. Python is the only front end where that narrowing is applied, so a fan here is tighter than the same shape would be in Java or TypeScript. diff --git a/graph/bundle/build.ts b/graph/bundle/build.ts index 84c9fcb9e..d066ead75 100644 --- a/graph/bundle/build.ts +++ b/graph/bundle/build.ts @@ -10,9 +10,19 @@ import * as fs from 'fs'; import * as path from 'path'; -import { Header, readHeader, readRaw, readRfc4180 } from '@/bundle/csv'; +import { Header, type Pick, readHeader, readRaw, readRfc4180 } from '@/bundle/csv'; import type { LanguageAdapter, RawSource } from '@/bundle/languages'; +/** AXIOM_BUNDLE_PROFILE=1: one stderr line per stage with its wall time, for measuring the bundle. */ +const PROFILE = !!process.env.AXIOM_BUNDLE_PROFILE; +let lapAt = Date.now(); +export function lap(what: string): void { + if (!PROFILE) return; + const now = Date.now(); + process.stderr.write(` [bundle-profile] ${((now - lapAt) / 1000).toFixed(2)}s ${what}\n`); + lapAt = now; +} + export type Cell = string | number | null; export type Row = Cell[]; @@ -114,9 +124,10 @@ async function libSource(inp: BuildInputs, file: string): Promise { +async function* rowsOf(src: EntitySource, pick?: Pick): AsyncGenerator { let first = src.hasHeaderRow; - for await (const r of readRfc4180(src.path)) { + // the reader never filters a file's first row: here that is the header, and a headerless file is read whole + for await (const r of readRfc4180(src.path, src.hasHeaderRow ? pick : undefined)) { if (first) { first = false; continue; } yield r; } @@ -136,6 +147,7 @@ export async function buildCore(inp: BuildInputs): Promise { const rawFieldAccess = A.raw.fieldAccess ? await readSource(rawDir, A.raw.fieldAccess) : []; const rawTypeUse = A.raw.typeUse ? await readSource(rawDir, A.raw.typeUse) : []; const rawConfigBinding = A.raw.configBinding ? await readSource(rawDir, A.raw.configBinding) : []; + lap('read raw relations'); log(` raw: ${rawEdges.length} edge rows, ${rawAncestors.length} ancestor rows, ${rawOverrides.length} override rows`); // call_edges: split the raw ToMethod into a method key or a label by provenance @@ -179,6 +191,7 @@ export async function buildCore(inp: BuildInputs): Promise { return [ref!, nul(type), context!, int(depth) ?? 0, ownerKind!, owner!, nul(enclMethod), nul(enclType), nul(prov), tier!]; }); + lap('transform raw rows (call_edges, field_access, type_use)'); // ── 2. which ids the bundle must name ───────────────────────────────────── const wantMethods = new Set(); const wantTypes = new Set(); @@ -211,6 +224,7 @@ export async function buildCore(inp: BuildInputs): Promise { if (r[2] === 'field' && r[3] !== null) wantFields.add(r[3] as string); } + lap('collect wanted ids'); // ── 3. client entities (all of them — the client is the subject) ────────── // The IR spells visibility differently per entity file (PUBLIC on methods/types, PUBLIC_ACCESS on // fields); normalise to one vocabulary so a rule can match on it. '' when the language has none. @@ -367,6 +381,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } + lap('read client modules/methods/types/fields IR'); // ── 4. library entities — only the referenced ones ──────────────────────── const missingMethods = new Set([...wantMethods].filter((id) => !methods.has(id))); const missingTypesEarly = [...wantTypes].some((id) => !types.has(id)); @@ -447,6 +462,7 @@ export async function buildCore(inp: BuildInputs): Promise { if (FI?.ownerQualifiedNameFromType) fillOwnerNames(fields, 3, 4); // A library module's file path is not staged; a library site never occurs (edges start in the client). + lap('read library entities + generated/external'); // ── 5. call sites: name + position for every site the edges mention ─────── // row: id, caller_id, kind, callee_name, file_path, start_line, start_column, end_line, end_column const sites = new Map(); @@ -466,7 +482,7 @@ export async function buildCore(inp: BuildInputs): Promise { const h = src.header; const ce = h.col(C.expressionId), cn = h.col(C.calleeName), cl = h.col(C.startLine), cc = h.col(C.startColumn); const cel = C.endLine ? h.col(C.endLine) : -1; - for await (const r of rowsOf(src)) { + for await (const r of rowsOf(src, { column: ce, has: (id) => sites.has(id) })) { const row = sites.get(r[ce] ?? ''); if (!row) continue; fill(row, 3, nul(r[cn])); fill(row, 4, fileOf(C.fileVia, h, r)); @@ -475,6 +491,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } } + lap('position sites from call-site IR'); { const E = A.ir.expressions; const src = await clientSource(inp.clientIrDir, E.file); @@ -483,7 +500,7 @@ export async function buildCore(inp: BuildInputs): Promise { const ci = h.col(E.id), ck = h.col(E.kind), cl = h.col(E.startLine), cc = h.col(E.startColumn), cel = h.col(E.endLine), cec = h.col(E.endColumn); const cname = E.calleeName ? h.col(E.calleeName.column) : -1; const nameKinds = new Set(E.calleeName?.kinds ?? []); - for await (const r of rowsOf(src)) { + for await (const r of rowsOf(src, { column: ci, has: (id) => sites.has(id) || fieldSites.has(id) })) { const id = r[ci] ?? ''; // A FIELD ACCESS site is an expression too, and this is the one pass over a table that // can be GB-scale, so both are positioned here rather than streaming it twice. @@ -501,6 +518,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } } + lap('position sites from expressions IR'); if (A.ir.decorators) { const D = A.ir.decorators; const src = await clientSource(inp.clientIrDir, D.file); @@ -572,6 +590,7 @@ export async function buildCore(inp: BuildInputs): Promise { const unplaced = [...sites.values()].filter((r) => r[5] === null).length; if (unplaced > 0) log(` ! ${unplaced} of ${sites.size} call sites have no position in the IR`); + lap('position sites from decorators/locals/field sites + fallbacks'); // ── 5b. what the parser did NOT read ────────────────────────────────────── // A file the parser skipped has no row in any table above, and until #1148 nothing in the // bundle said so: a consumer could not tell a file with no calls from a file that was @@ -599,9 +618,10 @@ export async function buildCore(inp: BuildInputs): Promise { } if (skipped.length > 0) log(` skipped: ${skipped.length} file(s) the parser did not read`); + lap('read skipped'); // ── 6. assemble ─────────────────────────────────────────────────────────── const run: Row[] = Object.entries(inp.meta).map(([k, v]) => [k, v]); - return { + const core = { run, methods: [...methods.values()], types: [...types.values()], @@ -619,6 +639,8 @@ export async function buildCore(inp: BuildInputs): Promise { type_use, skipped: dedupe(skipped), }; + lap('assemble + dedupe'); + return core; } function dedupe(rows: Row[]): Row[] { diff --git a/graph/bundle/catalog.ts b/graph/bundle/catalog.ts index 3276e855b..9e216e7d6 100644 --- a/graph/bundle/catalog.ts +++ b/graph/bundle/catalog.ts @@ -16,6 +16,21 @@ export interface ExtRelation { description: string; } +/** + * The ext relations the query layer reads (plugins/, hooks/: impact, path, context, test-impact, the MCP server). + * Only these get the c0 index: an index on a relation nothing queries is time and disk at every build, and the + * schema promises the table, not an index on it. Not a filter on what is loaded — every manifest relation is still + * an ext table. graph/test/tools/ext-queried-test.sh fails when a script names an ext relation missing here. + */ +export const EXT_QUERIED: ReadonlySet = new Set([ + 'bean_def', 'call_runs_edge', 'call_site_unresolved', 'config_affects_method', 'config_binding', 'config_key_ref', + 'config_unresolved', 'ctor_implicit_type', 'decorated_name_target', 'di_edge', 'fn_value_call', 'framework_edge', + 'inject_point', 'lib_callback_edge', 'lib_callback_site', 'library_receiver', 'method_returns_method', + 'method_returns_param', 'persistence_field', 'remote_edge', 'remote_unsent', 'remote_unserved', 'return_value', + 'state_call_alloc', 'state_call_open', 'state_gate', 'state_gate_alloc', 'state_world_of_gated', + 'type_base_unresolved', 'unresolved_value_callee', +]); + /** relationfile per line; comments and blanks skipped. */ export function readManifest(langDir: string): { relation: string; file: string }[] { const p = path.join(langDir, 'souffle', 'export_manifest.tsv'); diff --git a/graph/bundle/cli.ts b/graph/bundle/cli.ts index c66c43611..fcefeba3a 100644 --- a/graph/bundle/cli.ts +++ b/graph/bundle/cli.ts @@ -17,7 +17,7 @@ import * as path from 'path'; process.removeAllListeners('warning'); process.on('warning', (w) => { if (w.name !== 'ExperimentalWarning') console.error(w.stack ?? w.message); }); -import { buildCore } from '@/bundle/build'; +import { buildCore, lap } from '@/bundle/build'; import { catalogExt, readLibMap } from '@/bundle/catalog'; import { adapterFor } from '@/bundle/languages'; import { renderSchemaMarkdown, SCHEMA_VERSION } from '@/bundle/schema'; @@ -95,7 +95,9 @@ export async function main(argv = process.argv.slice(2)): Promise { const dbPath = path.join(a.out!, 'graph.sqlite'); if (haveSqlite) { - await writeSqlite({ dbPath, language: adapter.language, core, ext: catalogExt(langDir), rawDir: a.raw!, log }); + const ext = catalogExt(langDir); + lap('lift ext docs (catalogExt)'); + await writeSqlite({ dbPath, language: adapter.language, core, ext, rawDir: a.raw!, log }); log(`▶ wrote ${dbPath}${a.debug ? ' (+ csv/*.csv, --debug)' : ''}`); } else { fs.rmSync(dbPath, { force: true }); diff --git a/graph/bundle/csv.ts b/graph/bundle/csv.ts index 43fc7ef97..befee56b5 100644 --- a/graph/bundle/csv.ts +++ b/graph/bundle/csv.ts @@ -14,43 +14,120 @@ import * as fs from 'fs'; const TAB = '\t'; -/** One RFC 4180 (tab-delimited) file as an async stream of rows. A zero-byte file yields nothing. */ -export async function* readRfc4180(path: string): AsyncGenerator { +/** One RFC 4180 (tab-delimited) file as an async stream of rows. A zero-byte file yields nothing. + * + * A LINE WITH NO QUOTE IN IT IS A WHOLE ROW, and nearly every line is one, so it is split on tabs as it + * stands: a quote is the only thing that can make a field hold a tab or a newline. Only a line that holds + * a quote, and the lines a quoted field carries it across, goes through the character-by-character reader + * (`Rfc4180Row`), which is the definition. Reading every character that way cost most of the bundle's time + * on a GB-scale expression table. */ +export async function* readRfc4180(path: string, pick?: Pick): AsyncGenerator { const stream = fs.createReadStream(path, { encoding: 'utf8', highWaterMark: 1 << 20 }); - let field = ''; - let row: string[] = []; - let quoted = false; // inside a quoted field - let afterQuote = false; // just saw a quote inside a quoted field — next char decides - let atFieldStart = true; + const slow = new Rfc4180Row(); + let rest = ''; + let first = true; // the first row, a header in a headered file, is never filtered for await (const chunk of stream as AsyncIterable) { - for (let i = 0; i < chunk.length; i++) { + const text = rest.length > 0 ? rest + chunk : chunk; + let at = 0; + let quote = text.indexOf('"'); // the next quote at or after `at`, -1 when the chunk holds no more + for (;;) { + const nl = text.indexOf('\n', at); + if (nl < 0) break; + if (quote >= 0 && quote < at) quote = text.indexOf('"', at); + if (slow.open || (quote >= 0 && quote < nl)) { + // the line holds a quote, or a quoted field opened on an earlier line runs on into it + const row = slow.feed(text, at, nl + 1); + if (row) yield row; + } else if (first || !pick || pick.has(fieldAt(text, at, nl, pick.column))) { + yield splitLine(text, at, nl); + } + first = false; + at = nl + 1; + } + rest = text.slice(at); + } + // final row without a trailing newline + if (slow.open || rest.indexOf('"') >= 0) { + slow.feed(rest, 0, rest.length); + const row = slow.end(); + if (row) yield row; + } else if (rest.length > 0 && (first || !pick || pick.has(fieldAt(rest, 0, rest.length, pick.column)))) { + yield splitLine(rest, 0, rest.length); + } +} + +/** Rows the caller will drop, skipped before they are split: a row is read only when `has` holds for its + * `column`th field ('' when the row is shorter, as `row[column] ?? ''` reads it). A row a quote touches is + * always returned, so the caller must still test every row it gets; this only saves the split of the rest. */ +export interface Pick { column: number; has: (value: string) => boolean } + +/** The `col`th tab-separated field of text[a, b), a line holding no quote; a trailing \r on the last field dropped. */ +function fieldAt(text: string, a: number, b: number, col: number): string { + let start = a; + for (let i = 0; i < col; i++) { + const t = text.indexOf(TAB, start); + if (t < 0 || t >= b) return ''; + start = t + 1; + } + const t = text.indexOf(TAB, start); + if (t >= 0 && t < b) return text.slice(start, t); + const v = text.slice(start, b); + return v.endsWith('\r') ? v.slice(0, -1) : v; +} + +/** text[a, b) — one line holding no quote, without its newline — as a row: split on tabs, a trailing \r dropped. */ +function splitLine(text: string, a: number, b: number): string[] { + const line = text.slice(a, b); + const row = line.split(TAB); + const last = row[row.length - 1]!; + if (last.endsWith('\r')) row[row.length - 1] = last.slice(0, -1); + return row; +} + +/** The RFC 4180 reader, a character at a time: fed whole lines, it returns a row when one ends on a newline outside quotes. */ +class Rfc4180Row { + private field = ''; + private row: string[] = []; + private quoted = false; // inside a quoted field + private afterQuote = false; // just saw a quote inside a quoted field — next char decides + private atFieldStart = true; + /** a row has begun and not ended: a quoted field crossed a newline */ + get open(): boolean { return this.quoted || this.afterQuote || this.row.length > 0 || this.field.length > 0 || !this.atFieldStart; } + feed(chunk: string, from: number, to: number): string[] | null { + let out: string[] | null = null; + for (let i = from; i < to; i++) { const ch = chunk[i]!; - if (quoted) { - if (afterQuote) { - afterQuote = false; - if (ch === '"') { field += '"'; continue; } // doubled quote → literal quote - quoted = false; // the closing quote — fall through to unquoted handling + if (this.quoted) { + if (this.afterQuote) { + this.afterQuote = false; + if (ch === '"') { this.field += '"'; continue; } // doubled quote → literal quote + this.quoted = false; // the closing quote — fall through to unquoted handling } else { - if (ch === '"') afterQuote = true; else field += ch; + if (ch === '"') this.afterQuote = true; else this.field += ch; continue; } } - if (ch === '"' && atFieldStart) { quoted = true; atFieldStart = false; continue; } - atFieldStart = false; - if (ch === TAB) { row.push(field); field = ''; atFieldStart = true; continue; } + if (ch === '"' && this.atFieldStart) { this.quoted = true; this.atFieldStart = false; continue; } + this.atFieldStart = false; + if (ch === TAB) { this.row.push(this.field); this.field = ''; this.atFieldStart = true; continue; } if (ch === '\n') { - row.push(field.endsWith('\r') ? field.slice(0, -1) : field); - field = ''; atFieldStart = true; - yield row; row = []; + this.row.push(this.field.endsWith('\r') ? this.field.slice(0, -1) : this.field); + this.field = ''; this.atFieldStart = true; + out = this.row; this.row = []; continue; } - field += ch; + this.field += ch; } + return out; } - // final row without a trailing newline - if (field.length > 0 || row.length > 0) { - row.push(field.endsWith('\r') ? field.slice(0, -1) : field); - yield row; + /** end of input: the row in progress, if any */ + end(): string[] | null { + if (this.field.length > 0 || this.row.length > 0) { + this.row.push(this.field.endsWith('\r') ? this.field.slice(0, -1) : this.field); + const r = this.row; this.row = []; this.field = ''; + return r; + } + return null; } } diff --git a/graph/bundle/schema.ts b/graph/bundle/schema.ts index 731fb61b6..9d71c3906 100644 --- a/graph/bundle/schema.ts +++ b/graph/bundle/schema.ts @@ -524,6 +524,7 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'call_edges', column: 'tier', value: 'dynamic_terminal', languages: S, meaning: '`obj[expr]()`, `eval`, `import()`: no static target by construction; callee is NULL.' }, { table: 'call_edges', column: 'tier', value: 'fan_capped', languages: ['javascript', 'java', 'csharp'], meaning: 'More targets than --dispatch-cap: the set was refused rather than emitted. JavaScript: callee is NULL. Java and C#: callee is the declared base method the fan would have started from; dispatch-capped-sites.csv carries the refused count.' }, { table: 'call_edges', column: 'tier', value: 'callback_registered', languages: ['javascript', 'typescript'], meaning: 'The site HANDS the callee this function (`xs.forEach(f)`, `p.then(f)`, `emitter.on(\'x\', h)`, `setTimeout(f)`), which may invoke it. Not the site\'s own callee; a reachability edge, labelled so it is never read as a resolved call.' }, + { table: 'call_edges', column: 'tier', value: 'callback_registered', languages: P, meaning: 'A method or function named as a VALUE and handed over, stored or returned (`pool.submit(self.work)`, `Thread(target=svc.run)`, `sorted(xs, key=self.rank)`, `[h.on_event]`), always kind METHOD_REFERENCE. Its target is resolved the way a call on the same receiver would be; whoever receives the value calls it. A reachability edge, labelled so it is never read as a resolved call.' }, { table: 'call_edges', column: 'tier', value: 'event_dispatch', languages: S, meaning: '`x.emit(\'name\')` reaching a handler registered by `x.on(\'name\', h)` on a value x may hold — name-sensitive for literal names, every handler on that value for a computed one.' }, { table: 'call_edges', column: 'tier', value: 'event_dispatch', languages: T, meaning: 'A publish on a project class\'s keyed registry reaching each function filed under the same key: `bus.publish(\'x\', p)` where `publish` (or a method it passes the key to) calls what `this.handlers.get(name)` holds, to the function a `bus.subscribe(\'x\', f)` stored there, or an entry `{ [\'x\']: f }` of a table a subscribe-all method files by key. A literal, a const or a const-object member key; a key only known at run time matches every key, and a constant the dispatch also looks up (a wildcard) matches every publish. The site is the publish call (call-edge-generation/keyed_registry.dl).' }, { table: 'call_edges', column: 'tier', value: 'event_dispatch', languages: J, meaning: 'A Spring application event: `publishEvent(e)` reaching each listener (`@EventListener`, `@TransactionalEventListener`, `ApplicationListener.onApplicationEvent`) whose declared event type e\'s static type is, or is a subtype of. Added beside the publishEvent boundary row, never in place of it (call-edge-generation/event_dispatch.dl). Also a JPA entity write (`save`, `persist`, `merge`, `delete` on a repository or EntityManager) reaching the `@PrePersist`/`@PreUpdate`/`@PreRemove`-style callbacks of the written entity\'s type and of the listeners `@EntityListeners` names on it or a superclass (kind entity_callback; call-edge-generation/entity_lifecycle.dl).' }, @@ -684,6 +685,7 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'call_edges', column: 'kind', value: 'SUBSCRIPT_PROTOCOL', languages: P, meaning: '`x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver\'s class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call.' }, { table: 'call_edges', column: 'kind', value: 'BUILTIN_PROTOCOL', languages: P, meaning: '`repr(x)`, `str(x)`, `len(x)`, `hash(x)`, `bool(x)`, `iter(x)`, `next(x)`, `abs(x)`, `format(x)` and `reversed(x)` run the matching dunder of the argument\'s class (str falls back to `__repr__`, bool to `__len__`). The written call is to the builtin; this edge is the dunder it runs. The site is the argument expression. Its own kind so it is never counted as a written call.' }, { table: 'call_edges', column: 'kind', value: 'OPERATOR_PROTOCOL', languages: P, meaning: 'An operator runs a dunder of its operands: `a + b` the left operand\'s `__add__` and the right one\'s `__radd__`, a comparison the left slot and the right one\'s mirror (`a < b` -> `b.__gt__`, `!=` falls back to `__eq__`), `x in c` the right operand\'s `__contains__` (or `__iter__`), `-a`/`+a`/`~a` its `__neg__`/`__pos__`/`__invert__`. No written call; the site is the operator expression.' }, + { table: 'call_edges', column: 'kind', value: 'METHOD_REFERENCE', languages: P, meaning: 'A method or function READ AS A VALUE, not called: an argument (`pool.submit(self.work, x)`, `map(Cls.convert, rows)`, `partial(svc.cancel, oid)`), a keyword (`Thread(target=svc.run)`), an element, a stored or returned value. The target is what the expression resolves to (a typed receiver through the MRO, a class, a scoped name, a module member); an untyped receiver gives no row. Always tier callback_registered; the site is the reference expression (call-edge-generation/method-reference.dl).' }, { table: 'call_edges', column: 'kind', value: 'TRUTH_PROTOCOL', languages: P, meaning: 'A truth test runs the tested value\'s `__bool__`, or `__len__` when its class has none: `if x:`, `while x:`, `assert x`, `not x`, a conditional or comprehension condition, and an operand of `and`/`or` (the right one only when the whole expression is tested). No written call; the site is the tested expression.' }, // entry_points.reason @@ -763,7 +765,7 @@ export const NOTES: readonly NoteSpec[] = [ { language: 'typescript', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The TypeScript dispatch envelope is in dispatch_candidates, with basis `nominal` or `structural`.' }, { language: 'typescript', table: 'type_instantiated', note: 'Every row has how = `new`. Not restricted to client provenance: a type the library constructs is still a type that exists at run time, and dropping it would narrow the envelope unsoundly.' }, { language: 'typescript', table: 'call_sites', note: 'PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor\'s name is on the callee\'s methods row. Filter them out with kind NOT IN (…) when counting calls.' }, - { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, + { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METHOD_REFERENCE, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, { language: 'python', table: 'call_sites', note: 'The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class\'s TYPE hash for METACLASS_CREATION, positioned at the class declaration.' }, { language: 'python', table: 'call_edges', note: 'A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row.' }, { language: 'java', table: 'call_edges', note: 'A `boundary_lib` edge with callee_provenance external names a method of an ancestor type no staged IR declares (callee_label `external:.`, no methods row). A site whose receiver is declared as such a type is multi_inferred even with one client override: the platform method itself, and the platform\'s own subclasses, are the other possible targets. Stage the library to replace the label with the real method.' }, @@ -771,7 +773,7 @@ export const NOTES: readonly NoteSpec[] = [ { language: 'python', table: 'entry_points', note: 'Framework entry points only: url, http, orm_hook, task, signal_receiver, fixture, di_provider and grpc_service. There is no test and no main reason: a pytest test is recognised by the query layer from its file and name, not here.' }, { language: 'python', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The Python dispatch envelope is in dispatch_candidates with basis `mro`, and `value` for a function assigned onto an instance\'s member; the raw linearisation is in ext_mro_position.' }, { language: 'python', table: 'type_instantiated', note: 'Every row has how = `new`: the rule set records that some client call constructs the class, not which form.' }, - { language: 'all', table: 'call_edges', note: 'THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge and multi_inferred in every language, and boundary_lib where the library is staged (--library) — without it boundary_lib names the target in callee_label and leaves callee_method_id NULL; ALSO ambient_terminal in TypeScript, fan_capped in Java and C# (the declared base, the fan refused), and runtime_observed in C#. HANDED OVER (callee set, but the site passes the function rather than calling it): callback_registered and event_dispatch in JavaScript and TypeScript; event_dispatch in C# too, where a mediator Send or Publish runs the handler for the request type, beside the row for the site itself. CORRECT END (callee NULL, and nothing is missing): intrinsic_terminal in TypeScript; ambient_terminal, implicit_constructor and dynamic_terminal in JavaScript; known_implicit_ctor, known_builtin_operator and boundary_generated (callee_label set) in C#. BLIND SPOT (callee NULL; exactly the tiers named `ambiguous_*`, which are what unresolved_sites holds): ambiguous_unknown everywhere, ALSO ambiguous_anon in Java and ambiguous_dynamic in C#. CAPPED (callee NULL, not in unresolved_sites): fan_capped in JavaScript. Python emits only the four shared tiers. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in every language but Python — derive the set from this note or from unresolved_sites, never from a hardcoded list.' }, + { language: 'all', table: 'call_edges', note: 'THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge and multi_inferred in every language, and boundary_lib where the library is staged (--library) — without it boundary_lib names the target in callee_label and leaves callee_method_id NULL; ALSO ambient_terminal in TypeScript, fan_capped in Java and C# (the declared base, the fan refused), and runtime_observed in C#. HANDED OVER (callee set, but the site passes the function rather than calling it): callback_registered and event_dispatch in JavaScript and TypeScript, callback_registered (kind METHOD_REFERENCE) in Python; event_dispatch in C# too, where a mediator Send or Publish runs the handler for the request type, beside the row for the site itself. CORRECT END (callee NULL, and nothing is missing): intrinsic_terminal in TypeScript; ambient_terminal, implicit_constructor and dynamic_terminal in JavaScript; known_implicit_ctor, known_builtin_operator and boundary_generated (callee_label set) in C#. BLIND SPOT (callee NULL; exactly the tiers named `ambiguous_*`, which are what unresolved_sites holds): ambiguous_unknown everywhere, ALSO ambiguous_anon in Java and ambiguous_dynamic in C#. CAPPED (callee NULL, not in unresolved_sites): fan_capped in JavaScript. Python emits the four shared tiers and callback_registered. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in every language but Python — derive the set from this note or from unresolved_sites, never from a hardcoded list.' }, { language: 'java', table: 'call_edges', note: 'A multi_inferred fan is CHA-wide: it is every override the hierarchy admits, bounded only by the dispatch cap. type_instantiated is computed and exported but NOT read by any rule, so the fan is not narrowed to types the program constructs. Narrow it yourself by joining dispatch_candidates to type_instantiated — see the dispatch_envelope_of query. A receiver is ALSO typed by what flows into it (a local\'s initializer, the arguments callers pass to a parameter, the receivers callers invoke a method on for its `this`), and each flow-in type resolves its member directly, outside the fan: that is why a `fan_capped` site still carries edges, and why they are the types the program was seen to hand over, not the whole hierarchy.' }, { language: 'typescript', table: 'call_edges', note: 'A multi_inferred fan is CHA-wide, as in Java: type_instantiated is computed and exported but NOT read by any rule. The fan also has sources that are not virtual dispatch at all — an overload set or a union-typed receiver produces one too.' }, { language: 'python', table: 'call_edges', note: 'A multi_inferred fan IS narrowed by the instantiation set: type_instantiated_reachable (the constructed classes and their bases) bounds dispatch in resolution/dispatch.dl. Python is the only front end where that narrowing is applied, so a fan here is tighter than the same shape would be in Java or TypeScript.' }, diff --git a/graph/bundle/write.ts b/graph/bundle/write.ts index d00899fee..6898777b5 100644 --- a/graph/bundle/write.ts +++ b/graph/bundle/write.ts @@ -7,8 +7,8 @@ import * as fs from 'fs'; import * as path from 'path'; import { readRaw, writeTable } from '@/bundle/csv'; -import type { CoreTables, Row } from '@/bundle/build'; -import type { ExtRelation } from '@/bundle/catalog'; +import { lap, type CoreTables, type Row } from '@/bundle/build'; +import { EXT_QUERIED, type ExtRelation } from '@/bundle/catalog'; import { CATALOG_TABLES, CORE_TABLES, GUIDE, LANGUAGES, NOTES, QUERIES, SCHEMA_VERSION, VOCAB, type Language, type TableSpec } from '@/bundle/schema'; export function writeCoreCsv(graphDir: string, core: CoreTables, log: (s: string) => void): void { @@ -48,6 +48,7 @@ export interface SqliteInputs { // VALUES list amortizes both; 128 rows per statement keeps the flattened argument array well // under sqlite's bind limit at the widest table. const BATCH = Math.max(1, Number(process.env.AXIOM_BUNDLE_BATCH ?? 128) || 128); +const PAGE_SIZE = [4096, 8192, 16384, 32768, 65536].includes(Number(process.env.AXIOM_BUNDLE_PAGE_SIZE)) ? Number(process.env.AXIOM_BUNDLE_PAGE_SIZE) : 16384; type DbLike = { prepare(sql: string): { run(...a: (string | number | null)[]): unknown } }; function batchInsert(db: DbLike, table: string, arity: number, verb: string): { push(r: (string | number | null)[]): void; flush(): void } { const one = `(${Array.from({ length: arity }, () => '?').join(', ')})`; @@ -72,12 +73,35 @@ function batchInsert(db: DbLike, table: string, arity: number, verb: string): { }; } +/** + * A keyed table's rows in key order, for loading. A WITHOUT ROWID table is stored in key order whatever order its + * rows arrive in, so this changes nothing stored; but rows arriving in hash order land all over the B-tree and + * split its pages again and again, while rows in key order are appended. The sort is stable, so of two rows with + * one key the first is still the one INSERT OR IGNORE keeps. A copy is sorted: the caller's array is not reordered. + */ +function inKeyOrder(t: TableSpec, rows: Row[]): Row[] { + const keys = t.columns.flatMap((c, i) => (c.key ? [i] : [])); + if (keys.length === 0 || rows.length < 2) return rows; + const str = (v: Row[number] | undefined) => (v === null || v === undefined ? '' : String(v)); + return rows.slice().sort((a, b) => { + for (const i of keys) { + const x = str(a[i]), y = str(b[i]); + if (x !== y) return x < y ? -1 : 1; + } + return 0; + }); +} + export async function writeSqlite(inp: SqliteInputs): Promise { // Loaded lazily so the CSV path works on a Node without the module. const { DatabaseSync } = await import('node:sqlite'); fs.rmSync(inp.dbPath, { force: true }); fs.rmSync(inp.dbPath + '-journal', { force: true }); const db = new DatabaseSync(inp.dbPath); + // 16 KiB PAGES, set before the first table (AXIOM_BUNDLE_PAGE_SIZE overrides). The bundle is a bulk load of a file + // that reaches GBs, and at SQLite's default 4 KiB every table and index B-tree has four times the pages to split + // and write; the queries over the graph read it no slower. The page size changes no row and no answer. + db.exec(`PRAGMA page_size = ${PAGE_SIZE};`); db.exec('PRAGMA journal_mode = OFF; PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;'); try { // ── core ──────────────────────────────────────────────────────────────── @@ -86,11 +110,13 @@ export async function writeSqlite(inp: SqliteInputs): Promise { db.exec(createSql(t)); const rows = (inp.core as unknown as Record)[t.name] ?? []; const ins = batchInsert(db, t.name, t.columns.length, 'INSERT OR IGNORE'); - for (const r of rows) ins.push(r as (string | number | null)[]); + for (const r of inKeyOrder(t, rows)) ins.push(r as (string | number | null)[]); ins.flush(); inp.log(` sqlite ${t.name}: ${rows.length} rows`); + lap(`insert ${t.name} (${rows.length} rows)`); } db.exec('COMMIT;'); + lap('commit core'); // ── ext ───────────────────────────────────────────────────────────────── db.exec('BEGIN;'); @@ -107,8 +133,10 @@ export async function writeSqlite(inp: SqliteInputs): Promise { } ins.flush(); if (n > 0) inp.log(` sqlite ${name}: ${n} rows`); + lap(`insert ${name} (${n} rows)`); } db.exec('COMMIT;'); + lap('commit ext'); // ── catalog ───────────────────────────────────────────────────────────── db.exec('BEGIN;'); @@ -165,14 +193,17 @@ export async function writeSqlite(inp: SqliteInputs): Promise { const query = db.prepare('INSERT INTO schema_queries VALUES (?, ?, ?, ?)'); for (const q of QUERIES) query.run(q.name, q.question, q.params, q.sql); db.exec('COMMIT;'); + lap('catalog + vocab'); // ── indexes, after the load ───────────────────────────────────────────── db.exec('BEGIN;'); for (const t of CORE_TABLES) { - for (const cc of t.columns) if (cc.indexed) db.exec(`CREATE INDEX idx_${t.name}_${cc.name} ON ${t.name}(${cc.name});`); + for (const cc of t.columns) if (cc.indexed) { db.exec(`CREATE INDEX idx_${t.name}_${cc.name} ON ${t.name}(${cc.name});`); lap(`index ${t.name}.${cc.name}`); } } - for (const e of inp.ext) if (e.arity > 0) db.exec(`CREATE INDEX idx_ext_${e.relation}_c0 ON ext_${e.relation}(c0);`); + // an ext table the tools query keeps its c0 index; the rest are loaded and left unindexed (catalog.ts EXT_QUERIED) + for (const e of inp.ext) if (e.arity > 0 && EXT_QUERIED.has(e.relation)) { db.exec(`CREATE INDEX idx_ext_${e.relation}_c0 ON ext_${e.relation}(c0);`); lap(`index ext_${e.relation}.c0`); } db.exec('COMMIT;'); + lap('commit indexes'); db.exec(`PRAGMA user_version = ${Number(SCHEMA_VERSION)};`); } finally { db.close(); diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index f903e654b..b0ce60a1f 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -88,6 +88,8 @@ ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; } # shellcheck source=souffle-include.sh . "$SRC/pipeline/souffle-include.sh" +# shellcheck source=stage-rows.sh +. "$SRC/pipeline/stage-rows.sh" # shellcheck source=engine.conf . "$SRC/pipeline/engine.conf" # Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). @@ -817,7 +819,7 @@ while IFS=$'\t' read -r rel csv; do # deterministically on some inputs — takes down a run of tens of thousands of sites. The # header's field count is the contract; a row that does not meet it is dropped and # COUNTED, so the loss is visible rather than fatal and never silent. - if [ -f "$CLIENT/$csv.csv" ]; then awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" + if [ -f "$CLIENT/$csv.csv" ]; then stage_rows "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" else : > "$FACTS/$rel.facts"; fi CLIENT_INPUTS="$CLIENT_INPUTS$rel"$'\n' done < <(read_map "$TPL/client-ir.map"; if [ -f "$TPL/client-extra.map" ]; then read_map "$TPL/client-extra.map"; fi) @@ -879,7 +881,7 @@ if [ ! -d "$LIBDIR" ]; then : > "$TMPDIR_L/$rel.facts" for root in "${LIB_ROOTS[@]}"; do while IFS= read -r mod; do - [ -f "$mod/$csv.csv" ] && awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts" + [ -f "$mod/$csv.csv" ] && stage_rows "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts" done < <(lib_modules "$root") done done < <(printf '%s' "$LIB_SIG_RELS") diff --git a/graph/pipeline/stage-rows.sh b/graph/pipeline/stage-rows.sh new file mode 100644 index 000000000..200ff7670 --- /dev/null +++ b/graph/pipeline/stage-rows.sh @@ -0,0 +1,50 @@ +# stage-rows.sh: copy one IR CSV into a Soufflé fact file, dropping torn rows. Sourced by run-souffle.sh. +# +# stage_rows FILE print FILE's rows without its header, keeping only the rows whose tab-separated +# field count equals the header's; report how many were dropped on stderr. +# +# The filter is line based: a row is one newline-terminated line, a field count is the number of tabs +# plus one (an empty line has none), and a last line without a newline is printed with one. Those are +# exactly awk -F'\t' semantics, which is what this step used to run. +# +# PERL FIRST, AWK AS THE FALLBACK. The two print byte-identical output, but BSD awk (macOS) spends most +# of a build's staging time splitting fields it never reads: 13.6 CPU-s on a 352 MB expressions file, +# against 0.65 s for perl, which only counts tabs. perl ships with macOS, every mainstream Linux +# distribution and Git for Windows' bash (the shell this runs under on win32); a minimal image without +# it (Alpine, distroless) gets the same rows from awk, just slower. graph/test/tools/stage-rows-test.sh +# holds both to the same bytes on the edge cases (trailing tabs, empty lines, CRLF, no final newline). +# +# AXIOM_STAGE_WITH=awk forces the fallback (the test uses it; so can anyone bisecting a staging change). + +if [ "${AXIOM_STAGE_WITH:-}" != awk ] && command -v perl >/dev/null 2>&1 && perl -e 1 >/dev/null 2>&1; then + STAGE_ROWS_TOOL=perl +else + STAGE_ROWS_TOOL=awk +fi + +stage_rows(){ + if [ "$STAGE_ROWS_TOOL" = perl ]; then + # Bytes in, bytes out (no PerlIO layers, whatever PERL_UNICODE says), and $/ pinned to "\n". + perl -e ' + my $f = $ARGV[0]; + open(my $h, "<:raw", $f) or die "cannot read $f: $!\n"; + binmode STDOUT, ":raw"; + local $/ = "\n"; + my $hdr = <$h>; + exit 0 unless defined $hdr; + my $hl = length($hdr) - (substr($hdr, -1) eq "\n" ? 1 : 0); + my $n = $hl ? ($hdr =~ tr/\t//) + 1 : 0; + my $bad = 0; + while (my $l = <$h>) { + my $nl = substr($l, -1) eq "\n"; + my $nf = (length($l) - ($nl ? 1 : 0)) ? ($l =~ tr/\t//) + 1 : 0; + if ($nf == $n) { print($nl ? $l : "$l\n") } else { $bad++ } + } + close($h); + close(STDOUT) or die "cannot write the staged rows of $f: $!\n"; + printf STDERR " ! dropped %d malformed row(s) from %s\n", $bad, $f if $bad > 0; + ' "$1" + else + awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$1" + fi +} diff --git a/graph/python/engine/call-edge-generation/method-reference.dl b/graph/python/engine/call-edge-generation/method-reference.dl new file mode 100644 index 000000000..c3bcf5da2 --- /dev/null +++ b/graph/python/engine/call-edge-generation/method-reference.dl @@ -0,0 +1,123 @@ +// ============================================================================ +// CALL-EDGE-GENERATION · METHOD REFERENCES (a callable named as a VALUE, not called) +// +// ── THE GAP ───────────────────────────────────────────────────────────────── +// executor.submit(self._do_work, item) threading.Thread(target=svc.run_job) +// loop.run_in_executor(None, svc.cancel, oid) sorted(xs, key=self._rank) +// callbacks = [handler.on_event] functools.partial(svc.cancel, oid) +// app.add_url_rule("/x", view_func=views.index) return self._handle +// Each hands a method or function over as a value; whoever receives it calls it, later, from +// a frame the graph cannot see (a worker thread, a builtin, a library). There is no call site +// for it, so before this file the method had NO caller here at all, and `impact +// OrderService.cancel` did not name the function that queued it. The engine already KNEW what +// each of these expressions denotes -- expr_denotes_method resolves `self._do_work` through the +// receiver's MRO exactly as it resolves `self._do_work()` -- and dropped the answer because no +// CALL wrapped it. +// +// ── WHAT IT EMITS ─────────────────────────────────────────────────────────── +// call_chain_edge(Expr, Caller, "-", Target, "client", "callback_registered", "METHOD_REFERENCE") +// Like PROPERTY_READ and the protocol rows in call_chain.dl, an edge with no call site: Site is +// the reference expression itself, and the kind is its own so a consumer can always tell it +// from a written call. The tier is the one every other front end uses for a function handed +// over (`xs.forEach(f)`, a JSX handler attribute): HANDED OVER, never `resolved`, so an answer +// says "handed over as a value" instead of "calls it". +// +// ── WHAT IT REFUSES ───────────────────────────────────────────────────────── +// - ONLY WHAT IS RESOLVED. The target comes from the same resolution an ordinary call's +// callee gets: a receiver typed through expr_type and the MRO, a class named by its +// declaration, a name bound by its scope, a module member. `thing.cancel` on an untyped +// parameter denotes nothing, so it emits nothing -- the same blind spot `thing.cancel()` +// has. No rule here matches by name. +// - NOT A FLOWED VALUE. `def run(cb): pool.submit(cb)` denotes, through +// param_arg_method, every function any caller passed in. The reference that matters is +// the one the caller WROTE (`run(self._work)`), which already has its own row; crediting +// `run` as well would make it reach the union of every callable handed to it. The same +// holds for an attribute that holds a function (see (b) below). +// - NOT A CALLEE, NOT A WRITE, NOT AN OBJECT OF ANOTHER ATTRIBUTE. `self.m()` is a call +// (call_chain.dl owns it); `self.m = x` is a store; `self.m.__name__` and +// `self.m.cache_clear()` read the function object and never run it. +// - NOT A PROPERTY, NOT A DECORATOR. `order.total` on a @property (or @cached_property) +// reads the VALUE the getter computes: call_chain.dl already emits that as PROPERTY_READ, +// and the function object is never handed anywhere. `@audited` is a decorator +// application, which call_chain.dl emits as DECORATOR_*. Before these two guards they +// were the two largest wrong classes in a sampled review. +// - NOT THE ARGUMENT OF A ONE-DUNDER BUILTIN. `str(fn)`, `repr(fn)`, `hash(fn)` +// (builtins.dl's builtin_protocol_slot) run a dunder of the FUNCTION OBJECT and never call +// it; a test asserting on `str(svc.cancel)` is not a caller of cancel. Other inspecting +// builtins (`id`, `callable`, `isinstance`) are not catalogued and still count. +// - NOT A COMPARISON OPERAND. `assert job.on_done is svc.cancel` tests identity. +// - ONLY NAMES AND ATTRIBUTE READS. A lambda, a subscript (`table[k]`) and a call that +// returns a function (`partial(...)`, a factory) also denote methods; each already has its +// own edge where it is invoked, or IS a call with its own row. +// The receiver-typing, decorator-chain and import rules this reads all live in resolution/; +// nothing here widens them. +// ============================================================================ + +.decl method_ref_expr(e:symbol, m:symbol) +// (a) a NAME: what resolution/ says it denotes -- a def bound in scope, an import, a +// decorated name's wrapper -- minus a name that holds a flowed VALUE: a parameter, or a +// local or module name assigned from an expression (`cb = job.on_done`). That +// assignment's right-hand side is credited where it is written when it is a reference +// itself; when it is a field read, what the name holds is the union of every write to that +// field, which was the largest wrong shape in a sampled review (`assert cb is not None` +// credited with every callback any job was ever given). +method_ref_expr(e, m) :- + expr_node("client", "NAME_REFERENCE", _, _, e), + expr_denotes_method("client", e, m), + !expr_names_param("client", e, _), + !method_ref_name_holds_value(e). +.decl method_ref_name_holds_value(e:symbol) +method_ref_name_holds_value(e) :- + expr_node("client", "NAME_REFERENCE", _, _, e), + expr_binding("client", b, ctx, e), ctx != "STORE", + binding_lookup("client", b, b2), binding_value_method("client", b2, _). +// (b) an ATTRIBUTE: a method found through the typed receiver's MRO (`svc.cancel`, +// `self._work`), or a module member (`views.index`, decorated or not). NOT an attribute +// that merely HOLDS a function some write put there (`job.on_done`): that is a flowed +// value like a parameter's, and a sampled review found exactly that shape wrong: +// `job.on_done is NO_OP`, credited to whatever any write stored. +method_ref_expr(e, m) :- expr_denotes_bound_method("client", e, m). +method_ref_expr(e, m) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, _, e), + expr_attr_names_method("client", e, _), + expr_denotes_method("client", e, m). +// (c) an UNBOUND method read off its class: `map(OrderService.convert, rows)`. The class +// object types the receiver, and the MRO names the method -- the rule +// callee-resolution.dl uses for `OrderService.convert(svc, row)`, without the call. +method_ref_expr(e, m) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), + expr_parent("client", e, "ATTRIBUTE_OBJECT", _, obj), + expr_type_class_object("client", obj, t), + mro_lookup("client", t, n, m). + +// the shape guards: read as a value, not called, not a write, not the object of another read +.decl method_ref_site(e:symbol, m:symbol) +method_ref_site(e, m) :- + method_ref_expr(e, m), + !call_callee_expr(_, e), + !expr_name_context("client", "STORE", e), !expr_name_context("client", "DEL", e), + !expr_parent("client", _, "ATTRIBUTE_OBJECT", _, e), + !decorator_expr("client", e, _), + !builtin_protocol_arg(_, _, e), + !method_ref_compared(e), + method_kind("client", kind, _, m), + kind != "MODULE_INITIALIZER", kind != "CLASS_INITIALIZER", kind != "OVERLOAD_STUB", kind != "LAMBDA", + kind != "PROPERTY_GETTER", kind != "PROPERTY_SETTER", kind != "PROPERTY_DELETER". + +// an operand of a comparison: `job.on_done is svc.cancel`, `h in self.handlers`, `f == g` test +// the function object's identity or equality and never call it +.decl method_ref_compared(e:symbol) +method_ref_compared(e) :- method_ref_expr(e, _), expr_parent("client", c, _, _, e), expr_node("client", "COMPARISON", _, _, c). + +// the caller: the function or lambda whose body evaluates the reference. A module-level +// reference and a parameter default are owned by the synthetic , a class-body one by +// -- the parser records that owner on the expression, as it does for a call. +.decl method_ref_caller(e:symbol, c:symbol) +method_ref_caller(e, c) :- method_ref_site(e, _), expr_owner("client", "METHOD", c, _, e). +method_ref_caller(e, c) :- method_ref_site(e, _), expr_owner("client", "LAMBDA", c, _, e). +method_ref_caller(e, c) :- method_ref_site(e, _), + !expr_owner("client", "METHOD", _, _, e), !expr_owner("client", "LAMBDA", _, _, e), + expr_ultimate_method("client", e, c). + +call_chain_edge(e, c, "-", m, "client", "callback_registered", "METHOD_REFERENCE") :- + method_ref_site(e, m), method_ref_caller(e, c). diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 18db81e1d..e2a48d386 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -99,6 +99,13 @@ fi if ! bash "$ROOT/graph/test/tools/lib-cache-key-test.sh"; then echo "FAIL: the library-facts cache key is not a function of the staged modules' content (#588)"; exit 1 fi +# ── Staging must not depend on which row filter the machine has ───────────── +# graph/pipeline/stage-rows.sh runs perl where it exists and awk where it does not; both must stage +# the same bytes, or the facts every language solves from would differ by machine. +if ! bash "$ROOT/graph/test/tools/stage-rows-test.sh"; then + echo "aborting: perl and awk stage different rows from the same IR" + exit 1 +fi # ── The -I for soufflé's headers must be the one that actually compiles ────── # Also a preflight, and for the same reason: the old resolution returned a path that only built on # the machine it was written on, and every run here compiles the engine through it. diff --git a/graph/test/python/expected/01-plain-calls.edges b/graph/test/python/expected/01-plain-calls.edges index c37ab9bcd..9df3ca724 100644 --- a/graph/test/python/expected/01-plain-calls.edges +++ b/graph/test/python/expected/01-plain-calls.edges @@ -1,4 +1,5 @@ boundary_lib SIMPLE_CALL main.main -> builtin:print +callback_registered METHOD_REFERENCE main.make_adder -> main.make_adder..add known_edge SIMPLE_CALL main. -> main.main known_edge SIMPLE_CALL main.main -> main.make_adder known_edge SIMPLE_CALL main.main -> main.make_adder..add diff --git a/graph/test/python/expected/01-plain-calls.tiers b/graph/test/python/expected/01-plain-calls.tiers index 5b235883c..0c7ce9f47 100644 --- a/graph/test/python/expected/01-plain-calls.tiers +++ b/graph/test/python/expected/01-plain-calls.tiers @@ -1,10 +1,12 @@ -distinct call sites emitted: 12 +distinct call sites emitted: 13 --- by tier: edge ROWS, and the distinct SITES they cover --- 3 rows 3 sites boundary_lib + 1 rows 1 sites callback_registered 9 rows 9 sites known_edge --- edge rows by call kind --- + 1 METHOD_REFERENCE 12 SIMPLE_CALL --- unresolved reasons --- @@ -16,7 +18,7 @@ distinct call sites emitted: 12 9 known_edge --- reconciling rows against the conserved site count --- - edge rows 12 + edge rows 13 minus extra rows from multi-target sites 0 - = tier/site pairs 12 + = tier/site pairs 13 engine's conserved site total 12 diff --git a/graph/test/python/expected/07-decorators.edges b/graph/test/python/expected/07-decorators.edges index 41e8ab4f3..fa1975f0d 100644 --- a/graph/test/python/expected/07-decorators.edges +++ b/graph/test/python/expected/07-decorators.edges @@ -8,6 +8,11 @@ boundary_lib SIMPLE_CALL main.inner_deco..inner_w -> builtin:str boundary_lib SIMPLE_CALL main.main -> builtin:print boundary_lib SIMPLE_CALL main.outer_deco..outer_w -> builtin:str boundary_lib SIMPLE_CALL main.wrapping..wrapper -> builtin:str +callback_registered METHOD_REFERENCE main.inner_deco -> main.inner_deco..inner_w +callback_registered METHOD_REFERENCE main.outer_deco -> main.outer_deco..outer_w +callback_registered METHOD_REFERENCE main.replacing -> main.replacing..substitute +callback_registered METHOD_REFERENCE main.route -> main.route..register +callback_registered METHOD_REFERENCE main.wrapping -> main.wrapping..wrapper known_edge DECORATOR_APPLICATION main. -> main.route..register known_edge DECORATOR_BARE main. -> main.inner_deco known_edge DECORATOR_BARE main. -> main.outer_deco diff --git a/graph/test/python/expected/07-decorators.tiers b/graph/test/python/expected/07-decorators.tiers index e9686fea6..33c95f323 100644 --- a/graph/test/python/expected/07-decorators.tiers +++ b/graph/test/python/expected/07-decorators.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 30 +distinct call sites emitted: 35 --- by tier: edge ROWS, and the distinct SITES they cover --- 3 rows 3 sites ambiguous_unknown 11 rows 11 sites boundary_lib + 5 rows 5 sites callback_registered 16 rows 16 sites known_edge --- edge rows by call kind --- 4 DECORATOR_APPLICATION 5 DECORATOR_BARE 4 DECORATOR_CALL + 5 METHOD_REFERENCE 16 SIMPLE_CALL 1 SUBSCRIPT_CALL @@ -22,7 +24,7 @@ distinct call sites emitted: 30 16 known_edge --- reconciling rows against the conserved site count --- - edge rows 30 + edge rows 35 minus extra rows from multi-target sites 0 - = tier/site pairs 30 + = tier/site pairs 35 engine's conserved site total 30 diff --git a/graph/test/python/expected/08-callables.edges b/graph/test/python/expected/08-callables.edges index 5b0c619d2..24c61eb13 100644 --- a/graph/test/python/expected/08-callables.edges +++ b/graph/test/python/expected/08-callables.edges @@ -2,6 +2,8 @@ ambiguous_unknown SELF_CALL main.Holder.use_partial -> - boundary_lib METHOD_CALL main.Holder.__init__ -> external:functools.partial boundary_lib METHOD_CALL main.main -> external:functools.partial boundary_lib SIMPLE_CALL main.main -> builtin:print +callback_registered METHOD_REFERENCE main.Holder.__init__ -> main.add +callback_registered METHOD_REFERENCE main.main -> main.add known_edge CHAINED_CALL main.main -> main.Holder.use known_edge CHAINED_CALL main.main -> main.Holder.use_partial known_edge SELF_CALL main.Holder.use -> main.add diff --git a/graph/test/python/expected/08-callables.tiers b/graph/test/python/expected/08-callables.tiers index 9d83aad23..7a2631ca0 100644 --- a/graph/test/python/expected/08-callables.tiers +++ b/graph/test/python/expected/08-callables.tiers @@ -1,13 +1,15 @@ -distinct call sites emitted: 20 +distinct call sites emitted: 23 --- by tier: edge ROWS, and the distinct SITES they cover --- 1 rows 1 sites ambiguous_unknown 8 rows 8 sites boundary_lib + 3 rows 3 sites callback_registered 11 rows 11 sites known_edge --- edge rows by call kind --- 2 CHAINED_CALL 3 METHOD_CALL + 3 METHOD_REFERENCE 3 SELF_CALL 11 SIMPLE_CALL 1 SUBSCRIPT_CALL @@ -22,7 +24,7 @@ distinct call sites emitted: 20 11 known_edge --- reconciling rows against the conserved site count --- - edge rows 20 + edge rows 23 minus extra rows from multi-target sites 0 - = tier/site pairs 20 + = tier/site pairs 23 engine's conserved site total 20 diff --git a/graph/test/python/expected/09-args-forwarding.edges b/graph/test/python/expected/09-args-forwarding.edges index d05a79897..1dfc354c7 100644 --- a/graph/test/python/expected/09-args-forwarding.edges +++ b/graph/test/python/expected/09-args-forwarding.edges @@ -1,5 +1,6 @@ boundary_lib SIMPLE_CALL main.main -> builtin:print boundary_lib SIMPLE_CALL main.target -> builtin:len +callback_registered METHOD_REFERENCE main.main -> main.target known_edge DYNAMIC_CALL main.main -> main.Delegator.__call__ known_edge SELF_CALL main.Delegator.__call__ -> main.target known_edge SIMPLE_CALL main. -> main.main diff --git a/graph/test/python/expected/09-args-forwarding.tiers b/graph/test/python/expected/09-args-forwarding.tiers index 61fade854..13ce90bc8 100644 --- a/graph/test/python/expected/09-args-forwarding.tiers +++ b/graph/test/python/expected/09-args-forwarding.tiers @@ -1,11 +1,13 @@ -distinct call sites emitted: 15 +distinct call sites emitted: 16 --- by tier: edge ROWS, and the distinct SITES they cover --- 5 rows 5 sites boundary_lib + 1 rows 1 sites callback_registered 10 rows 10 sites known_edge --- edge rows by call kind --- 1 DYNAMIC_CALL + 1 METHOD_REFERENCE 1 SELF_CALL 13 SIMPLE_CALL @@ -18,7 +20,7 @@ distinct call sites emitted: 15 10 known_edge --- reconciling rows against the conserved site count --- - edge rows 15 + edge rows 16 minus extra rows from multi-target sites 0 - = tier/site pairs 15 + = tier/site pairs 16 engine's conserved site total 15 diff --git a/graph/test/python/expected/12-blind-spots.edges b/graph/test/python/expected/12-blind-spots.edges index 014a110bf..910099c29 100644 --- a/graph/test/python/expected/12-blind-spots.edges +++ b/graph/test/python/expected/12-blind-spots.edges @@ -7,6 +7,7 @@ boundary_lib SIMPLE_CALL main.Installer.__new__ -> builtin:super boundary_lib SIMPLE_CALL main.call_by_computed_name -> builtin:getattr boundary_lib SIMPLE_CALL main.main -> builtin:print boundary_lib SUPER_CALL main.Installer.__new__ -> builtin:type.__new__ +callback_registered METHOD_REFERENCE main. -> main.patched known_edge METACLASS_CREATION main. -> main.Installer.__new__ known_edge METHOD_CALL main.main -> main.patched known_edge SIMPLE_CALL main. -> main.main diff --git a/graph/test/python/expected/12-blind-spots.tiers b/graph/test/python/expected/12-blind-spots.tiers index c2de53b23..77b9517f7 100644 --- a/graph/test/python/expected/12-blind-spots.tiers +++ b/graph/test/python/expected/12-blind-spots.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 19 +distinct call sites emitted: 20 --- by tier: edge ROWS, and the distinct SITES they cover --- 5 rows 5 sites ambiguous_unknown 10 rows 10 sites boundary_lib + 1 rows 1 sites callback_registered 4 rows 4 sites known_edge --- edge rows by call kind --- @@ -10,6 +11,7 @@ distinct call sites emitted: 19 1 DYNAMIC_CALL 1 METACLASS_CREATION 2 METHOD_CALL + 1 METHOD_REFERENCE 13 SIMPLE_CALL 1 SUPER_CALL @@ -25,8 +27,8 @@ distinct call sites emitted: 19 3 known_edge --- reconciling rows against the conserved site count --- - edge rows 19 + edge rows 20 minus extra rows from multi-target sites 0 - = tier/site pairs 19 + = tier/site pairs 20 engine's conserved site total 18 of which 1 are METACLASS_CREATION, an edge with no call site (README decision 5) diff --git a/graph/test/python/expected/17-rebound-metaclass-slot.edges b/graph/test/python/expected/17-rebound-metaclass-slot.edges index 8555ff828..01bd9313b 100644 --- a/graph/test/python/expected/17-rebound-metaclass-slot.edges +++ b/graph/test/python/expected/17-rebound-metaclass-slot.edges @@ -10,6 +10,7 @@ boundary_lib SUPER_CALL main.EarlyMeta.__init__ -> builtin:type.__init__ boundary_lib SUPER_CALL main.EarlyMeta.__new__ -> builtin:type.__new__ boundary_lib SUPER_CALL main.LateMeta.__init__ -> builtin:type.__init__ boundary_lib SUPER_CALL main.LateMeta.__new__ -> builtin:type.__new__ +callback_registered METHOD_REFERENCE main. -> main._late_new known_edge CHAINED_CALL main.main -> main.After.run known_edge CHAINED_CALL main.main -> main.AlsoBefore.run known_edge CHAINED_CALL main.main -> main.Before.run diff --git a/graph/test/python/expected/17-rebound-metaclass-slot.tiers b/graph/test/python/expected/17-rebound-metaclass-slot.tiers index 36fff263c..7b9785d53 100644 --- a/graph/test/python/expected/17-rebound-metaclass-slot.tiers +++ b/graph/test/python/expected/17-rebound-metaclass-slot.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 20 +distinct call sites emitted: 22 --- by tier: edge ROWS, and the distinct SITES they cover --- 3 rows 3 sites ambiguous_unknown 11 rows 11 sites boundary_lib + 2 rows 2 sites callback_registered 8 rows 6 sites known_edge --- edge rows by call kind --- 3 CHAINED_CALL 5 METACLASS_CREATION 1 METHOD_CALL + 2 METHOD_REFERENCE 9 SIMPLE_CALL 4 SUPER_CALL @@ -23,8 +25,8 @@ distinct call sites emitted: 20 3 known_edge --- reconciling rows against the conserved site count --- - edge rows 22 + edge rows 24 minus extra rows from multi-target sites 2 - = tier/site pairs 20 + = tier/site pairs 22 engine's conserved site total 17 of which 5 are METACLASS_CREATION, an edge with no call site (README decision 5) diff --git a/graph/test/python/expected/21-url-and-signal-dispatch.edges b/graph/test/python/expected/21-url-and-signal-dispatch.edges index 9d07c714d..0d85ec445 100644 --- a/graph/test/python/expected/21-url-and-signal-dispatch.edges +++ b/graph/test/python/expected/21-url-and-signal-dispatch.edges @@ -3,6 +3,13 @@ ambiguous_unknown DECORATOR_APPLICATION main. -> - ambiguous_unknown METHOD_CALL main.not_a_signal_send -> - boundary_lib SIMPLE_CALL main. -> builtin:object.__init__ boundary_lib SIMPLE_CALL signals. -> builtin:object.__init__ +callback_registered METHOD_REFERENCE main. -> main.detail +callback_registered METHOD_REFERENCE main. -> main.index +callback_registered METHOD_REFERENCE main. -> main.legacy +callback_registered METHOD_REFERENCE main. -> main.on_order_connected +callback_registered METHOD_REFERENCE main.not_a_route_table -> main.detail +callback_registered METHOD_REFERENCE urls. -> views.profile +callback_registered METHOD_REFERENCE urls. -> views.settings_page known_edge DECORATOR_CALL handlers. -> handlers.receiver known_edge DECORATOR_CALL main. -> main.receiver known_edge METHOD_CALL main. -> main.Signal.connect diff --git a/graph/test/python/expected/21-url-and-signal-dispatch.tiers b/graph/test/python/expected/21-url-and-signal-dispatch.tiers index 56e700521..03db6da82 100644 --- a/graph/test/python/expected/21-url-and-signal-dispatch.tiers +++ b/graph/test/python/expected/21-url-and-signal-dispatch.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 32 +distinct call sites emitted: 39 --- by tier: edge ROWS, and the distinct SITES they cover --- 5 rows 5 sites ambiguous_unknown 4 rows 4 sites boundary_lib + 7 rows 7 sites callback_registered 23 rows 23 sites known_edge --- edge rows by call kind --- 4 DECORATOR_APPLICATION 4 DECORATOR_CALL 4 METHOD_CALL + 7 METHOD_REFERENCE 20 SIMPLE_CALL --- unresolved reasons --- @@ -22,7 +24,7 @@ distinct call sites emitted: 32 23 known_edge --- reconciling rows against the conserved site count --- - edge rows 32 + edge rows 39 minus extra rows from multi-target sites 0 - = tier/site pairs 32 + = tier/site pairs 39 engine's conserved site total 32 diff --git a/graph/test/python/expected/22-di-providers.edges b/graph/test/python/expected/22-di-providers.edges index f73dff093..f02424a43 100644 --- a/graph/test/python/expected/22-di-providers.edges +++ b/graph/test/python/expected/22-di-providers.edges @@ -3,6 +3,12 @@ ambiguous_unknown DECORATOR_APPLICATION routes. -> - ambiguous_unknown METHOD_CALL main.read_item -> - boundary_lib SIMPLE_CALL main. -> builtin:object.__init__ boundary_lib SIMPLE_CALL routes. -> builtin:object.__init__ +callback_registered METHOD_REFERENCE deps. -> deps.get_repo +callback_registered METHOD_REFERENCE main. -> main.get_service +callback_registered METHOD_REFERENCE main. -> main.get_settings +callback_registered METHOD_REFERENCE main. -> main.get_user +callback_registered METHOD_REFERENCE routes. -> deps.get_page_size +callback_registered METHOD_REFERENCE routes. -> deps.get_repo known_edge DECORATOR_CALL main. -> main._App.get known_edge DECORATOR_CALL main. -> main._App.post known_edge DECORATOR_CALL routes. -> routes._Router.get diff --git a/graph/test/python/expected/22-di-providers.tiers b/graph/test/python/expected/22-di-providers.tiers index 193e49ac0..c1e8f4c11 100644 --- a/graph/test/python/expected/22-di-providers.tiers +++ b/graph/test/python/expected/22-di-providers.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 22 +distinct call sites emitted: 29 --- by tier: edge ROWS, and the distinct SITES they cover --- 4 rows 4 sites ambiguous_unknown 2 rows 2 sites boundary_lib + 7 rows 7 sites callback_registered 16 rows 16 sites known_edge --- edge rows by call kind --- 3 DECORATOR_APPLICATION 3 DECORATOR_CALL 1 METHOD_CALL + 7 METHOD_REFERENCE 15 SIMPLE_CALL --- unresolved reasons --- @@ -22,7 +24,7 @@ distinct call sites emitted: 22 16 known_edge --- reconciling rows against the conserved site count --- - edge rows 22 + edge rows 29 minus extra rows from multi-target sites 0 - = tier/site pairs 22 + = tier/site pairs 29 engine's conserved site total 22 diff --git a/graph/test/python/expected/25-imported-decorated-names.edges b/graph/test/python/expected/25-imported-decorated-names.edges index 7f267280d..1c4482423 100644 --- a/graph/test/python/expected/25-imported-decorated-names.edges +++ b/graph/test/python/expected/25-imported-decorated-names.edges @@ -3,6 +3,8 @@ ambiguous_unknown DECORATOR_ATTRIBUTE pkg.lib. -> - ambiguous_unknown METHOD_CALL main.run -> - ambiguous_unknown SIMPLE_CALL main.run -> - boundary_lib DECORATOR_CALL pkg.lib.wraps_deco -> external:functools.wraps +callback_registered METHOD_REFERENCE pkg.lib.plain_deco -> pkg.lib.plain_deco..wrapper +callback_registered METHOD_REFERENCE pkg.lib.wraps_deco -> pkg.lib.wraps_deco..wrapper known_edge DECORATOR_BARE pkg.lib. -> pkg.lib.passthrough known_edge DECORATOR_BARE pkg.lib. -> pkg.lib.plain_deco known_edge DECORATOR_BARE pkg.lib. -> pkg.lib.wraps_deco diff --git a/graph/test/python/expected/25-imported-decorated-names.tiers b/graph/test/python/expected/25-imported-decorated-names.tiers index b4fa51e62..2cd48680e 100644 --- a/graph/test/python/expected/25-imported-decorated-names.tiers +++ b/graph/test/python/expected/25-imported-decorated-names.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 24 +distinct call sites emitted: 26 --- by tier: edge ROWS, and the distinct SITES they cover --- 4 rows 4 sites ambiguous_unknown 1 rows 1 sites boundary_lib + 2 rows 2 sites callback_registered 19 rows 19 sites known_edge --- edge rows by call kind --- @@ -11,6 +12,7 @@ distinct call sites emitted: 24 3 DECORATOR_BARE 1 DECORATOR_CALL 5 METHOD_CALL + 2 METHOD_REFERENCE 13 SIMPLE_CALL --- unresolved reasons --- @@ -26,7 +28,7 @@ distinct call sites emitted: 24 19 known_edge --- reconciling rows against the conserved site count --- - edge rows 24 + edge rows 26 minus extra rows from multi-target sites 0 - = tier/site pairs 24 + = tier/site pairs 26 engine's conserved site total 24 diff --git a/graph/test/python/expected/27-returned-expression.edges b/graph/test/python/expected/27-returned-expression.edges index fed3df32d..c8699cfe3 100644 --- a/graph/test/python/expected/27-returned-expression.edges +++ b/graph/test/python/expected/27-returned-expression.edges @@ -6,6 +6,9 @@ boundary_lib METHOD_CALL main.Expr.copy -> external:copy.copy boundary_lib SIMPLE_CALL main. -> builtin:object.__init__ boundary_lib SIMPLE_CALL main.Expr.__init__ -> builtin:object.__init__ boundary_lib SIMPLE_CALL main.Expr.label -> builtin:str +callback_registered METHOD_REFERENCE main.Library.filter -> main.Library.filter..dec +callback_registered METHOD_REFERENCE main.defer_result -> main.defer_result..decorator +callback_registered METHOD_REFERENCE main.defer_result..decorator -> main.defer_result..decorator..wrapper known_edge DECORATOR_APPLICATION main. -> main.Library.filter..dec known_edge DECORATOR_BARE main. -> main.defer_text known_edge DECORATOR_CALL main. -> main.Library.filter diff --git a/graph/test/python/expected/27-returned-expression.tiers b/graph/test/python/expected/27-returned-expression.tiers index f44e09e11..61c44026e 100644 --- a/graph/test/python/expected/27-returned-expression.tiers +++ b/graph/test/python/expected/27-returned-expression.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 23 +distinct call sites emitted: 26 --- by tier: edge ROWS, and the distinct SITES they cover --- 2 rows 2 sites ambiguous_unknown 6 rows 6 sites boundary_lib + 3 rows 3 sites callback_registered 15 rows 15 sites known_edge 2 rows 1 sites multi_inferred @@ -12,6 +13,7 @@ distinct call sites emitted: 23 3 DECORATOR_CALL 1 DYNAMIC_CALL 5 METHOD_CALL + 3 METHOD_REFERENCE 3 SELF_CALL 10 SIMPLE_CALL @@ -27,7 +29,7 @@ distinct call sites emitted: 23 1 multi_inferred --- reconciling rows against the conserved site count --- - edge rows 25 + edge rows 28 minus extra rows from multi-target sites 1 - = tier/site pairs 24 + = tier/site pairs 27 engine's conserved site total 23 diff --git a/graph/test/python/expected/28-assigned-members.edges b/graph/test/python/expected/28-assigned-members.edges index c1af28b71..355e252d5 100644 --- a/graph/test/python/expected/28-assigned-members.edges +++ b/graph/test/python/expected/28-assigned-members.edges @@ -1,6 +1,8 @@ boundary_lib SIMPLE_CALL main. -> external:TypeVar boundary_lib SIMPLE_CALL main.Schema.__init__ -> builtin:object.__init__ boundary_lib SIMPLE_CALL main.main -> builtin:print +callback_registered METHOD_REFERENCE main.init -> main.Internals.parse +callback_registered METHOD_REFERENCE main.init -> main.init.._run known_edge SIMPLE_CALL main. -> main.main known_edge SIMPLE_CALL main.main -> main.Schema.__init__ known_edge SIMPLE_CALL main.main -> main.init diff --git a/graph/test/python/expected/28-assigned-members.tiers b/graph/test/python/expected/28-assigned-members.tiers index 4f1ba55ef..c4536e576 100644 --- a/graph/test/python/expected/28-assigned-members.tiers +++ b/graph/test/python/expected/28-assigned-members.tiers @@ -1,12 +1,14 @@ -distinct call sites emitted: 19 +distinct call sites emitted: 21 --- by tier: edge ROWS, and the distinct SITES they cover --- 4 rows 4 sites boundary_lib + 2 rows 2 sites callback_registered 7 rows 7 sites known_edge 30 rows 8 sites multi_inferred --- edge rows by call kind --- 30 METHOD_CALL + 2 METHOD_REFERENCE 11 SIMPLE_CALL --- unresolved reasons --- @@ -19,7 +21,7 @@ distinct call sites emitted: 19 8 multi_inferred --- reconciling rows against the conserved site count --- - edge rows 41 + edge rows 43 minus extra rows from multi-target sites 22 - = tier/site pairs 19 + = tier/site pairs 21 engine's conserved site total 19 diff --git a/graph/test/python/expected/33-url-view-forms.edges b/graph/test/python/expected/33-url-view-forms.edges index c05006003..0de9e0620 100644 --- a/graph/test/python/expected/33-url-view-forms.edges +++ b/graph/test/python/expected/33-url-view-forms.edges @@ -6,6 +6,16 @@ boundary_lib SIMPLE_CALL catalog.urls. -> external:path boundary_lib SIMPLE_CALL shop.urls. -> external:include boundary_lib SIMPLE_CALL shop.urls. -> external:path boundary_lib SIMPLE_CALL shop.urls. -> external:re_path +callback_registered METHOD_REFERENCE catalog.urls. -> catalog.views.catalog_index +callback_registered METHOD_REFERENCE shop.decorators.audited -> shop.decorators.audited..wrapper +callback_registered METHOD_REFERENCE shop.urls. -> shop.decorators.audited..wrapper +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.api_status +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_credit_note +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_draft +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_export +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_legacy +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_list +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_report known_edge DECORATOR_BARE shop.views. -> shop.decorators.audited known_edge SIMPLE_CALL shop.decorators.audited..wrapper -> shop.views.order_history known_edge SIMPLE_CALL shop.views.BaseWidgetView.get -> shop.views._render diff --git a/graph/test/python/expected/33-url-view-forms.tiers b/graph/test/python/expected/33-url-view-forms.tiers index f05de8d3f..fd0afb6f1 100644 --- a/graph/test/python/expected/33-url-view-forms.tiers +++ b/graph/test/python/expected/33-url-view-forms.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 51 +distinct call sites emitted: 61 --- by tier: edge ROWS, and the distinct SITES they cover --- 6 rows 6 sites ambiguous_unknown 27 rows 27 sites boundary_lib + 10 rows 10 sites callback_registered 18 rows 18 sites known_edge --- edge rows by call kind --- @@ -10,6 +11,7 @@ distinct call sites emitted: 51 5 DECORATOR_BARE 1 DECORATOR_CALL 1 METHOD_CALL + 10 METHOD_REFERENCE 43 SIMPLE_CALL --- unresolved reasons --- @@ -24,7 +26,7 @@ distinct call sites emitted: 51 18 known_edge --- reconciling rows against the conserved site count --- - edge rows 51 + edge rows 61 minus extra rows from multi-target sites 0 - = tier/site pairs 51 + = tier/site pairs 61 engine's conserved site total 51 diff --git a/graph/test/python/expected/36-call-through-model-decorators.edges b/graph/test/python/expected/36-call-through-model-decorators.edges index 266e2812c..69c96d257 100644 --- a/graph/test/python/expected/36-call-through-model-decorators.edges +++ b/graph/test/python/expected/36-call-through-model-decorators.edges @@ -6,6 +6,7 @@ boundary_lib DECORATOR_CALL app.orders. -> external:pydantic.validate_ca boundary_lib METHOD_CALL app.checkout.show -> builtin:float.hex boundary_lib SIMPLE_CALL app.orders.apply_discount -> builtin:round boundary_lib SIMPLE_CALL app.orders.apply_tax -> builtin:round +callback_registered METHOD_REFERENCE app.orders.audit -> app.orders.audit..wrapper known_edge DECORATOR_BARE app.orders. -> app.orders.audit known_edge PROPERTY_READ app.checkout.show -> app.orders.Order.grand_total known_edge PROPERTY_READ app.checkout.show -> app.orders.Order.net_total diff --git a/graph/test/python/expected/36-call-through-model-decorators.tiers b/graph/test/python/expected/36-call-through-model-decorators.tiers index 2e14b72cd..e9d9a11ce 100644 --- a/graph/test/python/expected/36-call-through-model-decorators.tiers +++ b/graph/test/python/expected/36-call-through-model-decorators.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 19 +distinct call sites emitted: 20 --- by tier: edge ROWS, and the distinct SITES they cover --- 4 rows 4 sites ambiguous_unknown 6 rows 6 sites boundary_lib + 1 rows 1 sites callback_registered 9 rows 9 sites known_edge --- edge rows by call kind --- @@ -10,6 +11,7 @@ distinct call sites emitted: 19 6 DECORATOR_BARE 1 DECORATOR_CALL 1 METHOD_CALL + 1 METHOD_REFERENCE 3 PROPERTY_READ 7 SIMPLE_CALL @@ -24,8 +26,8 @@ distinct call sites emitted: 19 6 known_edge --- reconciling rows against the conserved site count --- - edge rows 19 + edge rows 20 minus extra rows from multi-target sites 0 - = tier/site pairs 19 + = tier/site pairs 20 engine's conserved site total 16 of which 3 are PROPERTY_READ, an edge with no call site (README decision 5) diff --git a/graph/test/python/expected/37-model-signals.edges b/graph/test/python/expected/37-model-signals.edges index 675318292..aed5cf7af 100644 --- a/graph/test/python/expected/37-model-signals.edges +++ b/graph/test/python/expected/37-model-signals.edges @@ -9,4 +9,5 @@ boundary_lib DECORATOR_CALL models. -> external:receiver boundary_lib METHOD_CALL models. -> external:post_save.connect boundary_lib METHOD_CALL models.Invoice. -> external:models.IntegerField boundary_lib METHOD_CALL models.Order. -> external:models.CharField +callback_registered METHOD_REFERENCE models. -> models.audit_order known_edge METHOD_CALL views.save_cache -> views.Cache.save diff --git a/graph/test/python/expected/37-model-signals.tiers b/graph/test/python/expected/37-model-signals.tiers index e87fd3fad..10a3c4b9a 100644 --- a/graph/test/python/expected/37-model-signals.tiers +++ b/graph/test/python/expected/37-model-signals.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 14 +distinct call sites emitted: 15 --- by tier: edge ROWS, and the distinct SITES they cover --- 8 rows 8 sites ambiguous_unknown 5 rows 5 sites boundary_lib + 1 rows 1 sites callback_registered 1 rows 1 sites known_edge --- edge rows by call kind --- 2 DECORATOR_APPLICATION 2 DECORATOR_CALL 10 METHOD_CALL + 1 METHOD_REFERENCE --- unresolved reasons --- 2 decorator_factory_result_untyped @@ -23,7 +25,7 @@ distinct call sites emitted: 14 1 known_edge --- reconciling rows against the conserved site count --- - edge rows 14 + edge rows 15 minus extra rows from multi-target sites 0 - = tier/site pairs 14 + = tier/site pairs 15 engine's conserved site total 14 diff --git a/graph/test/python/expected/38-app-wiring.edges b/graph/test/python/expected/38-app-wiring.edges index 39d93d28d..8813d217c 100644 --- a/graph/test/python/expected/38-app-wiring.edges +++ b/graph/test/python/expected/38-app-wiring.edges @@ -17,6 +17,12 @@ boundary_lib SIMPLE_CALL main. -> external:Starlette boundary_lib SIMPLE_CALL main. -> external:WebSocketRoute boundary_lib SIMPLE_CALL shop.jobs.nightly -> external:call_command boundary_lib SIMPLE_CALL shop.jobs.unknown -> external:call_command +callback_registered METHOD_REFERENCE main. -> main.cancel +callback_registered METHOD_REFERENCE main. -> main.feed +callback_registered METHOD_REFERENCE main. -> main.health +callback_registered METHOD_REFERENCE main. -> main.lifespan +callback_registered METHOD_REFERENCE main. -> main.plugin_hook +callback_registered METHOD_REFERENCE main. -> main.tick known_edge METHOD_CALL main. -> main.Registry.add_route known_edge SIMPLE_CALL main. -> main.configure known_edge SIMPLE_CALL shop.management.commands.close_orders.Command.handle -> shop.orders.close_all diff --git a/graph/test/python/expected/38-app-wiring.tiers b/graph/test/python/expected/38-app-wiring.tiers index 8496a4b97..ebe6e2258 100644 --- a/graph/test/python/expected/38-app-wiring.tiers +++ b/graph/test/python/expected/38-app-wiring.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 26 +distinct call sites emitted: 32 --- by tier: edge ROWS, and the distinct SITES they cover --- 8 rows 8 sites ambiguous_unknown 15 rows 15 sites boundary_lib + 6 rows 6 sites callback_registered 3 rows 3 sites known_edge --- edge rows by call kind --- @@ -11,6 +12,7 @@ distinct call sites emitted: 26 1 DECORATOR_BARE 4 DECORATOR_CALL 3 METHOD_CALL + 6 METHOD_REFERENCE 13 SIMPLE_CALL --- unresolved reasons --- @@ -25,7 +27,7 @@ distinct call sites emitted: 26 3 known_edge --- reconciling rows against the conserved site count --- - edge rows 26 + edge rows 32 minus extra rows from multi-target sites 0 - = tier/site pairs 26 + = tier/site pairs 32 engine's conserved site total 26 diff --git a/graph/test/python/expected/39-model-construction-hooks.edges b/graph/test/python/expected/39-model-construction-hooks.edges index 3c8b20b60..705a286e3 100644 --- a/graph/test/python/expected/39-model-construction-hooks.edges +++ b/graph/test/python/expected/39-model-construction-hooks.edges @@ -14,5 +14,7 @@ boundary_lib SIMPLE_CALL models.Widget.check_size -> builtin:max boundary_lib SIMPLE_CALL models.Widget.dump_size -> builtin:str boundary_lib SIMPLE_CALL service.make_gizmo -> builtin:object.__init__ boundary_lib SIMPLE_CALL service.make_plain -> builtin:object.__init__ +callback_registered METHOD_REFERENCE models.Gizmo. -> models.new_tags +callback_registered METHOD_REFERENCE models.Widget. -> models.new_id known_edge SELF_CALL models.Gadget.__init__ -> models.Gadget.check_size known_edge SIMPLE_CALL service.make_gadget -> models.Gadget.__init__ diff --git a/graph/test/python/expected/39-model-construction-hooks.tiers b/graph/test/python/expected/39-model-construction-hooks.tiers index a4e4b8aff..d1f820899 100644 --- a/graph/test/python/expected/39-model-construction-hooks.tiers +++ b/graph/test/python/expected/39-model-construction-hooks.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 21 +distinct call sites emitted: 23 --- by tier: edge ROWS, and the distinct SITES they cover --- 8 rows 8 sites ambiguous_unknown 11 rows 11 sites boundary_lib + 2 rows 2 sites callback_registered 2 rows 2 sites known_edge --- edge rows by call kind --- @@ -10,6 +11,7 @@ distinct call sites emitted: 21 3 DECORATOR_BARE 3 DECORATOR_CALL 2 METHOD_CALL + 2 METHOD_REFERENCE 1 SELF_CALL 9 SIMPLE_CALL @@ -27,7 +29,7 @@ distinct call sites emitted: 21 2 known_edge --- reconciling rows against the conserved site count --- - edge rows 21 + edge rows 23 minus extra rows from multi-target sites 0 - = tier/site pairs 21 + = tier/site pairs 23 engine's conserved site total 21 diff --git a/graph/test/python/expected/41-di-marker-forms.edges b/graph/test/python/expected/41-di-marker-forms.edges index 0a449dee2..3cc836455 100644 --- a/graph/test/python/expected/41-di-marker-forms.edges +++ b/graph/test/python/expected/41-di-marker-forms.edges @@ -8,3 +8,11 @@ boundary_lib SIMPLE_CALL handlers. -> external:APIRouter boundary_lib SIMPLE_CALL handlers. -> external:Depends boundary_lib SIMPLE_CALL handlers. -> external:FastAPI boundary_lib SIMPLE_CALL other. -> external:Depends +callback_registered METHOD_REFERENCE deps. -> deps.load_cart +callback_registered METHOD_REFERENCE deps.ShapeFilter. -> deps.load_shape_size +callback_registered METHOD_REFERENCE handlers. -> deps.check_app +callback_registered METHOD_REFERENCE handlers. -> deps.check_quota +callback_registered METHOD_REFERENCE handlers. -> deps.check_region +callback_registered METHOD_REFERENCE handlers. -> deps.load_owner +callback_registered METHOD_REFERENCE handlers. -> deps.load_user +callback_registered METHOD_REFERENCE other. -> other.check_other diff --git a/graph/test/python/expected/41-di-marker-forms.tiers b/graph/test/python/expected/41-di-marker-forms.tiers index 85ad730a4..e2ec4f880 100644 --- a/graph/test/python/expected/41-di-marker-forms.tiers +++ b/graph/test/python/expected/41-di-marker-forms.tiers @@ -1,12 +1,14 @@ -distinct call sites emitted: 32 +distinct call sites emitted: 40 --- by tier: edge ROWS, and the distinct SITES they cover --- 7 rows 7 sites ambiguous_unknown 25 rows 25 sites boundary_lib + 8 rows 8 sites callback_registered --- edge rows by call kind --- 7 DECORATOR_APPLICATION 8 DECORATOR_CALL + 8 METHOD_REFERENCE 17 SIMPLE_CALL --- unresolved reasons --- @@ -18,7 +20,7 @@ distinct call sites emitted: 32 25 boundary_lib --- reconciling rows against the conserved site count --- - edge rows 32 + edge rows 40 minus extra rows from multi-target sites 0 - = tier/site pairs 32 + = tier/site pairs 40 engine's conserved site total 32 diff --git a/graph/test/python/expected/42-signal-forms.edges b/graph/test/python/expected/42-signal-forms.edges index 456f0118c..f8aa33ff1 100644 --- a/graph/test/python/expected/42-signal-forms.edges +++ b/graph/test/python/expected/42-signal-forms.edges @@ -4,6 +4,7 @@ ambiguous_unknown METHOD_CALL orders.close -> - ambiguous_unknown METHOD_CALL orders.ship -> - boundary_lib DECORATOR_CALL handlers. -> external:receiver boundary_lib SIMPLE_CALL signals. -> builtin:object.__init__ +callback_registered METHOD_REFERENCE handlers. -> handlers.on_closed known_edge DECORATOR_CALL handlers. -> handlers.remember known_edge METHOD_CALL orders.not_a_signal -> orders.Outbox.asend known_edge METHOD_CALL orders.pay -> signals.Signal.asend diff --git a/graph/test/python/expected/42-signal-forms.tiers b/graph/test/python/expected/42-signal-forms.tiers index e780b18a8..f608211b9 100644 --- a/graph/test/python/expected/42-signal-forms.tiers +++ b/graph/test/python/expected/42-signal-forms.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 22 +distinct call sites emitted: 23 --- by tier: edge ROWS, and the distinct SITES they cover --- 8 rows 8 sites ambiguous_unknown 9 rows 9 sites boundary_lib + 1 rows 1 sites callback_registered 5 rows 5 sites known_edge --- edge rows by call kind --- 5 DECORATOR_APPLICATION 5 DECORATOR_CALL 7 METHOD_CALL + 1 METHOD_REFERENCE 5 SIMPLE_CALL --- unresolved reasons --- @@ -22,7 +24,7 @@ distinct call sites emitted: 22 5 known_edge --- reconciling rows against the conserved site count --- - edge rows 22 + edge rows 23 minus extra rows from multi-target sites 0 - = tier/site pairs 22 + = tier/site pairs 23 engine's conserved site total 22 diff --git a/graph/test/python/expected/project-name-collision.tiers b/graph/test/python/expected/project-name-collision.tiers index ee405f873..d47e460f7 100644 --- a/graph/test/python/expected/project-name-collision.tiers +++ b/graph/test/python/expected/project-name-collision.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 91 +distinct call sites emitted: 92 --- by tier: edge ROWS, and the distinct SITES they cover --- 1 rows 1 sites ambiguous_unknown 34 rows 34 sites boundary_lib + 1 rows 1 sites callback_registered 49 rows 49 sites known_edge 25 rows 7 sites multi_inferred @@ -12,6 +13,7 @@ distinct call sites emitted: 91 1 DECORATOR_BARE 1 DECORATOR_CALL 25 METHOD_CALL + 1 METHOD_REFERENCE 5 SELF_CALL 58 SIMPLE_CALL 6 SUPER_CALL @@ -27,7 +29,7 @@ distinct call sites emitted: 91 7 multi_inferred --- reconciling rows against the conserved site count --- - edge rows 109 + edge rows 110 minus extra rows from multi-target sites 18 - = tier/site pairs 91 + = tier/site pairs 92 engine's conserved site total 91 diff --git a/graph/test/python/expected/project-two-service-fastapi.tiers b/graph/test/python/expected/project-two-service-fastapi.tiers index 0b54e3f31..d50f26912 100644 --- a/graph/test/python/expected/project-two-service-fastapi.tiers +++ b/graph/test/python/expected/project-two-service-fastapi.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 229 +distinct call sites emitted: 239 --- by tier: edge ROWS, and the distinct SITES they cover --- 10 rows 10 sites ambiguous_unknown 71 rows 71 sites boundary_lib + 10 rows 10 sites callback_registered 126 rows 126 sites known_edge 59 rows 22 sites multi_inferred @@ -12,6 +13,7 @@ distinct call sites emitted: 229 8 DECORATOR_BARE 9 DECORATOR_CALL 82 METHOD_CALL + 10 METHOD_REFERENCE 2 PROPERTY_READ 27 SELF_CALL 104 SIMPLE_CALL @@ -29,8 +31,8 @@ distinct call sites emitted: 229 22 multi_inferred --- reconciling rows against the conserved site count --- - edge rows 266 + edge rows 276 minus extra rows from multi-target sites 37 - = tier/site pairs 229 + = tier/site pairs 239 engine's conserved site total 227 of which 2 are PROPERTY_READ, an edge with no call site (README decision 5) diff --git a/graph/test/python/tools/literal_gate.py b/graph/test/python/tools/literal_gate.py index 0e7af6f71..b42121347 100644 --- a/graph/test/python/tools/literal_gate.py +++ b/graph/test/python/tools/literal_gate.py @@ -13,7 +13,9 @@ provenance "client" / "lib" -- assigned by which fact file a row is staged into, not by anything in the source. confidence tiers "known_edge", "multi_inferred", "boundary_lib", - "ambiguous_unknown" -- the engine's own output vocabulary. + "ambiguous_unknown", "callback_registered" (a method + reference, call-edge-generation/method-reference.dl) -- the + engine's own output vocabulary. diagnostic labels "no_rule", "escape_hatch", ... These are OUTPUT arguments of call_unresolvable / site_reason / expr_type_untypable and are never matched against input, so they cannot bias resolution. @@ -48,7 +50,7 @@ raise SystemExit(f"literal gate: engine dir not found at {ENGINE} -- refusing to pass vacuously") PROVENANCE = {"client", "lib", "external", "builtin"} -TIERS = {"known_edge", "multi_inferred", "boundary_lib", "ambiguous_unknown"} +TIERS = {"known_edge", "multi_inferred", "boundary_lib", "ambiguous_unknown", "callback_registered"} SCAFFOLD = {"", "-", ".", "/", "true", "false", "lib:", "builtin:", "external:", "untyped_receiver:", "builtin:object.__init__", "_total_sites"} # "/" joins "." above: both are separators fixed by a specification (a dotted name, a URL diff --git a/graph/test/python/torture/expected/torture.edges b/graph/test/python/torture/expected/torture.edges index 557d3208c..54c66e115 100644 --- a/graph/test/python/torture/expected/torture.edges +++ b/graph/test/python/torture/expected/torture.edges @@ -350,6 +350,11 @@ boundary_lib SUBSCRIPT_CALL f30_builtin_elements.an_annotated_local -> builtin:s boundary_lib SUBSCRIPT_CALL f30_builtin_elements.annotated_parameter_indexed -> builtin:str.upper boundary_lib SUBSCRIPT_CALL f30_builtin_elements.through_any_object -> builtin:str.upper boundary_lib SUPER_CALL f41_class_attribute_absent.ModelMeta.__new__ -> builtin:type.__new__ +callback_registered METHOD_REFERENCE f12_value_flow.method_alias_on_class -> f12_value_flow.Widget.make +callback_registered METHOD_REFERENCE f12_value_flow.method_alias_on_instance -> f12_value_flow.Widget.emit +callback_registered METHOD_REFERENCE rxfacade. -> rxaccel.boost +callback_registered METHOD_REFERENCE rxfacade. -> rxlib.fallback +callback_registered METHOD_REFERENCE rxfacade. -> rxlib.slow known_edge CHAINED_CALL f01_inheritance.client_chain -> f01_inheritance.LocalBase.kind known_edge CHAINED_CALL f01_inheritance.client_chain -> f01_inheritance.LocalLeaf.kind known_edge CHAINED_CALL f02_callables.attribute_callables -> f02_callables.HoldsCallables.run diff --git a/graph/test/tools/bundle-test.sh b/graph/test/tools/bundle-test.sh index 06ead93d8..8f3245bdf 100755 --- a/graph/test/tools/bundle-test.sh +++ b/graph/test/tools/bundle-test.sh @@ -310,6 +310,30 @@ if ! diff -q <(BUNDLE --print-schema) "$ROOT/graph/bundle/SCHEMA.md" >/dev/null; bad "graph/bundle/SCHEMA.md is stale — run: npm run schema-doc" fi +# 8. every ext relation a script queries is in EXT_QUERIED, so it keeps its c0 index (catalog.ts). The others are +# loaded unindexed: a new query on one of them would still answer, only slower, which no other check would see. +missing="$(python3 - "$ROOT" <<'PY' +import os, re, sys +root = sys.argv[1] +rels = set() +for lang in os.listdir(os.path.join(root, 'graph')): + m = os.path.join(root, 'graph', lang, 'souffle', 'export_manifest.tsv') + if os.path.isfile(m): + rels |= {l.split('\t')[0].strip() for l in open(m) if l.strip() and not l.lstrip().startswith('#')} +cat = open(os.path.join(root, 'graph', 'bundle', 'catalog.ts')).read() +queried = set(re.findall(r"'(\w+)'", cat[cat.index('EXT_QUERIED'):cat.index(']);', cat.index('EXT_QUERIED'))])) +used = set() +for d in ('plugins', 'hooks', 'bin'): + for dp, dns, fns in os.walk(os.path.join(root, d)): + dns[:] = [x for x in dns if x not in ('node_modules', 'dist', '.git')] + for fn in fns: + try: text = open(os.path.join(dp, fn), errors='replace').read() + except OSError: continue + used |= {r for r in re.findall(r'\bext_(\w+)', text) if r in rels} +print(' '.join(sorted(used - queried))) +PY +)" +[ -z "$missing" ] || bad "a script queries ext relation(s) missing from EXT_QUERIED in graph/bundle/catalog.ts (no c0 index): $missing" # ── the DEFAULT writes the database and nothing else ──────────────────────── # The deliverable is graph.sqlite. csv/*.csv carries the same core tables, so writing @@ -323,7 +347,7 @@ if [ -d "$d/csv" ] && [ -n "$(ls -A "$d/csv" 2>/dev/null)" ]; then fi if [ "$fail" -eq 0 ]; then - echo "bundle: ok (3 languages$( [ "$HAVE_SQLITE" = 1 ] && echo ', one schema across them, canonical queries run' || echo ', csv only — no sqlite3 CLI'), schema doc current)" + echo "bundle: ok (3 languages$( [ "$HAVE_SQLITE" = 1 ] && echo ', one schema across them, canonical queries run' || echo ', csv only — no sqlite3 CLI'), schema doc current, queried ext relations indexed)" else echo "bundle: $fail failure(s)"; exit 1 fi diff --git a/graph/test/tools/stage-rows-test.sh b/graph/test/tools/stage-rows-test.sh new file mode 100644 index 000000000..67abcca69 --- /dev/null +++ b/graph/test/tools/stage-rows-test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# Staging copies every IR CSV into a Soufflé fact file and drops the rows whose field count differs +# from the header's (graph/pipeline/stage-rows.sh). It runs perl where perl exists and awk where it +# does not, so the two must print the same bytes: a difference would make the facts, and so the +# graph, a function of which tools the machine happens to have. +# +# Each fixture below is staged by both, and the stdout bytes and the stderr report must match. The +# fixtures are the shapes where a line-based tab counter and awk's field splitting could part: a +# trailing tab (an empty last field), an empty line, CRLF, a last line with no newline, a header with +# no newline, an empty file, a quoted field holding a tab (both see two fields: neither parses quotes), +# bytes that are not UTF-8, and torn rows of both kinds (too few fields, too many). +# A NUL byte is left out on purpose: BSD awk ends the line at it (and drops the row as torn) while gawk +# and perl keep the whole line, so awk does not agree with itself there and is no reference. +# ───────────────────────────────────────────────────────────────────────────── +set -uo pipefail +unset AXIOM_STAGE_WITH +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" + +command -v perl >/dev/null 2>&1 || { echo "stage-rows-test: FAILED (no perl on PATH; this test compares perl against awk)"; exit 1; } + +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +fail=0; checks=0 +bad(){ checks=$((checks+1)); printf ' FAIL %s\n' "$1"; fail=1; } +ok(){ checks=$((checks+1)); } + +# The tool the sourced file picks: perl here, awk when forced. +( . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = perl ] ) && ok || bad "stage-rows.sh did not pick perl although perl is on PATH" +( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = awk ] ) && ok || bad "AXIOM_STAGE_WITH=awk did not force awk" + +printf 'a\tb\tc\n1\t2\t3\n4\t5\t6\n' > "$W/clean.csv" +printf 'a\tb\tc\n1\t2\t3\n1\t2\n1\t2\t3\t4\n7\t8\t9\n' > "$W/torn.csv" +printf 'a\tb\tc\n1\t2\t\n\t\t\n1\t2\t3\t\n' > "$W/trailing-tab.csv" +printf 'a\tb\tc\n1\t2\t3\n\n4\t5\t6\n\n' > "$W/empty-line.csv" +printf 'a\tb\tc\r\n1\t2\t3\r\n4\t5\r\n' > "$W/crlf.csv" +printf 'a\tb\tc\n1\t2\t3\n4\t5\t6' > "$W/no-final-newline.csv" +printf 'a\tb\tc\n1\t2' > "$W/torn-last-line.csv" +printf 'a\tb\tc' > "$W/header-only-no-newline.csv" +printf 'a\tb\tc\n' > "$W/header-only.csv" +: > "$W/empty.csv" +printf 'a\tb\n"x\ty"\tz\n"p\tq"\n' > "$W/quoted-tab.csv" +printf 'a\tb\n\xff\xfe\t\xc3\x28\n' > "$W/bytes.csv" +printf 'a\n1\n\n2\n' > "$W/one-column.csv" +printf '\n\n1\n\n' > "$W/empty-header.csv" + +n=0 +for f in "$W"/*.csv; do + n=$((n+1)); name="$(basename "$f")" + ( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/p.out" 2> "$W/p.err"; prc=$? + ( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/a.out" 2> "$W/a.err"; arc=$? + [ "$prc" = 0 ] && [ "$arc" = 0 ] && ok || bad "$name: exit status perl=$prc awk=$arc" + cmp -s "$W/p.out" "$W/a.out" && ok || { bad "$name: staged rows differ"; od -c "$W/a.out" | sed 's/^/ awk /'; od -c "$W/p.out" | sed 's/^/ perl /'; } + cmp -s "$W/p.err" "$W/a.err" && ok || bad "$name: report differs: perl '$(cat "$W/p.err")' awk '$(cat "$W/a.err")'" +done + +# Not vacuous: the fixtures must hold rows that are kept and rows that are dropped. +( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/torn.csv" ) > "$W/t.out" 2> "$W/t.err" +[ "$(printf '1\t2\t3\n7\t8\t9\n')" = "$(cat "$W/t.out")" ] && ok || bad "torn.csv: expected the two whole rows, got '$(cat "$W/t.out")'" +grep -q "dropped 2 malformed row(s) from $W/torn.csv" "$W/t.err" && ok || bad "torn.csv: expected a report of 2 dropped rows, got '$(cat "$W/t.err")'" +# The control: a clean file comes through as its rows, header removed, nothing reported. +( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/clean.csv" ) > "$W/c.out" 2> "$W/c.err" +tail -n +2 "$W/clean.csv" | cmp -s - "$W/c.out" && [ ! -s "$W/c.err" ] && ok || bad "clean.csv: not passed through unchanged" + +[ "$n" -ge 14 ] || bad "only $n fixtures were staged" +[ $fail = 0 ] && echo "stage-rows-test: ok ($checks checks over $n fixtures)" || { echo "stage-rows-test: FAILED"; exit 1; } diff --git a/graph/typescript/engine/expression-resolution/expr-type.dl b/graph/typescript/engine/expression-resolution/expr-type.dl index d8b6b23b4..06396ef12 100644 --- a/graph/typescript/engine/expression-resolution/expr-type.dl +++ b/graph/typescript/engine/expression-resolution/expr-type.dl @@ -477,6 +477,10 @@ expr_static_type(e, prov, t) :- property_access_recv(e, n, q), expr_callable(e, cm) :- property_access_recv(e, n, q), expr_namespace(q, _, m), module_export(_, m, n, "METHOD", cm). +// A namespace exported from a namespace — `z.coerce`, `z.iso` — is itself a module. +expr_namespace(e, prov, ns) :- property_access_recv(e, n, q), + expr_namespace(q, _, m), + module_export_namespace(prov, m, n, ns). // A STATIC METHOD read as a VALUE — `Util.inSorted` on its own, not called. The // receiver is the class itself, so the member comes from the STATIC table, exactly diff --git a/graph/typescript/engine/resolution/module-graph.dl b/graph/typescript/engine/resolution/module-graph.dl index 1fb223ece..628b0b088 100644 --- a/graph/typescript/engine/resolution/module-graph.dl +++ b/graph/typescript/engine/resolution/module-graph.dl @@ -382,8 +382,12 @@ module_export_runtime_row(prov, mod, en) :- export_decl(prov, en, _, _, mod, h), module_export(prov, mod, en, k, h) :- module_export_any(prov, mod, en, k, h), !module_export_type_only(prov, mod, en). +// EXPORT_STAR_AS_NAMESPACE is not a star. `export * as core from "./m"` exports ONE +// name, `core`, whose value is m's namespace; m's members are reachable only as +// `core.x`. Expanding it here put every member of m on this module's own surface, so +// a barrel holding `export * as coerce` beside `export * from "./schemas"` exported +// two `string`s. It is module_export_namespace's rule (b) instead. export_kind_is_star("EXPORT_STAR"). -export_kind_is_star("EXPORT_STAR_AS_NAMESPACE"). export_kind_is_star("TYPE_ONLY_STAR"). export_kind_is_local_name("NAMED_EXPORT"). export_kind_is_local_name("NAMED_ALIAS"). @@ -725,6 +729,75 @@ import_binds_namespace_any(h, prov, m) :- import_binding("client", ik, _, _, _, import_target_module(h, prov, m). import_binds_namespace(h, prov, m) :- import_binds_namespace_any(h, prov, m), !import_is_type_only("client", h). + +// ── module_export_namespace(NsProv, Module, ExportedName, NsModule) ───────── +// A module can export a NAMESPACE under a name, not only a declaration: +// +// import * as z from "./external"; export { z }; export default z; +// export * as core from "./core"; +// +// module_export_any cannot carry it — its rows name an entity, and a namespace is a +// module — so an importer's `import { z } from "pkg"` bound nothing, and every +// `z.string()` after it had an untyped receiver. Kept as its own relation for the +// reason import_binds_namespace is: the value is an export table, not a type. +// +// (a) a bare `export { z }` / `export { z as y }` / `export default z` whose local +// name is a namespace import binding in the same module. The import row resolves to +// its target, so this is a join, as in module_export_any (b2). +module_export_namespace(prov, mod, en, ns) :- export_decl(eprov, en, ln, ek, mod, h), + export_kind_names_local_binding(ek), + export_source(eprov, "", _, h), + ln != "", + import_binding(eprov, ik, ln, _, mod, ih), + import_kind_binds_namespace(ik), + namespace_import_target(eprov, ih, prov, ns). +// (b) `export * as core from "./m"`: the row's exported name is the namespace. +module_export_namespace(prov, mod, en, ns) :- export_decl(_, en, _, "EXPORT_STAR_AS_NAMESPACE", mod, h), + en != "", + export_target_module(h, prov, ns). +// (c) the name travels on: `export { z } from "./barrel"` and `export * from "./barrel"` +// carry a namespace export exactly as they carry a declaration (module_export_any (c), (d)). +module_export_namespace(prov, mod, en, ns) :- export_decl(_, en, ln, ek, mod, h), + export_kind_is_local_name(ek), + ln != "", + export_target_module(h, _, src), + module_export_namespace(prov, src, ln, ns). +module_export_namespace(prov, mod, en, ns) :- export_decl(_, _, _, ek, mod, h), + export_kind_is_star(ek), + export_target_module(h, _, src), + module_export_namespace(prov, src, en, ns), + en != "default". +// (d) a barrel that imports the namespace by name and exports it bare: +// `import { z } from "./inner"; export { z }`. +module_export_namespace(prov, mod, en, ns) :- export_decl(eprov, en, ln, ek, mod, h), + export_kind_names_local_binding(ek), + export_source(eprov, "", _, h), + ln != "", + import_binding(eprov, ik, ln, orig, mod, ih), + import_kind_binds_named(ik), + orig != "", + namespace_import_target(eprov, ih, _, m), + module_export_namespace(prov, m, orig, ns). + +export_kind_names_local_binding(ek) :- export_kind_is_local_name(ek). +export_kind_names_local_binding("DEFAULT_EXPRESSION"). + +// A client import row resolves through import_target_module, a library's own through +// lib_import_target; the barrel above may be either. +namespace_import_target("client", ih, prov, m) :- import_target_module(ih, prov, m). +namespace_import_target("lib", ih, prov, m) :- lib_import_target(ih, prov, m). + +// The importer's side: a named or default import of a name its target module exports +// as a namespace binds that namespace, and member calls on it read its export table. +import_binds_namespace_any(h, prov, ns) :- import_binding("client", ik, _, orig, _, h), + import_kind_binds_named(ik), + orig != "", + import_target_module(h, _, m), + module_export_namespace(prov, m, orig, ns). +import_binds_namespace_any(h, prov, ns) :- import_binding("client", ik, _, _, _, h), + import_kind_binds_default(ik), + import_target_module(h, _, m), + module_export_namespace(prov, m, "default", ns). // An export-assignment module imported as a namespace still exposes its members by // name, so it is reachable through the same relation. import_kind_binds_namespace("NAMESPACE"). diff --git a/graph/typescript/souffle/decls_all.dl b/graph/typescript/souffle/decls_all.dl index c8e368d11..be835f2c9 100644 --- a/graph/typescript/souffle/decls_all.dl +++ b/graph/typescript/souffle/decls_all.dl @@ -398,6 +398,9 @@ .decl module_decl(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol,c5:symbol) .decl module_default_export(c0:symbol,c1:symbol,c2:symbol) .decl module_export(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol) +.decl module_export_namespace(c0:symbol,c1:symbol,c2:symbol,c3:symbol) +.decl export_kind_names_local_binding(c0:symbol) +.decl namespace_import_target(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl module_export_any(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol) .decl module_export_assignment(c0:symbol,c1:symbol,c2:symbol) .decl module_export_recovered(c0:symbol,c1:symbol) diff --git a/package.json b/package.json index eac44e3dc..17f29b528 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@axiomcode/code-graph", - "version": "0.2.0", + "version": "0.2.1", "description": "AxiomCode Graph: a resolved call graph of your codebase grounded in formal methods, so you and your coding agents can see who calls what, what a change breaks, and which tests it reaches.", "repository": { "type": "git", @@ -31,11 +31,11 @@ ], "license": "FSL-1.1-Apache-2.0", "optionalDependencies": { - "@axiomcode/engine-darwin-arm64": "0.2.0", - "@axiomcode/engine-darwin-x64": "0.2.0", - "@axiomcode/engine-linux-x64": "0.2.0", - "@axiomcode/engine-linux-arm64": "0.2.0", - "@axiomcode/engine-win32-x64": "0.2.0" + "@axiomcode/engine-darwin-arm64": "0.2.1", + "@axiomcode/engine-darwin-x64": "0.2.1", + "@axiomcode/engine-linux-x64": "0.2.1", + "@axiomcode/engine-linux-arm64": "0.2.1", + "@axiomcode/engine-win32-x64": "0.2.1" }, "bin": { "axiomcode": "bin/axiomcode.js" diff --git a/parser/src/analysis-types/typescript/TsExpressionRegistry.ts b/parser/src/analysis-types/typescript/TsExpressionRegistry.ts index fafe44dad..56d406ebc 100644 --- a/parser/src/analysis-types/typescript/TsExpressionRegistry.ts +++ b/parser/src/analysis-types/typescript/TsExpressionRegistry.ts @@ -172,6 +172,17 @@ export class TsExpressionRegistry implements EntityIdentifiable { this.potentialQualifiedName = potentialQualifiedName; } + /** + * The tag of an intrinsic element (`div`, `svg:circle`, `my-el`), written as in the source. + * + * A component tag is a METHOD_NAME child that resolves; an intrinsic one references nothing, so its + * name has no child to live on and goes in this slot, with `referencedEntityKind` left empty. The + * engine's projections never read this column, so it cannot reach name resolution. + */ + setJsxTagName(tag: string): void { + this.potentialQualifiedName = tag; + } + getReferencedEntityHash(): string { return this.referencedEntityHash; } diff --git a/parser/src/deno-libraries.ts b/parser/src/deno-libraries.ts new file mode 100644 index 000000000..f8bb62ed3 --- /dev/null +++ b/parser/src/deno-libraries.ts @@ -0,0 +1,128 @@ +/** + * The library roots a Deno project's imports reach, for `--library auto`: + * + * node dist/deno-libraries.js + * + * prints one JSON object, `{"mirrors": ["

", …], "packages": ["", …]}`: + * + * mirrors each Deno project's mirror, filled by resolving every import: each + * `jsr:` and URL module the project reaches, and what those import, + * copied from Deno's cache as `vendor/` lays them out + * packages every npm package directory an `npm:` import resolved into, in + * Deno's npm cache or `node_modules` (real paths, because Deno's + * `node_modules` entries are symlinks into `node_modules/.deno`) + * + * Library discovery runs BEFORE the project is parsed, so the mirror has to be + * filled here rather than as a side effect of the parse. Resolution is the same + * code the parser uses, so the files staged are the files the imports name. + * A repository with no Deno config prints `{"mirrors": [], "packages": []}`. + */ +import * as fs from 'fs'; +import * as path from 'path'; + +import { configIn, denoContextForDir, isDenoVendorDirectory } from '@/parsers/typescript/deno/deno-config'; +import { mirrorRoot } from '@/parsers/typescript/deno/deno-cache'; +import { moduleSpecifiersOf, resolveDenoSpecifier } from '@/parsers/typescript/deno/deno-resolver'; + +const SKIP = new Set(['node_modules', '.git', 'dist', 'build', 'out', 'coverage', '.axiomcode']); +const SOURCE = /\.(tsx?|mts|cts|jsx?|mjs|cjs)$/; + +function sourceFiles(root: string): string[] { + const out: string[] = []; + const walk = (dir: string): void => { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (!SKIP.has(entry.name) && !entry.name.startsWith('.') && !isDenoVendorDirectory(dir, entry.name)) { + walk(full); + } + } else if (SOURCE.test(entry.name) && !entry.name.endsWith('.d.ts')) { + out.push(full); + } + } + }; + walk(root); + return out; +} + +/** + * `<…>/zod/3.25.76` for a file inside it: the nearest ancestor whose package.json + * names `packageName`. Nearest alone is not enough, because a package can ship a + * stub package.json per subpath (`zod/v4/package.json`). + */ +function packageDirOf(file: string, packageName: string): string | undefined { + let dir = path.dirname(file); + for (;;) { + const manifest = path.join(dir, 'package.json'); + if (fs.existsSync(manifest)) { + let name: unknown; + try { + name = (JSON.parse(fs.readFileSync(manifest, 'utf-8')) as { name?: unknown }).name; + } catch { + name = undefined; + } + if (name === packageName) { + try { + return fs.realpathSync(dir); + } catch { + return dir; + } + } + } + const parent = path.dirname(dir); + if (parent === dir) { + return undefined; + } + dir = parent; + } +} + +function main(): number { + const repo = path.resolve(process.argv[2] ?? '.'); + const mirrors = new Set(); + const packages = new Set(); + // Dependency specifiers that resolve to nothing: most often modules Deno has not + // downloaded (no `deno install` yet, or a cache the OS purged). + const unresolved = new Set(); + for (const file of sourceFiles(repo)) { + const context = denoContextForDir(path.dirname(file)); + if (context === undefined) { + continue; + } + let text: string; + try { + text = fs.readFileSync(file, 'utf-8'); + } catch { + continue; + } + for (const specifier of moduleSpecifiersOf(text, file)) { + const resolved = resolveDenoSpecifier(specifier, file); + if (resolved === undefined) { + if (!/^(\.{0,2}\/|[a-z][a-z0-9+.-]*:)/i.test(specifier) || /^(jsr|npm|https?):/.test(specifier)) { + unresolved.add(specifier); + } + continue; + } + if (resolved.kind === 'JSR' || resolved.kind === 'REMOTE') { + mirrors.add(mirrorRoot(context.root)); + } else if (resolved.kind === 'NPM_CACHE' || resolved.kind === 'NODE_MODULES') { + const dir = packageDirOf(resolved.resolvedFileName, resolved.packageName); + if (dir !== undefined) { + packages.add(dir); + } + } + } + } + // A mirror exists once something was copied into it; one that stayed empty stages nothing. + const filled = [...mirrors].filter((m) => configIn(m) !== undefined).sort(); + process.stdout.write(`${JSON.stringify({ mirrors: filled, packages: [...packages].sort(), unresolved: unresolved.size })}\n`); + return 0; +} + +process.exit(main()); diff --git a/parser/src/enums/SkippedFileReason.ts b/parser/src/enums/SkippedFileReason.ts index 43e650e42..928135fd0 100644 --- a/parser/src/enums/SkippedFileReason.ts +++ b/parser/src/enums/SkippedFileReason.ts @@ -66,4 +66,11 @@ export enum SkippedFileReason { * a consumer could not tell an unparsed file from an empty one (#660). */ NO_PROGRAM_CLAIMS_FILE = 'NO_PROGRAM_CLAIMS_FILE', + + /** + * A file a tsconfig claims that no program run over the root extracted. The program + * that owns it was never reached, so the file is absent from every relation; this + * says so instead of leaving it to look like a file with nothing in it. + */ + PROGRAM_NOT_EXTRACTED = 'PROGRAM_NOT_EXTRACTED', } diff --git a/parser/src/enums/typescript/expressions/TsExpressionKind.ts b/parser/src/enums/typescript/expressions/TsExpressionKind.ts index 6d42c2088..c0ada80a2 100644 --- a/parser/src/enums/typescript/expressions/TsExpressionKind.ts +++ b/parser/src/enums/typescript/expressions/TsExpressionKind.ts @@ -33,8 +33,10 @@ * A JSX element whose tag names a component IS a call to it, with the whole * props object as argument 0: JSX_ELEMENT / JSX_SELF_CLOSING, 1:1 with a call * site, with the tag as its METHOD_NAME child. Attribute values and children are - * rooted as their own JSX_EMBEDDED_EXPRESSION trees. An intrinsic element emits - * no row. + * rooted as their own JSX_EMBEDDED_EXPRESSION trees. An intrinsic element + * (`
`, ``, ``) is JSX_INTRINSIC_ELEMENT: a row with no + * children and no call site, so it exists only as the element its attributes + * belong to. * * Schema §4.14 c0, §4.15.1. */ @@ -99,6 +101,15 @@ export enum TsExpressionKind { JSX_ELEMENT = 'JSX_ELEMENT', /** `` naming a component. 1:1 with a call site. */ JSX_SELF_CLOSING = 'JSX_SELF_CLOSING', + /** + * `
`, ``, `` -- a tag that names no + * declaration. NOT a call: no call site, no METHOD_NAME child, so nothing can + * resolve it into `JSX.IntrinsicElements`. The row exists so that a plain + * tag's attributes (an event handler, a `ref` callback) have an element to + * belong to, as the JavaScript parser's JSX_INTRINSIC_ELEMENT does. The tag, + * as written (`div`, `svg:circle`, `my-el`), is its `potentialQualifiedName`. + */ + JSX_INTRINSIC_ELEMENT = 'JSX_INTRINSIC_ELEMENT', /** `delete a`, `typeof a`, `void a` — `operatorString` says which. */ DELETE_TYPEOF_VOID = 'DELETE_TYPEOF_VOID', /** `a, b` — the comma operator. */ diff --git a/parser/src/enums/typescript/imports/TsImportResolutionKind.ts b/parser/src/enums/typescript/imports/TsImportResolutionKind.ts index bc36e385b..a82c0f3a0 100644 --- a/parser/src/enums/typescript/imports/TsImportResolutionKind.ts +++ b/parser/src/enums/typescript/imports/TsImportResolutionKind.ts @@ -33,7 +33,7 @@ export enum TsImportResolutionKind { /** A relative specifier that resolved to a file. */ RELATIVE_FILE = 'RELATIVE_FILE', - /** Resolved through a tsconfig `paths` mapping. */ + /** Resolved through a tsconfig `paths` mapping, or a Deno import-map entry onto a project file. */ PATHS_ALIAS = 'PATHS_ALIAS', /** Resolved to a `.d.ts` inside `node_modules`. */ NODE_MODULES_TYPES = 'NODE_MODULES_TYPES', @@ -46,6 +46,16 @@ export enum TsImportResolutionKind { * source its `main` / `types` / `exports` entry is built from. */ WORKSPACE_PACKAGE = 'WORKSPACE_PACKAGE', + /** + * A `jsr:` package (directly, or through a Deno import map), resolved to the + * version the lockfile pins and the file its `exports` names. The path is the + * module's copy in the project's Deno mirror, laid out as `vendor/` is. + */ + JSR_PACKAGE = 'JSR_PACKAGE', + /** An `npm:` package resolved inside Deno's own npm cache rather than `node_modules`. */ + DENO_NPM_CACHE = 'DENO_NPM_CACHE', + /** An `https:` module a Deno project imports by URL, as downloaded (redirects and `x-typescript-types` followed). */ + REMOTE_MODULE = 'REMOTE_MODULE', /** Matched a `declare module "x"` in this analysis. No file, and none needed. */ AMBIENT_MODULE = 'AMBIENT_MODULE', /** A Node builtin, with or without the `node:` prefix. */ diff --git a/parser/src/language-detectors/typescript-detector.ts b/parser/src/language-detectors/typescript-detector.ts index 753d5d23f..6ad6da2d8 100644 --- a/parser/src/language-detectors/typescript-detector.ts +++ b/parser/src/language-detectors/typescript-detector.ts @@ -37,6 +37,10 @@ export class TypeScriptDetector implements LanguageDetector { 'tsconfig.json', 'tsconfig.base.json', 'jsconfig.json', + // A Deno project's config defines its module graph the way a tsconfig does, and a + // workspace root's lists members that import one another: one project, not one per member. + 'deno.json', + 'deno.jsonc', ]; private static readonly SKIP = new Set(TS_SKIP_DIRECTORIES); @@ -62,6 +66,7 @@ export class TypeScriptDetector implements LanguageDetector { async detectBuildSystem(projectPath: string): Promise { try { const files = await fs.readdir(projectPath); + if (files.includes('deno.json') || files.includes('deno.jsonc')) return 'Deno'; if (files.includes('pnpm-lock.yaml')) return 'pnpm'; if (files.includes('yarn.lock')) return 'Yarn'; if (files.includes('bun.lockb') || files.includes('bun.lock')) return 'Bun'; diff --git a/parser/src/parsers/typescript/deno/deno-cache.ts b/parser/src/parsers/typescript/deno/deno-cache.ts new file mode 100644 index 000000000..7d333d536 --- /dev/null +++ b/parser/src/parsers/typescript/deno/deno-cache.ts @@ -0,0 +1,311 @@ +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { DenoConfig, readJsonc } from '@/parsers/typescript/deno/deno-config'; + +/** + * Where Deno keeps what a project downloaded, read without running Deno. + * + * ## The two stores + * + * - **Remote modules** (`jsr:` packages and `https:` imports) live in + * `$DENO_DIR/remote///`: a file with no + * extension whose body is the module and whose LAST line is a + * `// denoCacheMetadata={…}` comment carrying the response headers. With + * `"vendor": true` the same modules sit under `/vendor//`. + * - **npm packages** live unpacked in `$DENO_DIR/npm////` + * (or in `node_modules` when the project has one). + * + * ## The mirror + * + * A hashed, extensionless cache file is not something a parser walk can stage as + * a library, and its path names no package. So a remote module a project imports + * is copied to a MIRROR laid out exactly as `vendor/` is (`jsr.io/@std/path/1.1.6/ + * mod.ts`), one mirror per Deno project. An import then resolves to a real `.ts` + * file, and the mirror is one library root whose module paths carry the package + * and version, which is what keeps `@std/path`'s `mod.ts` apart from `@std/fs`'s + * when the engine matches a resolved path against library modules. + */ + +/** `$DENO_DIR`, or the platform default Deno itself uses. */ +export function denoDir(): string { + if (process.env.DENO_DIR) { + return path.resolve(process.env.DENO_DIR); + } + const home = os.homedir(); + if (process.platform === 'darwin') { + return path.join(home, 'Library', 'Caches', 'deno'); + } + if (process.platform === 'win32') { + return path.join(process.env.LOCALAPPDATA ?? path.join(home, 'AppData', 'Local'), 'deno'); + } + return path.join(process.env.XDG_CACHE_HOME ?? path.join(home, '.cache'), 'deno'); +} + +export interface RemoteModule { + readonly url: string; + readonly text: string; + /** Response headers, lower-cased. Empty for a vendored file. */ + readonly headers: Readonly>; +} + +const METADATA_MARKER = '\n// denoCacheMetadata='; + +/** + * A remote module as the project's Deno would load it: from `vendor/` when the + * project vendors, else from `$DENO_DIR`. `undefined` when it was never downloaded. + */ +export function readRemote(url: string, root: DenoConfig): RemoteModule | undefined { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return undefined; + } + if (root.vendor) { + const vendored = path.join(root.dir, 'vendor', hostDir(parsed), ...decodedSegments(parsed)); + const text = readText(vendored); + if (text !== undefined) { + return { url, text, headers: {} }; + } + } + const hostCache = path.join(denoDir(), 'remote', parsed.protocol.replace(/:$/, ''), hostDir(parsed)); + let cached = ''; + let raw: string | undefined; + for (const key of cacheKeysOf(parsed)) { + cached = path.join(hostCache, crypto.createHash('sha256').update(key).digest('hex')); + raw = readText(cached); + if (raw !== undefined) { + break; + } + } + if (raw === undefined) { + return undefined; + } + const at = raw.lastIndexOf(METADATA_MARKER); + if (at >= 0) { + return { url, text: raw.slice(0, at), headers: headersOf(raw.slice(at + METADATA_MARKER.length)) }; + } + // Deno before the inline trailer kept the headers in a sidecar file. + const sidecar = readText(`${cached}.metadata.json`); + return { url, text: raw, headers: sidecar === undefined ? {} : headersOf(sidecar) }; +} + +/** + * The path and query Deno hashed for `url`. Deno's URL parser leaves a `^` in a + * path as written (`esm.sh/type-fest@^4.11.1`), where this one escapes it to + * `%5E`, so the unescaped spelling is tried as well. + */ +function cacheKeysOf(url: URL): string[] { + const key = url.pathname + url.search; + const unescaped = url.pathname.replace(/%5E/gi, '^') + url.search; + return unescaped === key ? [key] : [key, unescaped]; +} + +function headersOf(json: string): Record { + try { + const meta = JSON.parse(json.trim()) as { headers?: Record }; + const out: Record = {}; + for (const [k, v] of Object.entries(meta.headers ?? {})) { + if (typeof v === 'string') { + out[k.toLowerCase()] = v; + } + } + return out; + } catch { + return {}; + } +} + +/** `deno.land`, or `localhost_PORT8000` as Deno spells a host with a port. */ +function hostDir(url: URL): string { + return url.port === '' ? url.hostname : `${url.hostname}_PORT${url.port}`; +} + +function decodedSegments(url: URL): string[] { + return url.pathname.split('/').filter((s) => s !== '').map((s) => decodeURIComponent(s)); +} + +function readText(file: string): string | undefined { + try { + return fs.readFileSync(file, 'utf-8'); + } catch { + return undefined; + } +} + +/** The versions a lockfile pins, per package name, for each registry. */ +export interface DenoLock { + readonly jsr: ReadonlyMap; + readonly npm: ReadonlyMap; + /** Remote URL to the URL it redirected to (`esm.sh/ky@1.7` to `esm.sh/ky@1.7.5`). */ + readonly redirects: ReadonlyMap; + readonly text: string; +} + +const locks = new Map(); + +/** The lockfile beside `root`, in any of the v3 to v5 layouts, or `undefined`. */ +export function readLock(root: DenoConfig): DenoLock | undefined { + if (root.lockPath === false) { + return undefined; + } + const cached = locks.get(root.lockPath); + if (cached !== undefined) { + return cached ?? undefined; + } + const text = readText(root.lockPath); + let result: DenoLock | undefined; + if (text !== undefined) { + const json = readJsonc(root.lockPath) as Record | undefined; + if (json !== undefined && json !== null && typeof json === 'object') { + // v4 and v5 keep `jsr` and `npm` at the top; v3 nests them under `packages`. + const packages = (json.packages ?? {}) as Record; + result = { + jsr: versionsByName(json.jsr ?? packages.jsr), + npm: versionsByName(json.npm ?? packages.npm), + redirects: redirectsOf(json.redirects), + text, + }; + } + } + locks.set(root.lockPath, result ?? null); + return result; +} + +function redirectsOf(section: unknown): Map { + const out = new Map(); + if (section !== null && typeof section === 'object') { + for (const [from, to] of Object.entries(section as Record)) { + if (typeof to === 'string') { + out.set(from, to); + } + } + } + return out; +} + +/** `{"@std/path@1.1.6": …, "zod@3.25.76_peer@1.0.0": …}` to name → versions. */ +function versionsByName(section: unknown): Map { + const out = new Map(); + if (section === null || typeof section !== 'object') { + return out; + } + for (const key of Object.keys(section as Record)) { + const at = key.indexOf('@', 1); + if (at < 0) { + continue; + } + const name = key.slice(0, at); + // An npm key carries its peer resolution after `_`; the version is before it. + const version = key.slice(at + 1).split('_')[0]!; + out.set(name, [...(out.get(name) ?? []), version]); + } + return out; +} + +/** Every version of an npm package unpacked in `$DENO_DIR/npm`, with its directory. */ +export function cachedNpmVersions(name: string): Map { + const out = new Map(); + const npmRoot = path.join(denoDir(), 'npm'); + let registries: string[] = []; + try { + registries = fs.readdirSync(npmRoot); + } catch { + return out; + } + // The public registry first, so a mirror registry holding the same version does not win. + registries.sort((a, b) => (b === 'registry.npmjs.org' ? 1 : 0) - (a === 'registry.npmjs.org' ? 1 : 0)); + for (const registry of registries) { + const dir = path.join(npmRoot, registry, ...name.split('/')); + let versions: string[] = []; + try { + versions = fs.readdirSync(dir); + } catch { + continue; + } + for (const version of versions) { + const pkgDir = path.join(dir, version); + if (!out.has(version) && fs.existsSync(path.join(pkgDir, 'package.json'))) { + out.set(version, pkgDir); + } + } + } + return out; +} + +/** + * The mirror directory for a Deno project: `$AXIOMCODE_DENO_MIRROR` when set, + * else one directory per project under the axiomcode cache. + */ +export function mirrorRoot(root: DenoConfig): string { + // A library parse OF a mirror resolves inside it rather than minting a mirror of the mirror. + if (fs.existsSync(path.join(root.dir, MIRROR_MARKER))) { + return root.dir; + } + if (process.env.AXIOMCODE_DENO_MIRROR) { + return path.resolve(process.env.AXIOMCODE_DENO_MIRROR); + } + const base = process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), '.cache'); + let real = root.dir; + try { + real = fs.realpathSync(root.dir); + } catch { + // an unreadable root keeps its written path + } + const key = crypto.createHash('sha1').update(real).digest('hex').slice(0, 12); + return path.join(base, 'axiomcode', 'deno', `${path.basename(real) || 'root'}-${key}`); +} + +/** The file that marks a directory as a mirror this module wrote. */ +export const MIRROR_MARKER = '.axiomcode-deno-mirror'; + +const preparedMirrors = new Set(); + +/** + * Writes `text` to `/` unless it is already there, and returns + * the absolute path. The write is a rename, so two parse workers writing the same + * module never leave a torn file. + * + * The first write to a mirror also gives it a `deno.json` and the project's + * lockfile: a library parse of the mirror then resolves the `jsr:` imports INSIDE + * the mirrored packages to the same pinned versions the project uses. + */ +export function materialize(root: DenoConfig, relative: string, text: string): string | undefined { + const mirror = mirrorRoot(root); + const target = path.join(mirror, relative); + try { + if (!preparedMirrors.has(mirror)) { + fs.mkdirSync(mirror, { recursive: true }); + writeIfChanged(path.join(mirror, MIRROR_MARKER), ''); + writeIfChanged(path.join(mirror, 'deno.json'), '{}\n'); + const lock = readLock(root); + if (lock !== undefined) { + writeIfChanged(path.join(mirror, 'deno.lock'), lock.text); + } + preparedMirrors.add(mirror); + } + writeIfChanged(target, text); + return target; + } catch { + return undefined; + } +} + +function writeIfChanged(file: string, text: string): void { + if (readText(file) === text) { + return; + } + fs.mkdirSync(path.dirname(file), { recursive: true }); + const temporary = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`; + fs.writeFileSync(temporary, text); + fs.renameSync(temporary, file); +} + +/** Forgets cached lockfiles and prepared mirrors; for a test that rewrites them. */ +export function clearDenoCacheState(): void { + locks.clear(); + preparedMirrors.clear(); +} diff --git a/parser/src/parsers/typescript/deno/deno-config.ts b/parser/src/parsers/typescript/deno/deno-config.ts new file mode 100644 index 000000000..d1c54492e --- /dev/null +++ b/parser/src/parsers/typescript/deno/deno-config.ts @@ -0,0 +1,336 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import * as ts from 'typescript'; + +/** + * Which `deno.json` governs a file, and what it says. + * + * ## Why Deno needs its own reading + * + * A Deno program has no `tsconfig.json` and no `node_modules` to speak of. Its + * module graph is written in `deno.json`: an import map (`imports`, `scopes`, or + * an `importMap` file) turns a bare specifier into a file, a `jsr:` / `npm:` + * package or a URL, and a `workspace` lists the member packages that import one + * another by NAME. `ts.resolveModuleName` knows none of it, so before this file + * every `import { Hono } from "hono"` in a Deno service resolved to nothing and + * every call across a workspace member was matched by name only. + * + * ## Member and root + * + * The NEAREST config is the member that owns the file. A workspace root lists its + * members; its import map applies to every member, and the member's own entries + * win. A config that no workspace lists is its own root. Both are pure functions + * of the file system, so a parse worker reaches the same answer as the analyzer. + */ +export const DENO_CONFIG_NAMES = ['deno.json', 'deno.jsonc'] as const; + +/** One import map: bare or URL keys to targets, and the scopes that override it. */ +export interface DenoImportMap { + /** + * Key to target. A target written relative to the map (`./src/`) is stored + * ABSOLUTE; `jsr:`, `npm:`, `node:` and URL targets are stored as written. + */ + readonly imports: ReadonlyMap; + /** Scope prefix (an absolute directory with a trailing separator, or a URL) to its map. */ + readonly scopes: ReadonlyArray<{ readonly prefix: string; readonly imports: ReadonlyMap }>; +} + +export interface DenoConfig { + /** Absolute path of the `deno.json` / `deno.jsonc`. */ + readonly path: string; + readonly dir: string; + /** `name` as published (`@scope/pkg`), or `''`. */ + readonly name: string; + readonly version: string; + /** Export subpath (`.`, `./ids`) to the absolute file it names. */ + readonly exports: ReadonlyMap; + readonly importMap: DenoImportMap; + /** Absolute member directories, for a workspace root; empty otherwise. */ + readonly workspaceMembers: readonly string[]; + /** `"vendor": true`: remote modules are kept under `/vendor`. */ + readonly vendor: boolean; + /** + * `nodeModulesDir`: `none` (also `false`) serves npm packages from Deno's cache, `auto` + * (also `true`) installs them on use, `manual` expects an installed node_modules. + * `undefined` when the config does not say (Deno then expects node_modules beside a package.json). + */ + readonly nodeModulesDir: 'none' | 'auto' | 'manual' | undefined; + /** `false` when the config turns the lockfile off; the path otherwise (it may not exist). */ + readonly lockPath: string | false; + readonly compilerOptions: Readonly>; +} + +export interface DenoContext { + /** The config nearest the file. */ + readonly member: DenoConfig; + /** The workspace root that lists `member`, or `member` itself. */ + readonly root: DenoConfig; + /** Every member of `root`'s workspace, `root` included when it has a name. */ + readonly members: readonly DenoConfig[]; +} + +const parsedConfigs = new Map(); +const contextByDir = new Map(); + +/** The Deno config governing `absoluteFilePath`, or `undefined` outside any Deno project. */ +export function denoContextFor(absoluteFilePath: string): DenoContext | undefined { + return denoContextForDir(path.dirname(path.resolve(absoluteFilePath))); +} + +export function denoContextForDir(directory: string): DenoContext | undefined { + const dir = path.resolve(directory); + const cached = contextByDir.get(dir); + if (cached !== undefined) { + return cached ?? undefined; + } + let result: DenoContext | undefined; + const memberPath = findConfigUpward(dir); + const member = memberPath === undefined ? undefined : readDenoConfig(memberPath); + if (member !== undefined) { + let root: DenoConfig = member; + // The workspace root is an ANCESTOR config that lists this member's directory. + let up = path.dirname(member.dir); + for (;;) { + const candidatePath = findConfigUpward(up); + if (candidatePath === undefined) { + break; + } + const candidate = readDenoConfig(candidatePath); + if (candidate !== undefined && candidate.workspaceMembers.some((m) => samePath(m, member.dir))) { + root = candidate; + break; + } + const parent = path.dirname(path.dirname(candidatePath)); + if (parent === path.dirname(candidatePath)) { + break; + } + up = parent; + } + const members = root.workspaceMembers + .map((m) => { + const p = configIn(m); + return p === undefined ? undefined : readDenoConfig(p); + }) + .filter((c): c is DenoConfig => c !== undefined); + if (root.name !== '' && !members.some((m) => samePath(m.path, root.path))) { + members.push(root); + } + if (member.name !== '' && !members.some((m) => samePath(m.path, member.path))) { + members.push(member); + } + result = { member, root, members }; + } + contextByDir.set(dir, result ?? null); + return result; +} + +/** The Deno config file directly in `dir`, if any. */ +export function configIn(dir: string): string | undefined { + for (const name of DENO_CONFIG_NAMES) { + const candidate = path.join(dir, name); + if (fs.existsSync(candidate)) { + return candidate; + } + } + return undefined; +} + +function findConfigUpward(start: string): string | undefined { + let dir = start; + for (;;) { + const found = configIn(dir); + if (found !== undefined) { + return found; + } + const parent = path.dirname(dir); + if (parent === dir) { + return undefined; + } + dir = parent; + } +} + +function samePath(a: string, b: string): boolean { + return path.resolve(a) === path.resolve(b); +} + +/** JSON with comments and trailing commas, which is what `deno.jsonc` (and often `deno.json`) holds. */ +export function readJsonc(file: string): unknown { + let text: string; + try { + text = fs.readFileSync(file, 'utf-8'); + } catch { + return undefined; + } + const parsed = ts.parseConfigFileTextToJson(file, text); + return parsed.error ? undefined : parsed.config; +} + +export function readDenoConfig(configPath: string): DenoConfig | undefined { + const absolute = path.resolve(configPath); + const cached = parsedConfigs.get(absolute); + if (cached !== undefined) { + return cached ?? undefined; + } + const raw = readJsonc(absolute); + let result: DenoConfig | undefined; + if (raw !== null && typeof raw === 'object' && !Array.isArray(raw)) { + const json = raw as Record; + const dir = path.dirname(absolute); + const inline = importMapOf(json, dir); + const external = typeof json.importMap === 'string' + ? importMapFile(path.resolve(dir, json.importMap)) + : undefined; + result = { + path: absolute, + dir, + name: typeof json.name === 'string' ? json.name : '', + version: typeof json.version === 'string' ? json.version : '', + exports: exportsOf(json.exports, dir), + importMap: { + imports: new Map([...(external?.imports ?? []), ...inline.imports]), + scopes: [...inline.scopes, ...(external?.scopes ?? [])], + }, + workspaceMembers: workspaceMembersOf(json.workspace, dir), + vendor: json.vendor === true, + nodeModulesDir: json.nodeModulesDir === false || json.nodeModulesDir === 'none' ? 'none' + : json.nodeModulesDir === true || json.nodeModulesDir === 'auto' ? 'auto' + : json.nodeModulesDir === 'manual' ? 'manual' : undefined, + lockPath: lockPathOf(json.lock, dir), + compilerOptions: json.compilerOptions !== null && typeof json.compilerOptions === 'object' + ? json.compilerOptions as Record + : {}, + }; + } + parsedConfigs.set(absolute, result ?? null); + return result; +} + +function importMapFile(file: string): DenoImportMap | undefined { + const raw = readJsonc(file); + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) { + return undefined; + } + return importMapOf(raw as Record, path.dirname(file)); +} + +function importMapOf(json: Record, baseDir: string): DenoImportMap { + const scopes: Array<{ prefix: string; imports: ReadonlyMap }> = []; + if (json.scopes !== null && typeof json.scopes === 'object') { + for (const [prefix, map] of Object.entries(json.scopes as Record)) { + if (map !== null && typeof map === 'object') { + scopes.push({ prefix: absoluteTarget(prefix, baseDir), imports: entriesOf(map, baseDir) }); + } + } + // The most specific scope is consulted first, as the import-map spec orders them. + scopes.sort((a, b) => b.prefix.length - a.prefix.length); + } + return { imports: entriesOf(json.imports, baseDir), scopes }; +} + +function entriesOf(value: unknown, baseDir: string): Map { + const out = new Map(); + if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + for (const [key, target] of Object.entries(value as Record)) { + if (typeof target === 'string') { + out.set(key, absoluteTarget(target, baseDir)); + } + } + } + return out; +} + +/** `./src/` beside the map becomes an absolute directory, its trailing separator kept. */ +function absoluteTarget(target: string, baseDir: string): string { + if (target.startsWith('file://')) { + return withTrailing(target, decodeURIComponent(new URL(target).pathname)); + } + if (target.startsWith('./') || target.startsWith('../') || target === '.' || target === '..' + || target.startsWith('/')) { + return withTrailing(target, path.resolve(baseDir, target)); + } + return target; +} + +function withTrailing(original: string, resolved: string): string { + return original.endsWith('/') && !resolved.endsWith(path.sep) ? resolved + path.sep : resolved; +} + +function exportsOf(value: unknown, dir: string): Map { + const out = new Map(); + if (typeof value === 'string') { + out.set('.', path.resolve(dir, value)); + } else if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + for (const [subpath, target] of Object.entries(value as Record)) { + if (typeof target === 'string') { + out.set(subpath, path.resolve(dir, target)); + } + } + } + return out; +} + +function workspaceMembersOf(value: unknown, dir: string): string[] { + const listed = Array.isArray(value) + ? value + : value !== null && typeof value === 'object' && Array.isArray((value as { members?: unknown }).members) + ? (value as { members: unknown[] }).members + : []; + const out: string[] = []; + for (const entry of listed) { + if (typeof entry !== 'string') { + continue; + } + if (entry.endsWith('/*') || entry.endsWith('/*/')) { + // `./packages/*`: every directory under it that holds a Deno config. + const parent = path.resolve(dir, entry.replace(/\/\*\/?$/, '')); + let children: fs.Dirent[] = []; + try { + children = fs.readdirSync(parent, { withFileTypes: true }); + } catch { + // a glob over a missing directory names no member + } + for (const child of children) { + if (child.isDirectory() && configIn(path.join(parent, child.name)) !== undefined) { + out.push(path.join(parent, child.name)); + } + } + } else { + out.push(path.resolve(dir, entry)); + } + } + return out; +} + +function lockPathOf(value: unknown, dir: string): string | false { + if (value === false) { + return false; + } + if (typeof value === 'string') { + return path.resolve(dir, value); + } + if (value !== null && typeof value === 'object' && typeof (value as { path?: unknown }).path === 'string') { + return path.resolve(dir, (value as { path: string }).path); + } + return path.join(dir, 'deno.lock'); +} + +/** Forgets every parsed config; for a test that rewrites one in place. */ +export function clearDenoConfigCache(): void { + parsedConfigs.clear(); + contextByDir.clear(); +} + +/** + * Whether `parent/name` is the `vendor/` directory a Deno config beside it keeps + * its downloaded modules in. Those are third-party sources: walking them as + * project files would put every vendored package in the project's own graph. + */ +export function isDenoVendorDirectory(parent: string, name: string): boolean { + if (name !== 'vendor') { + return false; + } + const config = configIn(parent); + return config !== undefined && readDenoConfig(config)?.vendor === true; +} diff --git a/parser/src/parsers/typescript/deno/deno-resolver.ts b/parser/src/parsers/typescript/deno/deno-resolver.ts new file mode 100644 index 000000000..c3864cba6 --- /dev/null +++ b/parser/src/parsers/typescript/deno/deno-resolver.ts @@ -0,0 +1,684 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import * as ts from 'typescript'; + +import { DenoConfig, DenoContext, DenoImportMap, denoContextFor } from '@/parsers/typescript/deno/deno-config'; +import { + cachedNpmVersions, materialize, readLock, readRemote, RemoteModule, +} from '@/parsers/typescript/deno/deno-cache'; +import { maxSatisfying, satisfies } from '@/parsers/typescript/deno/deno-semver'; + +/** + * Resolves a module specifier the way Deno does, for a file inside a Deno project. + * + * Order, as Deno applies it: + * + * 1. the import map: the member's `scopes` and `imports`, then the workspace + * root's (a key ending in `/` maps a prefix; a `jsr:` / `npm:` target also + * maps the key's subpaths, `"hono": "jsr:@hono/hono"` serving `hono/cors`); + * 2. a workspace member by name (`@acme/shared`, `@acme/shared/ids`), through + * its `exports`; + * 3. the target's scheme: `jsr:` to the version the lockfile pins (or the + * highest cached one in range) and the file its `exports` names; `npm:` to + * `node_modules` or Deno's npm cache; `https:` to the downloaded module, + * following redirects and an `x-typescript-types` header as Deno does. + * + * Relative specifiers and bare ones nothing above claims are NOT answered here: + * `ts.resolveModuleName` already gets those right (`./a.ts` included) and a Deno + * project with a `package.json` resolves its bare npm imports through + * `node_modules` exactly as Node does. + * + * Every answer is a file that exists. A `jsr:` or URL module that was never + * downloaded resolves to nothing, which is the honest answer for this machine. + */ +export type DenoResolutionKind = + /** An import-map entry onto a project file. */ + | 'IMPORT_MAP' + /** A workspace member imported by its name. */ + | 'WORKSPACE_MEMBER' + | 'JSR' + /** An npm package in Deno's own cache. */ + | 'NPM_CACHE' + /** An npm package in the project's `node_modules`. */ + | 'NODE_MODULES' + | 'REMOTE' + | 'BUILTIN'; + +export interface DenoResolution { + readonly resolvedFileName: string; + /** The `ts.Extension` the file carries (`.ts`, `.d.ts`, …); `''` for a builtin. */ + readonly extension: string; + readonly kind: DenoResolutionKind; + /** `@std/path`, `zod`, `@acme/shared`; `''` for a project file or a URL. */ + readonly packageName: string; +} + +const resolutions = new Map(); + +export function resolveDenoSpecifier(specifier: string, containingFile: string): DenoResolution | undefined { + if (specifier === '' || isPathSpecifier(specifier)) { + return undefined; + } + const context = denoContextFor(containingFile); + if (context === undefined) { + return undefined; + } + const key = `${path.dirname(path.resolve(containingFile))}\0${specifier}`; + const cached = resolutions.get(key); + if (cached !== undefined) { + return cached ?? undefined; + } + let result: DenoResolution | undefined; + try { + result = resolveIn(specifier, path.resolve(containingFile), context); + } catch { + result = undefined; + } + // Scopes and member maps are keyed by directory, so the answer is too. + resolutions.set(key, result ?? null); + return result; +} + +function resolveIn(specifier: string, containingFile: string, context: DenoContext): DenoResolution | undefined { + const mapped = mapSpecifier(specifier, containingFile, context); + if (mapped !== undefined && path.isAbsolute(mapped)) { + const file = existingFile(mapped); + return file === undefined ? undefined : { ...fileResolution(file), kind: 'IMPORT_MAP', packageName: '' }; + } + const target = mapped ?? specifier; + if (target.startsWith('node:')) { + return { resolvedFileName: '', extension: '', kind: 'BUILTIN', packageName: '' }; + } + if (target.startsWith('jsr:')) { + return resolveJsr(target, context); + } + if (target.startsWith('npm:')) { + return resolveNpm(target, containingFile, context); + } + if (target.startsWith('https:') || target.startsWith('http:')) { + return resolveRemoteUrl(target, context.root); + } + return memberExport(target, context) ?? packageJsonDependency(target, containingFile, context); +} + +// ── 2b. a package.json dependency ─────────────────────────────────────────── + +/** + * A bare specifier no import map or member claims, named by the nearest + * package.json's dependencies: under `"nodeModulesDir": "none"` (or `auto`) Deno + * serves it from its npm cache (`vitest` in a member that declares it). Under + * `manual`, Deno's default beside a package.json, it expects node_modules and + * refuses otherwise, so this answers nothing and the compiler's resolution does. + */ +function packageJsonDependency(specifier: string, containingFile: string, context: DenoContext): + DenoResolution | undefined { + const mode = context.root.nodeModulesDir; + if (mode !== 'none' && mode !== 'auto') { + return undefined; + } + const m = /^((?:@[^/@]+\/)?[^/@]+)(\/.*)?$/.exec(specifier); + if (!m) { + return undefined; + } + const name = m[1]!; + const range = dependencyRange(name, path.dirname(containingFile), context.root.dir); + if (range === undefined) { + return undefined; + } + const installed = ts.resolveModuleName(specifier, containingFile, NPM_OPTIONS, ts.sys).resolvedModule; + if (installed !== undefined) { + return undefined; + } + const resolved = resolveNpm(`npm:${name}${range === '' ? '' : `@${range}`}${m[2] ?? ''}`, containingFile, context); + return resolved?.kind === 'NPM_CACHE' ? resolved : undefined; +} + +const DEPENDENCY_FIELDS = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']; + +/** `name`'s range in the nearest package.json at or above `dir` (up to `top`) that declares it; '' for a non-semver value. */ +function dependencyRange(name: string, dir: string, top: string): string | undefined { + for (let d = dir; ; d = path.dirname(d)) { + const manifest = readPackageJson(path.join(d, 'package.json')); + for (const field of DEPENDENCY_FIELDS) { + const value = (manifest?.[field] as Record | undefined)?.[name]; + if (typeof value === 'string') { + // `workspace:*` is a member (resolved above); `catalog:`, `file:` and tags name no range. + return /^[\d^~<>=*xX |.-]/.test(value) && !value.startsWith('workspace:') ? value.trim() : ''; + } + } + if (d === top || path.dirname(d) === d) { + return undefined; + } + } +} + +const packageJsons = new Map | null>(); + +function readPackageJson(file: string): Record | undefined { + let known = packageJsons.get(file); + if (known === undefined) { + try { + known = JSON.parse(fs.readFileSync(file, 'utf-8')) as Record; + } catch { + known = null; + } + packageJsons.set(file, known); + } + return known ?? undefined; +} + +// ── 1. the import map ─────────────────────────────────────────────────────── + +function mapSpecifier(specifier: string, containingFile: string, context: DenoContext): string | undefined { + const maps = context.member === context.root + ? [context.member.importMap] + : [context.member.importMap, context.root.importMap]; + for (const map of maps) { + const hit = applyImportMap(specifier, containingFile, map); + if (hit !== undefined) { + return hit; + } + } + return undefined; +} + +export function applyImportMap(specifier: string, referrer: string, map: DenoImportMap): string | undefined { + for (const scope of map.scopes) { + if (referrer.startsWith(scope.prefix)) { + const hit = matchImports(specifier, scope.imports); + if (hit !== undefined) { + return hit; + } + } + } + return matchImports(specifier, map.imports); +} + +function matchImports(specifier: string, imports: ReadonlyMap): string | undefined { + const exact = imports.get(specifier); + if (exact !== undefined) { + return exact; + } + // The longest `prefix/` key, as the import-map spec chooses among several. + let best: string | undefined; + for (const key of imports.keys()) { + if (key.endsWith('/') && specifier.startsWith(key) && (best === undefined || key.length > best.length)) { + best = key; + } + } + if (best !== undefined) { + return joinTarget(imports.get(best)!, specifier.slice(best.length)); + } + // Deno's own extension: a key that maps to a PACKAGE also serves its subpaths. + let packageKey: string | undefined; + for (const [key, target] of imports) { + if (!key.endsWith('/') && specifier.startsWith(`${key}/`) + && (target.startsWith('jsr:') || target.startsWith('npm:')) + && (packageKey === undefined || key.length > packageKey.length)) { + packageKey = key; + } + } + return packageKey === undefined + ? undefined + : `${imports.get(packageKey)!}${specifier.slice(packageKey.length)}`; +} + +function joinTarget(target: string, rest: string): string { + if (path.isAbsolute(target)) { + return path.join(target, ...rest.split('/')); + } + return target + rest; +} + +// ── 2. workspace members ──────────────────────────────────────────────────── + +function memberExport(specifier: string, context: DenoContext): DenoResolution | undefined { + for (const member of context.members) { + if (member.name === '' || (specifier !== member.name && !specifier.startsWith(`${member.name}/`))) { + continue; + } + const subpath = specifier === member.name ? '.' : `.${specifier.slice(member.name.length)}`; + const target = member.exports.get(subpath); + const file = target === undefined ? undefined : existingFile(target); + if (file !== undefined) { + return { ...fileResolution(file), kind: 'WORKSPACE_MEMBER', packageName: member.name }; + } + } + return undefined; +} + +// ── 3a. jsr: ──────────────────────────────────────────────────────────────── + +const JSR = /^jsr:\/?(@[^/@]+\/[^/@]+)(?:@([^/]*))?(\/.*)?$/; + +interface JsrMeta { + readonly exports?: Record; +} + +function resolveJsr(target: string, context: DenoContext): DenoResolution | undefined { + const m = JSR.exec(target); + if (!m) { + return undefined; + } + const name = m[1]!; + const range = m[2] ?? ''; + const subpath = m[3] ?? ''; + // `jsr:@acme/shared` inside the workspace that defines it is the member, not a download. + const member = memberExport(name + subpath, context); + if (member !== undefined) { + return member; + } + const root = context.root; + const version = jsrVersion(name, range, root); + if (version === undefined) { + return undefined; + } + const meta = readJson(`https://jsr.io/${name}/${version}_meta.json`, root); + const exported = meta?.exports?.[subpath === '' ? '.' : `.${subpath}`]; + if (exported === undefined) { + return undefined; + } + const file = exported.replace(/^\.\//, ''); + const module = readRemote(`https://jsr.io/${name}/${version}/${file}`, root); + const written = module === undefined ? undefined : materializeWithImports(module, root); + return written === undefined + ? undefined + : { ...fileResolution(written), kind: 'JSR', packageName: name }; +} + +function jsrVersion(name: string, range: string, root: DenoConfig): string | undefined { + const locked = readLock(root)?.jsr.get(name); + if (locked !== undefined && locked.length > 0) { + const pinned = maxSatisfying(locked, range); + if (pinned !== undefined) { + return pinned; + } + } + // No lockfile, or it pins no version in range: the highest version the cache holds. + const meta = readJson<{ versions?: Record }>(`https://jsr.io/${name}/meta.json`, root); + const available = Object.entries(meta?.versions ?? {}) + .filter(([version, info]) => info?.yanked !== true + && readRemote(`https://jsr.io/${name}/${version}_meta.json`, root) !== undefined) + .map(([version]) => version); + return maxSatisfying(available, range); +} + +function readJson(url: string, root: DenoConfig): T | undefined { + const module = readRemote(url, root); + if (module === undefined) { + return undefined; + } + try { + return JSON.parse(module.text) as T; + } catch { + return undefined; + } +} + +// ── 3b. npm: ──────────────────────────────────────────────────────────────── + +const NPM = /^npm:\/?((?:@[^/@]+\/)?[^/@]+)(?:@([^/]*))?(\/.*)?$/; + +/** What a Deno program resolves an npm package's entry with: ESM, `types` first. */ +const NPM_OPTIONS: ts.CompilerOptions = { + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + customConditions: ['deno'], + allowImportingTsExtensions: true, + noEmit: true, +}; + +function resolveNpm(target: string, containingFile: string, context: DenoContext): DenoResolution | undefined { + const m = NPM.exec(target); + if (!m) { + return undefined; + } + const name = m[1]!; + const range = m[2] ?? ''; + const bare = name + (m[3] ?? ''); + // A project with node_modules (`nodeModulesDir`, or a package.json) installs there. + const installed = ts.resolveModuleName(bare, containingFile, NPM_OPTIONS, ts.sys).resolvedModule; + const lockedNpm = readLock(context.root)?.npm.get(name) ?? []; + if (installed !== undefined && /[\\/]node_modules[\\/]/.test(installed.resolvedFileName)) { + // The version is checked only on the package itself: a package without types + // resolves to its `@types/` twin, whose version follows its own line. + const installedVersion = installed.packageId?.name === name ? installed.packageId.version : undefined; + if (range === '' || installedVersion === undefined || satisfies(installedVersion, range)) { + return { + resolvedFileName: path.normalize(installed.resolvedFileName), + extension: installed.extension, + kind: 'NODE_MODULES', + packageName: installed.packageId?.name ?? name, + }; + } + // The hoisted copy is another major (`npm:tailwindcss@^3` beside a hoisted 4): Deno + // keeps each version under node_modules/.deno/@/node_modules/. + const side = denoNodeModulesVersions(name, containingFile); + const version = maxSatisfying(lockedNpm.filter((v) => side.has(v)), range) ?? maxSatisfying(side.keys(), range); + const file = version === undefined ? undefined : resolveInPackage(bare, name, side.get(version)!); + if (file !== undefined) { + return { resolvedFileName: file.resolvedFileName, extension: file.extension, kind: 'NODE_MODULES', packageName: name }; + } + } + const cached = cachedNpmVersions(name); + if (cached.size === 0) { + return undefined; + } + const locked = lockedNpm.filter((v) => cached.has(v)); + const version = maxSatisfying(locked, range) ?? maxSatisfying(cached.keys(), range); + const packageDir = version === undefined ? undefined : cached.get(version); + if (packageDir === undefined) { + return undefined; + } + const file = resolveInPackage(bare, name, packageDir); + if (file === undefined) { + return undefined; + } + // A package that ships only JavaScript is type-checked against its @types twin, as the compiler does. + const typed = DECLARATION.test(file.resolvedFileName) ? undefined : typesTwin(bare, name, context); + return { ...(typed ?? file), kind: 'NPM_CACHE', packageName: name }; +} + +const DECLARATION = /\.d\.[cm]?ts$/; + +function typesTwin(bare: string, name: string, context: DenoContext): + { resolvedFileName: string; extension: string } | undefined { + const twin = `@types/${name.startsWith('@') ? name.slice(1).replace('/', '__') : name}`; + const cached = cachedNpmVersions(twin); + if (cached.size === 0) { + return undefined; + } + const locked = (readLock(context.root)?.npm.get(twin) ?? []).filter((v) => cached.has(v)); + const version = maxSatisfying(locked, '') ?? maxSatisfying(cached.keys(), ''); + return version === undefined ? undefined + : resolveInPackage(twin + bare.slice(name.length), twin, cached.get(version)!); +} + +/** + * Each version of `name` under the nearest `node_modules/.deno`, with its package + * directory: `@scope/pkg` lives at `.deno/@scope+pkg@1.2.3[_peer]/node_modules/@scope/pkg`. + */ +function denoNodeModulesVersions(name: string, containingFile: string): Map { + const out = new Map(); + const prefix = name.replace('/', '+') + '@'; + for (let dir = path.dirname(containingFile); ; dir = path.dirname(dir)) { + const store = path.join(dir, 'node_modules', '.deno'); + let entries: string[] | undefined; + try { + entries = fs.readdirSync(store); + } catch { + entries = undefined; + } + if (entries !== undefined) { + for (const entry of entries) { + if (entry.startsWith(prefix)) { + const version = entry.slice(prefix.length).split('_')[0]!; + const packageDir = path.join(store, entry, 'node_modules', ...name.split('/')); + if (!out.has(version) && fs.existsSync(packageDir)) { + out.set(version, packageDir); + } + } + } + return out; + } + if (path.dirname(dir) === dir) { + return out; + } + } +} + +/** + * `bare` resolved inside one unpacked package directory, honouring its + * `exports` / `types` the way the compiler does for `node_modules`. + * + * Deno's cache is not a `node_modules` tree, so the compiler is shown one: a host + * that answers for `/node_modules//…` from `packageDir` and for + * nothing else under the virtual root. + */ +function resolveInPackage(bare: string, name: string, packageDir: string): + { resolvedFileName: string; extension: string } | undefined { + const virtualRoot = path.join(path.parse(packageDir).root, '__axiomcode_deno_npm__'); + const virtualPackage = path.join(virtualRoot, 'node_modules', ...name.split('/')); + const toReal = (p: string): string | undefined => { + const normal = path.normalize(p); + if (normal === virtualPackage || normal.startsWith(virtualPackage + path.sep)) { + return packageDir + normal.slice(virtualPackage.length); + } + return undefined; + }; + const host: ts.ModuleResolutionHost = { + fileExists: (p) => { + const real = toReal(p); + return real !== undefined && fs.existsSync(real) && fs.statSync(real).isFile(); + }, + readFile: (p) => { + const real = toReal(p); + return real === undefined ? undefined : ts.sys.readFile(real); + }, + directoryExists: (p) => { + const normal = path.normalize(p); + if (virtualPackage.startsWith(normal + path.sep) || normal === virtualPackage) { + return true; + } + const real = toReal(p); + return real !== undefined && fs.existsSync(real) && fs.statSync(real).isDirectory(); + }, + realpath: (p) => p, + getCurrentDirectory: () => virtualRoot, + }; + const resolved = ts.resolveModuleName(bare, path.join(virtualRoot, 'index.ts'), NPM_OPTIONS, host).resolvedModule; + const real = resolved === undefined ? undefined : toReal(resolved.resolvedFileName); + return real === undefined ? undefined : { resolvedFileName: real, extension: resolved!.extension }; +} + +// ── 3c. https: ────────────────────────────────────────────────────────────── + +function resolveRemoteUrl(url: string, root: DenoConfig): DenoResolution | undefined { + let module = followRedirects(url, root); + if (module === undefined) { + return undefined; + } + // A CDN that serves JavaScript points at its declarations, and Deno type-checks against those. + const types = module.headers['x-typescript-types']; + if (types !== undefined) { + const declared = followRedirects(new URL(types, module.url).toString(), root); + if (declared !== undefined) { + module = declared; + } + } + const written = materializeWithImports(module, root); + return written === undefined ? undefined : { ...fileResolution(written), kind: 'REMOTE', packageName: '' }; +} + +/** URLs already copied to a mirror this process, so a cycle between modules ends. */ +const materializedUrls = new Map(); + +/** + * Copies `module` to the mirror and, transitively, every module it imports that + * Deno downloaded: a relative import by its URL, a `jsr:` / `npm:` / URL import + * through this resolver from the mirrored file. A library parse of the mirror + * then finds a package's whole reachable source, not only the entry the project + * imported. + */ +function materializeWithImports(module: RemoteModule, root: DenoConfig): string | undefined { + const known = materializedUrls.get(module.url); + if (known !== undefined) { + return known ?? undefined; + } + const relative = mirrorPathOf(module); + const written = relative === undefined ? undefined : materialize(root, relative, module.text); + materializedUrls.set(module.url, written ?? null); + if (written === undefined) { + return undefined; + } + for (const specifier of moduleSpecifiersOf(module.text, written)) { + if (specifier.startsWith('./') || specifier.startsWith('../') || specifier.startsWith('/')) { + let target: string; + try { + target = new URL(specifier, module.url).toString(); + } catch { + continue; + } + if (!materializedUrls.has(target)) { + const imported = followRedirects(target, root); + if (imported !== undefined) { + materializeWithImports(imported, root); + } + } + } else if (/^(jsr|npm|https?):/.test(specifier)) { + resolveDenoSpecifier(specifier, written); + } + } + return written; +} + +function followRedirects(url: string, root: DenoConfig): RemoteModule | undefined { + const redirects = readLock(root)?.redirects; + let current = url; + for (let hop = 0; hop < 10; hop++) { + // Deno 2 records a redirect in the lockfile and may keep no cache entry for it. + const locked = redirects?.get(current) ?? redirects?.get(current.replace(/%5E/gi, '^')); + if (locked !== undefined && locked !== current) { + current = locked; + continue; + } + const module = readRemote(current, root); + if (module === undefined) { + return undefined; + } + const location = module.headers.location; + if (location === undefined) { + return module; + } + current = new URL(location, current).toString(); + } + return undefined; +} + +const TS_LIKE = /\.(d\.ts|d\.mts|d\.cts|tsx?|mts|cts|jsx?|mjs|cjs)$/; + +/** `deno.land/std@0.224.0/fmt/colors.ts`; an extensionless URL takes one from its content type. */ +function mirrorPathOf(module: RemoteModule): string | undefined { + let parsed: URL; + try { + parsed = new URL(module.url); + } catch { + return undefined; + } + const host = parsed.port === '' ? parsed.hostname : `${parsed.hostname}_PORT${parsed.port}`; + const segments = parsed.pathname.split('/').filter((s) => s !== '').map((s) => decodeURIComponent(s)); + if (segments.length === 0) { + segments.push('index'); + } + let last = segments.pop()!; + if (parsed.search !== '') { + last += `_${Buffer.from(parsed.search).toString('base64url')}`; + } + if (!TS_LIKE.test(last) || parsed.search !== '') { + const type = module.headers['content-type'] ?? ''; + last += /typescript|\/tsx?\b/.test(type) ? (/tsx/.test(type) ? '.tsx' : '.ts') + : /javascript|ecmascript|\/jsx\b/.test(type) ? '.js' + : TS_LIKE.exec(parsed.pathname)?.[0] ?? '.ts'; + } + return path.join(host, ...segments, last); +} + +/** + * Every module specifier a file names, as the parser's extraction sees them: + * imports and re-exports (`export * as ns from` included, which the compiler's + * light pre-processor misses), `import x = require()`, `import()` / `require()` + * with a literal, `import("x")` types, and triple-slash references. Missing one + * here leaves a module out of the mirror until a later library parse copies it, + * which changes the mirror and makes every rebuild recompile it. + */ +export function moduleSpecifiersOf(text: string, fileName: string): string[] { + const out: string[] = []; + const sf = ts.createSourceFile(fileName, text, ts.ScriptTarget.Latest, false, + /\.tsx$|\.jsx$/.test(fileName) ? ts.ScriptKind.TSX : ts.ScriptKind.TS); + const literal = (node: ts.Node | undefined): void => { + if (node !== undefined && ts.isStringLiteralLike(node)) { + out.push(node.text); + } + }; + const visit = (node: ts.Node): void => { + if (ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) { + literal(node.moduleSpecifier); + } else if (ts.isImportEqualsDeclaration(node) && ts.isExternalModuleReference(node.moduleReference)) { + literal(node.moduleReference.expression); + } else if (ts.isCallExpression(node) + && (node.expression.kind === ts.SyntaxKind.ImportKeyword + || (ts.isIdentifier(node.expression) && node.expression.text === 'require'))) { + literal(node.arguments[0]); + } else if (ts.isImportTypeNode(node) && ts.isLiteralTypeNode(node.argument)) { + literal(node.argument.literal); + } + ts.forEachChild(node, visit); + }; + ts.forEachChild(sf, visit); + for (const reference of [...sf.referencedFiles, ...sf.typeReferenceDirectives]) { + out.push(reference.fileName); + } + return out; +} + +// ── files ─────────────────────────────────────────────────────────────────── + +function isPathSpecifier(specifier: string): boolean { + return specifier.startsWith('./') || specifier.startsWith('../') || specifier === '.' || specifier === '..' + || specifier.startsWith('/') || specifier.startsWith('file:') || /^[A-Za-z]:[\\/]/.test(specifier); +} + +/** `target` itself, as Deno requires an explicit extension, else what the compiler's probing finds. */ +function existingFile(target: string): string | undefined { + try { + if (fs.statSync(target).isFile()) { + return path.normalize(target); + } + } catch { + // not a file as written; probe below + } + const probed = ts.resolveModuleName(target, path.join(path.dirname(target), '__probe__.ts'), + NPM_OPTIONS, ts.sys).resolvedModule; + return probed === undefined ? undefined : path.normalize(probed.resolvedFileName); +} + +const EXTENSIONS: readonly string[] = [ + ts.Extension.Dts, ts.Extension.Dmts, ts.Extension.Dcts, + ts.Extension.Tsx, ts.Extension.Ts, ts.Extension.Mts, ts.Extension.Cts, + ts.Extension.Jsx, ts.Extension.Js, ts.Extension.Mjs, ts.Extension.Cjs, ts.Extension.Json, +]; + +function fileResolution(file: string): { resolvedFileName: string; extension: string } { + const extension = EXTENSIONS.find((e) => file.endsWith(e)) ?? ''; + return { resolvedFileName: path.normalize(file), extension }; +} + +/** + * The package a specifier names in a Deno project, from the config and syntax + * alone, for an import that did not resolve: `hono` mapped to `jsr:@hono/hono@^4` + * names `@hono/hono`. `undefined` outside a Deno project or for a specifier the + * import map and the schemes say nothing about; `''` for a URL or a project file. + */ +export function denoPackageNameOf(specifier: string, containingFile: string): string | undefined { + if (isPathSpecifier(specifier)) { + return undefined; + } + const context = denoContextFor(containingFile); + if (context === undefined) { + return undefined; + } + const target = mapSpecifier(specifier, path.resolve(containingFile), context) ?? specifier; + if (path.isAbsolute(target) || /^https?:/.test(target) || target.startsWith('node:')) { + return ''; + } + const scheme = target.startsWith('jsr:') ? JSR.exec(target) : target.startsWith('npm:') ? NPM.exec(target) : null; + return scheme ? scheme[1] : undefined; +} + +/** Forgets memoised resolutions; for a test that rewrites a project in place. */ +export function clearDenoResolutionCache(): void { + resolutions.clear(); + materializedUrls.clear(); + packageJsons.clear(); +} diff --git a/parser/src/parsers/typescript/deno/deno-semver.ts b/parser/src/parsers/typescript/deno/deno-semver.ts new file mode 100644 index 000000000..aeb3a8ff5 --- /dev/null +++ b/parser/src/parsers/typescript/deno/deno-semver.ts @@ -0,0 +1,214 @@ +/** + * The slice of semver a Deno import needs: does a locked or cached version satisfy + * the range a specifier wrote, and which of several is the highest. + * + * `jsr:@std/path@^1.0.0` and `npm:zod@~3.23` name RANGES. The version actually + * used is in `deno.lock`, or, without a lockfile, is the highest one the cache + * holds. The lockfile's own specifier keys cannot be looked up by the written + * range (Deno normalises `^1.0.0` to `1` there), so the range is matched against + * the locked VERSIONS instead, which is the same answer without depending on how + * a given Deno release spells its keys. + * + * Supported: exact, `^`, `~`, `x`/`*` wildcards, partial versions (`1`, `1.2`), + * comparators (`>=1.2 <2`), hyphen ranges and `||`. Pre-release versions satisfy + * only a range that names the same pre-release base, as in npm. + */ + +interface Version { + readonly major: number; + readonly minor: number; + readonly patch: number; + readonly pre: readonly string[]; +} + +const VERSION = /^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/; + +export function parseVersion(text: string): Version | undefined { + const m = VERSION.exec(text.trim()); + if (!m) { + return undefined; + } + return { + major: Number(m[1]), + minor: Number(m[2]), + patch: Number(m[3]), + pre: m[4] ? m[4].split('.') : [], + }; +} + +export function compareVersions(a: string, b: string): number { + const va = parseVersion(a); + const vb = parseVersion(b); + if (!va || !vb) { + return va ? 1 : vb ? -1 : a.localeCompare(b); + } + return compare(va, vb); +} + +function compare(a: Version, b: Version): number { + if (a.major !== b.major) return a.major - b.major; + if (a.minor !== b.minor) return a.minor - b.minor; + if (a.patch !== b.patch) return a.patch - b.patch; + if (a.pre.length === 0 || b.pre.length === 0) { + return (a.pre.length === 0 ? 1 : 0) - (b.pre.length === 0 ? 1 : 0); + } + for (let i = 0; i < Math.max(a.pre.length, b.pre.length); i++) { + const x = a.pre[i]; + const y = b.pre[i]; + if (x === undefined) return -1; + if (y === undefined) return 1; + const nx = /^\d+$/.test(x) ? Number(x) : undefined; + const ny = /^\d+$/.test(y) ? Number(y) : undefined; + if (nx !== undefined && ny !== undefined) { + if (nx !== ny) return nx - ny; + } else if (nx !== undefined) { + return -1; + } else if (ny !== undefined) { + return 1; + } else if (x !== y) { + return x < y ? -1 : 1; + } + } + return 0; +} + +/** The highest of `versions` that satisfies `range`, or `undefined`. */ +export function maxSatisfying(versions: Iterable, range: string): string | undefined { + let best: string | undefined; + for (const v of versions) { + if (satisfies(v, range) && (best === undefined || compareVersions(v, best) > 0)) { + best = v; + } + } + return best; +} + +type Comparator = { readonly op: '<' | '<=' | '>' | '>=' | '='; readonly v: Version }; + +export function satisfies(versionText: string, rangeText: string): boolean { + const version = parseVersion(versionText); + if (!version) { + return false; + } + const range = rangeText.trim(); + if (range === '' || range === '*' || range === 'latest' || range === 'x') { + return version.pre.length === 0; + } + return range.split('||').some((alternative) => { + const comparators = parseAlternative(alternative.trim()); + if (comparators === undefined) { + return false; + } + if (!comparators.every((c) => test(version, c))) { + return false; + } + // A pre-release only satisfies a range one of whose comparators names a + // pre-release on the same major.minor.patch. + return version.pre.length === 0 || comparators.some((c) => c.v.pre.length > 0 + && c.v.major === version.major && c.v.minor === version.minor && c.v.patch === version.patch); + }); +} + +function test(v: Version, c: Comparator): boolean { + const d = compare(v, c.v); + switch (c.op) { + case '<': return d < 0; + case '<=': return d <= 0; + case '>': return d > 0; + case '>=': return d >= 0; + default: return d === 0; + } +} + +/** A partial version: `1`, `1.2`, `1.2.3-pre`, with `x`/`*` for a missing part. */ +interface Partial { + readonly major?: number; + readonly minor?: number; + readonly patch?: number; + readonly pre: readonly string[]; +} + +function parsePartial(text: string): Partial | undefined { + const m = /^v?(\d+|[xX*])(?:\.(\d+|[xX*]))?(?:\.(\d+|[xX*]))?(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/.exec(text); + if (!m) { + return undefined; + } + const part = (s: string | undefined): number | undefined => + s === undefined || /^[xX*]$/.test(s) ? undefined : Number(s); + const major = part(m[1]); + const minor = major === undefined ? undefined : part(m[2]); + const patch = minor === undefined ? undefined : part(m[3]); + return { major, minor, patch, pre: m[4] ? m[4].split('.') : [] }; +} + +const v = (major: number, minor: number, patch: number, pre: readonly string[] = []): Version => + ({ major, minor, patch, pre }); + +function parseAlternative(text: string): Comparator[] | undefined { + const hyphen = /^(\S+)\s+-\s+(\S+)$/.exec(text); + if (hyphen) { + const lo = parsePartial(hyphen[1]!); + const hi = parsePartial(hyphen[2]!); + if (!lo || !hi) return undefined; + const out: Comparator[] = [{ op: '>=', v: floor(lo) }]; + if (hi.major === undefined) return out; + if (hi.minor === undefined) return [...out, { op: '<', v: v(hi.major + 1, 0, 0) }]; + if (hi.patch === undefined) return [...out, { op: '<', v: v(hi.major, hi.minor + 1, 0) }]; + return [...out, { op: '<=', v: v(hi.major, hi.minor, hi.patch, hi.pre) }]; + } + const out: Comparator[] = []; + // `>= 1.2` is legal spelling; glue an operator to the version that follows it. + const tokens = text.replace(/(>=|<=|>|<|=)\s+/g, '$1').split(/\s+/).filter((t) => t !== ''); + for (const token of tokens) { + const parsed = comparatorsOf(token); + if (parsed === undefined) return undefined; + out.push(...parsed); + } + return out; +} + +function floor(p: Partial): Version { + return v(p.major ?? 0, p.minor ?? 0, p.patch ?? 0, p.pre); +} + +function comparatorsOf(token: string): Comparator[] | undefined { + const m = /^(\^|~>?|>=|<=|>|<|=)?(.*)$/.exec(token); + if (!m) return undefined; + const op = m[1] ?? ''; + const p = parsePartial(m[2]!); + if (!p) return undefined; + if (p.major === undefined) { + return op === '<' || op === '>' ? [{ op: '<', v: v(0, 0, 0) }] : []; + } + const lo = floor(p); + switch (op) { + case '^': { + if (p.major > 0 || p.minor === undefined) return [{ op: '>=', v: lo }, { op: '<', v: v(p.major + 1, 0, 0) }]; + if (p.minor > 0 || p.patch === undefined) return [{ op: '>=', v: lo }, { op: '<', v: v(0, p.minor + 1, 0) }]; + return [{ op: '>=', v: lo }, { op: '<', v: v(0, 0, p.patch + 1) }]; + } + case '~': + case '~>': { + if (p.minor === undefined) return [{ op: '>=', v: lo }, { op: '<', v: v(p.major + 1, 0, 0) }]; + return [{ op: '>=', v: lo }, { op: '<', v: v(p.major, p.minor + 1, 0) }]; + } + case '>': { + if (p.minor === undefined) return [{ op: '>=', v: v(p.major + 1, 0, 0) }]; + if (p.patch === undefined) return [{ op: '>=', v: v(p.major, p.minor + 1, 0) }]; + return [{ op: '>', v: lo }]; + } + case '>=': return [{ op: '>=', v: lo }]; + case '<': return [{ op: '<', v: lo }]; + case '<=': { + if (p.minor === undefined) return [{ op: '<', v: v(p.major + 1, 0, 0) }]; + if (p.patch === undefined) return [{ op: '<', v: v(p.major, p.minor + 1, 0) }]; + return [{ op: '<=', v: lo }]; + } + default: { + // A bare partial is an x-range: `1` is >=1.0.0 <2.0.0, `1.2` is >=1.2.0 <1.3.0. + if (p.minor === undefined) return [{ op: '>=', v: lo }, { op: '<', v: v(p.major + 1, 0, 0) }]; + if (p.patch === undefined) return [{ op: '>=', v: lo }, { op: '<', v: v(p.major, p.minor + 1, 0) }]; + return [{ op: '=', v: lo }]; + } + } +} diff --git a/parser/src/parsers/typescript/extractors/ts-expression-extractor.ts b/parser/src/parsers/typescript/extractors/ts-expression-extractor.ts index f4183b369..94096b266 100644 --- a/parser/src/parsers/typescript/extractors/ts-expression-extractor.ts +++ b/parser/src/parsers/typescript/extractors/ts-expression-extractor.ts @@ -47,7 +47,9 @@ import { isVueTemplateTag } from '@/utils/vue-sfc'; * * A JSX element naming a component is a call (§4.15.1): a JSX_ELEMENT or * JSX_SELF_CLOSING row with its tag as the METHOD_NAME child, and a - * JSX_COMPONENT_CALL site. `JSX_ATTRIBUTE_VALUE` and `JSX_CHILD` stay reserved: + * JSX_COMPONENT_CALL site. An intrinsic element (`
`) is a + * JSX_INTRINSIC_ELEMENT row with no child and no call site; its tag (`div`) + * is the row's `potentialQualifiedName`. `JSX_ATTRIBUTE_VALUE` and `JSX_CHILD` stay reserved: * attribute values and children are rooted as their own trees by the walker, and * the gate asserts the emptiness so that switching them on shows up as a failure. */ @@ -174,6 +176,9 @@ export class TsExpressionExtractor { isTypeOnlyReachable: false, serviceVersionLinkHash: this.options.serviceVersionLinkHash, }); + if (kind === TsExpressionKind.JSX_INTRINSIC_ELEMENT) { + row.setJsxTagName(jsxTagText(item.node, this.sf)); + } this.expressions.push(row); this.rowByNode.set(nodeId(item.node, this.sf), row); this.emitted.push({ node: item.node, row }); @@ -453,10 +458,12 @@ export function expressionKindOf(node: ts.Node): TsExpressionKind | undefined { case ts.SyntaxKind.JsxElement: case ts.SyntaxKind.JsxSelfClosingElement: { // `` IS a call to Badge. `
` is an intrinsic element that no - // project declaration can bind, so it produces no row -- a call site that - // can only ever resolve into `JSX.IntrinsicElements` would be noise. + // project declaration can bind: a call site that can only ever resolve + // into `JSX.IntrinsicElements` would be noise, so it is its own kind, + // which is never a call (see `emit`) and carries no tag child. The row is + // what a plain tag's attributes -- `onClick={h}`, `ref={r}` -- belong to. if (jsxTagReference(node) === undefined) { - return undefined; + return TsExpressionKind.JSX_INTRINSIC_ELEMENT; } return ts.isJsxElement(node) ? TsExpressionKind.JSX_ELEMENT : TsExpressionKind.JSX_SELF_CLOSING; } @@ -909,6 +916,16 @@ export function jsxTagReference(node: ts.Node): ts.Expression | undefined { return undefined; } +/** A JSX element's tag as written: `div`, `svg:circle`, `my-el`, `ui.Card`. */ +export function jsxTagText(node: ts.Node, sf: ts.SourceFile): string { + const tagName = ts.isJsxElement(node) + ? node.openingElement.tagName + : ts.isJsxSelfClosingElement(node) + ? node.tagName + : undefined; + return tagName === undefined ? '' : tagName.getText(sf); +} + /** The receiver of a callee, or `undefined` for an unqualified call. */ export function receiverNodeOf(callee: ts.Node | undefined): ts.Node | undefined { if (!callee) { diff --git a/parser/src/parsers/typescript/extractors/ts-expression-walker.ts b/parser/src/parsers/typescript/extractors/ts-expression-walker.ts index 1b82d1018..5fda834bf 100644 --- a/parser/src/parsers/typescript/extractors/ts-expression-walker.ts +++ b/parser/src/parsers/typescript/extractors/ts-expression-walker.ts @@ -2,7 +2,6 @@ import * as ts from 'typescript'; import { ExpressionOwner, - jsxTagReference, TsExpressionExtractor, } from '@/parsers/typescript/extractors/ts-expression-extractor'; import { TsExpressionOwnerKind, TsRootContext } from '@/enums/typescript/expressions'; @@ -513,11 +512,12 @@ export class TsExpressionWalker { // attribute values, children, and spreads at once, wherever the JSX sits. // // A component ELEMENT is a call in its own right: `` calls Badge. - // One that is already a row -- `cond ? : `, `x && ` -- was - // reached by the worklist as an operand; any other is rooted here, which - // is what reaches an element nested in another element's children. + // An intrinsic one (`
`) is a JSX_INTRINSIC_ELEMENT row and no call, + // the element its attributes belong to. One that is already a row -- + // `cond ? : `, `x &&
` -- was reached by the worklist as an + // operand; any other is rooted here, which is what reaches an element + // nested in another element's children. if ((ts.isJsxElement(child) || ts.isJsxSelfClosingElement(child)) - && jsxTagReference(child) !== undefined && !this.options.extractor.rowByNode.has(nodeId(child, this.sf))) { this.root(child, TsRootContext.JSX_EMBEDDED_EXPRESSION); return; diff --git a/parser/src/parsers/typescript/extractors/ts-fact-extractor.ts b/parser/src/parsers/typescript/extractors/ts-fact-extractor.ts index d35d7582d..62845254f 100644 --- a/parser/src/parsers/typescript/extractors/ts-fact-extractor.ts +++ b/parser/src/parsers/typescript/extractors/ts-fact-extractor.ts @@ -27,6 +27,7 @@ import { TsDecoratorSystem } from '@/enums/typescript/decorators'; import { TsExportedEntityKind } from '@/enums/typescript/exports'; import { TsParseGapKind } from '@/enums/typescript/parse-gaps'; import { TsModuleResolutionMode } from '@/enums/typescript/modules'; +import { resolveDenoSpecifier } from '@/parsers/typescript/deno/deno-resolver'; import { bindSourceFile, BinderResult, nodeId } from '@/parsers/typescript/extractors/ts-binder'; import { TsDeclarationExtractor } from '@/parsers/typescript/extractors/ts-declaration-extractor'; @@ -175,13 +176,16 @@ export function extractTypeScriptFile(options: TsFileExtractionOptions): TsFileF // table. Without this, `Request` in `augmented-base.ts` and `Request` inside // `declare module "./augmented-base"` are two symbols instead of one. resolveModuleHash: (specifier) => { - const resolved = ts.resolveModuleName( + // A Deno import map or workspace member first, exactly as the import rows resolve it. + const deno = resolveDenoSpecifier(specifier, options.absoluteFilePath); + const resolved = deno !== undefined ? undefined : ts.resolveModuleName( specifier, options.absoluteFilePath, options.compilerOptions, ts.sys ); - const fileName = resolved.resolvedModule?.resolvedFileName + const fileName = (deno?.resolvedFileName || undefined) + ?? resolved?.resolvedModule?.resolvedFileName ?? resolveVueSpecifier(specifier, options.absoluteFilePath); const absolute = fileName ? path.normalize(fileName) : ''; const moduleHash = options.projectModuleHashes.get(absolute); diff --git a/parser/src/parsers/typescript/extractors/ts-import-extractor.ts b/parser/src/parsers/typescript/extractors/ts-import-extractor.ts index 5536a84fa..d78882095 100644 --- a/parser/src/parsers/typescript/extractors/ts-import-extractor.ts +++ b/parser/src/parsers/typescript/extractors/ts-import-extractor.ts @@ -4,6 +4,7 @@ import * as ts from 'typescript'; import { TsImportRegistry } from '@/analysis-types/typescript/TsImportRegistry'; import { TsImportKind, TsImportResolutionKind } from '@/enums/typescript/imports'; +import { denoPackageNameOf, DenoResolutionKind, resolveDenoSpecifier } from '@/parsers/typescript/deno/deno-resolver'; import { nodeId } from '@/parsers/typescript/extractors/ts-binder'; import { resolveVueSpecifier, VUE_EXTENSION } from '@/utils/vue-sfc'; @@ -436,6 +437,30 @@ export class TsImportExtractor { return result; } + /** + * A Deno project's own resolution, tried before the compiler's: an import map, + * a workspace member, a `jsr:` / `npm:` package or a URL. `undefined` for a file + * outside any Deno project and for a specifier Deno resolves as Node would. + */ + private resolveDeno(specifier: string): ResolvedSpecifier | undefined { + const deno = resolveDenoSpecifier(specifier, this.options.absoluteFilePath); + if (deno === undefined) { + return undefined; + } + if (deno.kind === 'BUILTIN') { + return { ...UNRESOLVED, kind: TsImportResolutionKind.BUILTIN_NODE }; + } + const absolute = path.normalize(deno.resolvedFileName); + return { + absolutePath: absolute, + relativePath: this.options.toProjectRelative(absolute), + moduleHash: this.options.projectModuleHashes.get(absolute) ?? '', + kind: DENO_RESOLUTION_KINDS[deno.kind](deno.extension), + extension: deno.extension, + packageName: deno.packageName, + }; + } + private doResolve(specifier: string): ResolvedSpecifier { // A Node builtin, with or without the `node:` prefix. Both forms must be // classified, and the unprefixed form is the common one: `import * as path @@ -447,6 +472,10 @@ export class TsImportExtractor { if (specifier.startsWith('node:') || NODE_BUILTIN_SPECIFIERS.has(specifier)) { return { ...UNRESOLVED, kind: TsImportResolutionKind.BUILTIN_NODE }; } + const deno = this.resolveDeno(specifier); + if (deno !== undefined) { + return deno; + } const resolved = ts.resolveModuleName( specifier, this.options.absoluteFilePath, @@ -498,7 +527,10 @@ export class TsImportExtractor { // import, and a consumer had to re-derive it from the specifier to answer // "which package would close these hops". The name is pure syntax, so it // is filled here whether resolution succeeded or not. - return { ...UNRESOLVED, packageName: packageNameOf(specifier) }; + return { + ...UNRESOLVED, + packageName: denoPackageNameOf(specifier, this.options.absoluteFilePath) ?? packageNameOf(specifier), + }; } return { absolutePath: absolute, @@ -558,6 +590,19 @@ function isDeclarationExtension(extension: string): boolean { || extension === ts.Extension.Dcts; } +/** Each Deno answer as the `resolutionKind` the engine reads. */ +const DENO_RESOLUTION_KINDS: Record TsImportResolutionKind> = { + IMPORT_MAP: () => TsImportResolutionKind.PATHS_ALIAS, + WORKSPACE_MEMBER: () => TsImportResolutionKind.WORKSPACE_PACKAGE, + JSR: () => TsImportResolutionKind.JSR_PACKAGE, + NPM_CACHE: () => TsImportResolutionKind.DENO_NPM_CACHE, + NODE_MODULES: (extension) => isDeclarationExtension(extension) + ? TsImportResolutionKind.NODE_MODULES_TYPES + : TsImportResolutionKind.NODE_MODULES_SOURCE, + REMOTE: () => TsImportResolutionKind.REMOTE_MODULE, + BUILTIN: () => TsImportResolutionKind.BUILTIN_NODE, +}; + interface ResolvedSpecifier { readonly absolutePath: string; readonly relativePath: string; @@ -596,16 +641,25 @@ const NODE_BUILTIN_SPECIFIERS = new Set([ * Node builtins never reach this: they are classified BUILTIN_NODE before * resolution is attempted, and they are not packages to stage. */ -function packageNameOf(specifier: string): string { +function packageNameOf(specifierIn: string): string { + // `npm:zod@^3/v4` and `jsr:@std/path@1/join` name a package after the scheme, + // with a version range glued to it; any other scheme (`https:`, `data:`) names none. + const scheme = /^(?:npm|jsr):\/?/.exec(specifierIn); + if (scheme === null && /^[A-Za-z][A-Za-z0-9+.-]*:/.test(specifierIn)) { + return ''; + } + const specifier = scheme === null ? specifierIn : specifierIn.slice(scheme[0].length); if (specifier === '' || specifier.startsWith('.') || specifier.startsWith('/') || specifier.includes('*')) { return ''; } const segments = specifier.split('/'); + const unversioned = (segment: string | undefined): string => + scheme === null || segment === undefined ? segment ?? '' : segment.replace(/(.)@.*$/, '$1'); if (specifier.startsWith('@')) { - return segments.length >= 2 ? `${segments[0]}/${segments[1]}` : ''; + return segments.length >= 2 ? `${segments[0]}/${unversioned(segments[1])}` : ''; } - return segments[0] ?? ''; + return unversioned(segments[0]); } const UNRESOLVED: ResolvedSpecifier = { diff --git a/parser/src/parsers/typescript/extractors/ts-ir-completeness.ts b/parser/src/parsers/typescript/extractors/ts-ir-completeness.ts index de5be2f9d..bf37ec53e 100644 --- a/parser/src/parsers/typescript/extractors/ts-ir-completeness.ts +++ b/parser/src/parsers/typescript/extractors/ts-ir-completeness.ts @@ -179,7 +179,15 @@ export function linkAmbientModuleImports(files: readonly ModuleGraphFacts[]): nu * parser's own column (§4.13 c8), and this stops one hop short of the call * graph — at the module, which is where `type-resolution.dl` takes over. */ -export function linkReExportSources(files: readonly ModuleGraphFacts[]): number { +export function linkReExportSources( + files: readonly ModuleGraphFacts[], + /** + * The project-relative file a NON-relative specifier names from `fromFilePath`, + * for a resolution the specifier alone cannot give: a Deno import map or + * workspace member re-exported by a barrel that never imports it. + */ + resolveBare?: (fromFilePath: string, specifier: string) => string | undefined +): number { const modulesByPath = new Map(); for (const facts of files) { for (const module of facts.modules) { @@ -206,9 +214,14 @@ export function linkReExportSources(files: readonly ModuleGraphFacts[]): number continue; } const viaImport = resolvedBySpecifier.get(exportRow.sourceSpecifier); + const bare = viaImport === undefined && !exportRow.sourceSpecifier.startsWith('.') + ? resolveBare?.(facts.filePath, exportRow.sourceSpecifier) + : undefined; const target = viaImport !== undefined ? modulesByPath.get(viaImport) - : modulesByPath.get(resolveRelative(facts.filePath, exportRow.sourceSpecifier)); + : bare !== undefined + ? modulesByPath.get(stripKnownExtension(bare)) + : modulesByPath.get(resolveRelative(facts.filePath, exportRow.sourceSpecifier)); if (target !== undefined) { exportRow.setResolvedSourceModuleLinkHash(target); linked += 1; diff --git a/parser/src/parsers/typescript/tsconfig-resolver.ts b/parser/src/parsers/typescript/tsconfig-resolver.ts index f36937469..d84fa4479 100644 --- a/parser/src/parsers/typescript/tsconfig-resolver.ts +++ b/parser/src/parsers/typescript/tsconfig-resolver.ts @@ -4,6 +4,7 @@ import * as path from 'path'; import * as ts from 'typescript'; import { TsDecoratorSystem } from '@/enums/typescript/decorators'; +import { denoContextFor } from '@/parsers/typescript/deno/deno-config'; import { TsModuleResolutionMode } from '@/enums/typescript/modules'; /** @@ -151,7 +152,7 @@ export class TsConfigResolver { dir = parent; } - const result: GoverningTsConfig = found ?? { + const result: GoverningTsConfig = found ?? denoGoverned(normalised) ?? { configPath: '', options: DEFAULT_OPTIONS, moduleResolutionMode: TsModuleResolutionMode.NODE10, @@ -269,6 +270,42 @@ export class TsConfigResolver { } } +/** + * A file no tsconfig claims but a `deno.json` governs compiles under Deno's + * defaults, not Node's: ES modules, explicit `.ts` extensions, strict mode, and + * whatever `compilerOptions` the config sets (`jsx`, `jsxImportSource`, + * `experimentalDecorators`, `strict`). `configPath` stays `""`, because no + * tsconfig governs the file and that fact stays visible in `ts_module`. + */ +function denoGoverned(file: string): GoverningTsConfig | undefined { + const context = denoContextFor(file); + if (context === undefined) { + return undefined; + } + const written = { ...context.root.compilerOptions, ...context.member.compilerOptions }; + const converted = ts.convertCompilerOptionsFromJson(written, context.member.dir).options; + const options: ts.CompilerOptions = { + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + allowImportingTsExtensions: true, + noEmit: true, + strict: true, + jsx: ts.JsxEmit.React, + ...converted, + }; + return { + configPath: '', + options, + moduleResolutionMode: TsModuleResolutionMode.BUNDLER, + decoratorSystem: options.experimentalDecorators === true + ? TsDecoratorSystem.LEGACY_EXPERIMENTAL + : TsDecoratorSystem.STANDARD_TC39, + fileNames: new Set(), + references: [], + }; +} + /** * `ts.ModuleResolutionKind` to the schema's token. * diff --git a/parser/src/schema/typescript/schema.json b/parser/src/schema/typescript/schema.json index 590c68a26..b91264822 100644 --- a/parser/src/schema/typescript/schema.json +++ b/parser/src/schema/typescript/schema.json @@ -829,6 +829,7 @@ "FUNCTION_EXPRESSION", "IDENTIFIER_REFERENCE", "JSX_ELEMENT", + "JSX_INTRINSIC_ELEMENT", "JSX_SELF_CLOSING", "LITERAL", "NEW_EXPRESSION", diff --git a/parser/src/test/deno-tests.ts b/parser/src/test/deno-tests.ts new file mode 100644 index 000000000..af46775dc --- /dev/null +++ b/parser/src/test/deno-tests.ts @@ -0,0 +1,548 @@ +/** + * DENO TESTS: module resolution for TypeScript in a Deno project. + * + * npx tsx src/test/deno-tests.ts # everything + * npx tsx src/test/deno-tests.ts --list # what runs, and what it proves + * + * NO DENO, NO NETWORK. Each check builds a throwaway project, and a fake + * `$DENO_DIR` laid out exactly as Deno 2 writes one (remote modules under + * `remote/https//` with the `// denoCacheMetadata=` trailer, + * npm packages unpacked under `npm/registry.npmjs.org//`), then + * runs the real resolver or the real extraction over it. + * + * The expected answers are what `deno info --json` reported for the same layouts + * on Deno 2.9: an import-map key that maps a package also serves its subpaths, a + * member's map wins over the root's, the lockfile's pinned version wins over a + * newer cached one, and a workspace member is reached by its `name` through its + * `exports`. + */ +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { extractProject } from '@/extract'; +import { clearDenoCacheState } from '@/parsers/typescript/deno/deno-cache'; +import { clearDenoConfigCache } from '@/parsers/typescript/deno/deno-config'; +import { clearDenoResolutionCache, resolveDenoSpecifier } from '@/parsers/typescript/deno/deno-resolver'; +import { compareVersions, maxSatisfying, satisfies } from '@/parsers/typescript/deno/deno-semver'; +import { ProjectLanguage } from '@/types/ProjectInfo'; +import { ProjectScanner } from '@/utils/project-scanner'; + +interface Check { + name: string; + proves: string; + rulesOut: string; + run: (tmp: string) => Promise; +} + +function write(root: string, files: Record): string { + for (const [relative, contents] of Object.entries(files)) { + const target = path.join(root, relative); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, contents); + } + return root; +} + +/** A module in the fake `$DENO_DIR` remote cache, keyed as Deno keys it. */ +function cacheRemote(denoDir: string, url: string, body: string, headers: Record = {}, + key?: string): void { + const u = new URL(url); + // `key`: the path Deno hashed, when it differs from this URL parser's spelling (`^` vs `%5E`). + const file = path.join(denoDir, 'remote', u.protocol.replace(':', ''), u.hostname, + crypto.createHash('sha256').update(key ?? u.pathname + u.search).digest('hex')); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${body}\n// denoCacheMetadata=${JSON.stringify({ headers, url, time: 0 })}`); +} + +/** A jsr package version in the fake cache: its `_meta.json` and every file given. */ +function cacheJsr(denoDir: string, name: string, version: string, exports: Record, + files: Record): void { + const manifest: Record = {}; + for (const [file, body] of Object.entries(files)) { + cacheRemote(denoDir, `https://jsr.io/${name}/${version}/${file}`, body, + { 'content-type': 'text/typescript' }); + manifest[`/${file}`] = { size: body.length, checksum: '' }; + } + cacheRemote(denoDir, `https://jsr.io/${name}/${version}_meta.json`, JSON.stringify({ manifest, exports })); +} + +function cacheJsrIndex(denoDir: string, name: string, versions: Record): void { + cacheRemote(denoDir, `https://jsr.io/${name}/meta.json`, JSON.stringify({ versions })); +} + +/** Points the resolver at a fresh fake `$DENO_DIR` and mirror for one check. */ +function freshEnvironment(tmp: string, name: string): { denoDir: string; mirror: string } { + const denoDir = path.join(tmp, `${name}-deno-dir`); + const mirror = path.join(tmp, `${name}-mirror`); + fs.mkdirSync(denoDir, { recursive: true }); + process.env.DENO_DIR = denoDir; + process.env.AXIOMCODE_DENO_MIRROR = mirror; + clearDenoConfigCache(); + clearDenoCacheState(); + clearDenoResolutionCache(); + return { denoDir, mirror }; +} + +function relativeTo(root: string, file: string | undefined): string { + return file === undefined ? '' : path.relative(root, file).split(path.sep).join('/'); +} + +function expectResolved(root: string, specifier: string, from: string, want: string, kind?: string): string | null { + const got = resolveDenoSpecifier(specifier, path.join(root, from)); + const rel = relativeTo(root, got?.resolvedFileName); + if (rel !== want) { + return `${specifier} from ${from}: resolved to ${rel}, expected ${want}`; + } + if (kind !== undefined && got?.kind !== kind) { + return `${specifier} from ${from}: kind ${got?.kind}, expected ${kind}`; + } + return null; +} + +function firstFailure(results: Array): string | null { + return results.find((r) => r !== null) ?? null; +} + +function readRows(file: string): Record[] { + const lines = fs.readFileSync(file, 'utf-8').split('\n').filter((l) => l !== ''); + const header = lines[0]!.split('\t'); + return lines.slice(1).map((line) => { + const cells = line.split('\t'); + return Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ''])); + }); +} + +const CHECKS: Check[] = [ + { + name: 'semver ranges pick what Deno picks', + proves: 'caret, tilde, partial, comparator and || ranges match the npm semantics Deno uses', + rulesOut: 'a resolver that takes the first or newest version regardless of the written range', + run: async () => { + const cases: Array<[string, string, boolean]> = [ + ['1.1.6', '^1.0.0', true], ['2.0.0', '^1.0.0', false], ['0.2.5', '^0.2.1', true], + ['0.3.0', '^0.2.1', false], ['1.2.9', '~1.2.3', true], ['1.3.0', '~1.2.3', false], + ['4.13.13', '4', true], ['4.13.13', '4.12', false], ['3.25.76', '>=3.23 <4', true], + ['1.0.0-rc.1', '^1.0.0', false], ['1.0.0-rc.2', '^1.0.0-rc.1', true], ['5.0.0', '^4 || ^5', true], + ['1.2.3', '1.2.3', true], ['1.2.4', '', true], ['1.5.0', '1.2 - 1.4', false], + ]; + for (const [version, range, want] of cases) { + if (satisfies(version, range) !== want) { + return `satisfies(${version}, "${range}") should be ${want}`; + } + } + if (maxSatisfying(['1.0.0', '1.1.6', '2.0.0'], '^1.0.0') !== '1.1.6') { + return 'maxSatisfying did not pick the highest in-range version'; + } + if (compareVersions('1.0.0-rc.1', '1.0.0') >= 0) { + return 'a pre-release must sort before its release'; + } + return null; + }, + }, + { + name: 'import map: keys, prefixes, package subpaths and scopes', + proves: 'exact keys, the longest `prefix/` key, Deno\'s package-subpath extension, scopes and an external importMap file', + rulesOut: 'a map applied as an exact lookup only, so `@/x.ts` and `hono/cors` stay unresolved', + run: async (tmp) => { + freshEnvironment(tmp, 'map'); + const root = write(path.join(tmp, 'map'), { + 'deno.json': JSON.stringify({ + imports: { '@/': './src/', 'util': './src/util.ts', 'lib/': './lib/', 'lib/deep/': './deeper/' }, + scopes: { './legacy/': { 'util': './legacy/old-util.ts' } }, + importMap: './extra-map.json', + }), + 'extra-map.json': JSON.stringify({ imports: { 'extra': './src/extra.ts' } }), + 'src/main.ts': '', 'src/util.ts': '', 'src/extra.ts': '', 'src/a.ts': '', + 'lib/x.ts': '', 'deeper/y.ts': '', 'legacy/caller.ts': '', 'legacy/old-util.ts': '', + }); + return firstFailure([ + expectResolved(root, '@/a.ts', 'src/main.ts', 'src/a.ts', 'IMPORT_MAP'), + expectResolved(root, 'util', 'src/main.ts', 'src/util.ts'), + expectResolved(root, 'lib/x.ts', 'src/main.ts', 'lib/x.ts'), + expectResolved(root, 'lib/deep/y.ts', 'src/main.ts', 'deeper/y.ts'), + expectResolved(root, 'util', 'legacy/caller.ts', 'legacy/old-util.ts'), + expectResolved(root, 'extra', 'src/main.ts', 'src/extra.ts'), + // Relative specifiers are the compiler's, never the map's. + expectResolved(root, './a.ts', 'src/main.ts', ''), + ]); + }, + }, + { + name: 'workspace: members by name, member map over root map', + proves: 'a member is reached by its name and its export subpaths, and a member\'s own import map wins over the root\'s', + rulesOut: 'a workspace analysed as unrelated programs, or a root map that shadows a member\'s entry', + run: async (tmp) => { + freshEnvironment(tmp, 'ws'); + const root = write(path.join(tmp, 'ws'), { + 'deno.json': JSON.stringify({ workspace: ['./api', './packages/*'], imports: { 'cfg': './root-cfg.ts' } }), + 'root-cfg.ts': '', + 'api/deno.json': JSON.stringify({ name: '@acme/api', imports: { 'cfg': './src/cfg.ts' } }), + 'api/src/main.ts': '', 'api/src/cfg.ts': '', + 'packages/shared/deno.json': JSON.stringify({ name: '@acme/shared', exports: { '.': './mod.ts', './ids': './src/ids.ts' } }), + 'packages/shared/mod.ts': '', 'packages/shared/src/ids.ts': '', + 'packages/other/deno.json': JSON.stringify({ name: '@acme/other', exports: './index.ts' }), + 'packages/other/index.ts': '', + }); + const scanned = await new ProjectScanner().scanForProjects(root); + const tsProjects = scanned.filter((p) => p.language === ProjectLanguage.TYPESCRIPT); + if (tsProjects.length !== 1) { + return `the workspace was discovered as ${tsProjects.length} TypeScript projects, expected 1`; + } + return firstFailure([ + expectResolved(root, '@acme/shared', 'api/src/main.ts', 'packages/shared/mod.ts', 'WORKSPACE_MEMBER'), + expectResolved(root, '@acme/shared/ids', 'api/src/main.ts', 'packages/shared/src/ids.ts'), + expectResolved(root, '@acme/other', 'api/src/main.ts', 'packages/other/index.ts'), + expectResolved(root, 'jsr:@acme/shared@^0.1.0', 'api/src/main.ts', 'packages/shared/mod.ts'), + expectResolved(root, 'cfg', 'api/src/main.ts', 'api/src/cfg.ts'), + expectResolved(root, 'cfg', 'packages/shared/mod.ts', 'root-cfg.ts'), + expectResolved(root, '@acme/shared/missing', 'api/src/main.ts', ''), + ]); + }, + }, + { + name: 'jsr: the locked version, its exports, and what it imports', + proves: 'the lockfile pins the version over a newer cached one, exports name the file, and the mirror receives the module and its transitive imports', + rulesOut: 'resolving to the newest cached version, ignoring exports, or a mirror with only the entry file', + run: async (tmp) => { + const { denoDir, mirror } = freshEnvironment(tmp, 'jsr'); + cacheJsr(denoDir, '@std/path', '1.1.6', { '.': './mod.ts', './join': './join.ts' }, { + 'mod.ts': 'export * from "./join.ts";\nimport { os } from "jsr:@std/internal@^1.0.0/os";\n', + 'join.ts': 'import { assertPath } from "./_common/assert_path.ts";\nexport function join() {}\n', + '_common/assert_path.ts': 'export function assertPath() {}\n', + }); + cacheJsr(denoDir, '@std/path', '1.2.0', { '.': './mod.ts' }, { 'mod.ts': 'export const newer = 1;\n' }); + cacheJsr(denoDir, '@std/internal', '1.0.14', { './os': './os.ts' }, { 'os.ts': 'export const os = "x";\n' }); + const root = write(path.join(tmp, 'jsr'), { + 'deno.json': JSON.stringify({ imports: { '@std/path': 'jsr:@std/path@^1.0.0' } }), + // v5 layout: `jsr` at the top level. + 'deno.lock': JSON.stringify({ version: '5', jsr: { '@std/path@1.1.6': {}, '@std/internal@1.0.14': {} } }), + 'main.ts': '', + }); + const failure = firstFailure([ + expectResolved(root, '@std/path', 'main.ts', relativeTo(root, path.join(mirror, 'jsr.io/@std/path/1.1.6/mod.ts')), 'JSR'), + expectResolved(root, '@std/path/join', 'main.ts', relativeTo(root, path.join(mirror, 'jsr.io/@std/path/1.1.6/join.ts'))), + expectResolved(root, 'jsr:@std/path@^1.0.0/join', 'main.ts', relativeTo(root, path.join(mirror, 'jsr.io/@std/path/1.1.6/join.ts'))), + expectResolved(root, 'jsr:@std/path@^1.0.0/missing', 'main.ts', ''), + expectResolved(root, 'jsr:@std/absent@1', 'main.ts', ''), + ]); + if (failure) { + return failure; + } + for (const copied of ['jsr.io/@std/path/1.1.6/_common/assert_path.ts', 'jsr.io/@std/internal/1.0.14/os.ts', 'deno.lock']) { + if (!fs.existsSync(path.join(mirror, copied))) { + return `the mirror is missing ${copied}`; + } + } + if (fs.existsSync(path.join(mirror, 'jsr.io/@std/path/1.2.0'))) { + return 'a version the lockfile does not pin was copied to the mirror'; + } + return null; + }, + }, + { + name: 'jsr: without a lockfile, the highest cached version in range', + proves: 'with no lockfile the range picks among cached versions, skipping yanked ones and ones never downloaded', + rulesOut: 'refusing every jsr import in a project with "lock": false', + run: async (tmp) => { + const { denoDir, mirror } = freshEnvironment(tmp, 'nolock'); + cacheJsrIndex(denoDir, '@hono/hono', { '4.6.0': {}, '4.7.0': {}, '4.8.0': { yanked: true }, '4.9.0': {}, '5.0.0': {} }); + cacheJsr(denoDir, '@hono/hono', '4.6.0', { '.': './src/index.ts' }, { 'src/index.ts': 'export class Hono {}\n' }); + cacheJsr(denoDir, '@hono/hono', '4.7.0', { '.': './src/index.ts', './cors': './src/cors.ts' }, + { 'src/index.ts': 'export class Hono {}\n', 'src/cors.ts': 'export function cors() {}\n' }); + cacheJsr(denoDir, '@hono/hono', '4.8.0', { '.': './src/index.ts' }, { 'src/index.ts': '' }); + cacheJsr(denoDir, '@hono/hono', '5.0.0', { '.': './src/index.ts' }, { 'src/index.ts': '' }); + const root = write(path.join(tmp, 'nolock'), { + 'deno.jsonc': '{\n // comments and trailing commas, as deno.jsonc allows\n "lock": false,\n "imports": { "hono": "jsr:@hono/hono@^4.6.0", },\n}\n', + 'main.ts': '', + }); + return firstFailure([ + expectResolved(root, 'hono', 'main.ts', relativeTo(root, path.join(mirror, 'jsr.io/@hono/hono/4.7.0/src/index.ts'))), + expectResolved(root, 'hono/cors', 'main.ts', relativeTo(root, path.join(mirror, 'jsr.io/@hono/hono/4.7.0/src/cors.ts'))), + ]); + }, + }, + { + name: 'npm: node_modules first, then Deno\'s npm cache through exports', + proves: 'an npm: import resolves into node_modules when installed there, else into $DENO_DIR/npm through the package\'s exports and types', + rulesOut: 'npm: specifiers left unresolved, or a cache lookup that ignores the package\'s own exports map', + run: async (tmp) => { + const { denoDir } = freshEnvironment(tmp, 'npm'); + const pkg = path.join(denoDir, 'npm', 'registry.npmjs.org', 'zod', '3.25.76'); + write(pkg, { + 'package.json': JSON.stringify({ name: 'zod', version: '3.25.76', exports: { '.': { types: './index.d.ts', import: './index.js' }, './v4': { types: './v4/index.d.ts' } } }), + 'index.d.ts': 'export declare const z: unknown;\n', 'index.js': '', 'v4/index.d.ts': '', + 'v4/package.json': JSON.stringify({ name: 'zod-v4-stub' }), + }); + write(path.join(denoDir, 'npm', 'registry.npmjs.org', 'zod', '4.0.0'), { + 'package.json': JSON.stringify({ name: 'zod', version: '4.0.0', types: './index.d.ts' }), 'index.d.ts': '', + }); + const root = write(path.join(tmp, 'npm'), { + 'deno.json': JSON.stringify({ imports: { 'zod': 'npm:zod@^3.23.0' } }), + 'deno.lock': JSON.stringify({ version: '4', npm: { 'zod@3.25.76': {} } }), + 'main.ts': '', + 'installed/deno.json': '{}', + 'installed/node_modules/left-pad/package.json': JSON.stringify({ name: 'left-pad', types: './index.d.ts' }), + 'installed/node_modules/left-pad/index.d.ts': '', + 'installed/main.ts': '', + }); + return firstFailure([ + expectResolved(root, 'zod', 'main.ts', relativeTo(root, path.join(pkg, 'index.d.ts')), 'NPM_CACHE'), + expectResolved(root, 'zod/v4', 'main.ts', relativeTo(root, path.join(pkg, 'v4/index.d.ts'))), + expectResolved(root, 'npm:zod@4', 'main.ts', + relativeTo(root, path.join(denoDir, 'npm', 'registry.npmjs.org', 'zod', '4.0.0', 'index.d.ts'))), + expectResolved(root, 'npm:left-pad@1', 'installed/main.ts', 'installed/node_modules/left-pad/index.d.ts', 'NODE_MODULES'), + expectResolved(root, 'npm:absent@1', 'main.ts', ''), + ]); + }, + }, + { + name: 'https: redirects, x-typescript-types and extensionless URLs', + proves: 'a URL import follows cached redirects, prefers the declarations a CDN points at, and gets an extension from its content type', + rulesOut: 'URL imports left unresolved, or mirrored under a name the parser walk would skip', + run: async (tmp) => { + const { denoDir, mirror } = freshEnvironment(tmp, 'remote'); + cacheRemote(denoDir, 'https://deno.land/x/oak/mod.ts', '', { location: '/x/oak@v12.6.1/mod.ts' }); + // `export * as` is the re-export form the compiler's light pre-processor does not report. + cacheRemote(denoDir, 'https://deno.land/x/oak@v12.6.1/mod.ts', 'export * as app from "./app.ts";\n', + { 'content-type': 'application/typescript' }); + cacheRemote(denoDir, 'https://deno.land/x/oak@v12.6.1/app.ts', 'export class Application {}\n', + { 'content-type': 'application/typescript' }); + cacheRemote(denoDir, 'https://esm.sh/preact@10.19.0', 'export {}\n', + { 'content-type': 'application/javascript', 'x-typescript-types': 'https://esm.sh/preact@10.19.0/src/index.d.ts' }); + cacheRemote(denoDir, 'https://esm.sh/preact@10.19.0/src/index.d.ts', 'export declare function h(): void;\n', + { 'content-type': 'application/typescript' }); + const root = write(path.join(tmp, 'remote'), { 'deno.json': '{}', 'main.ts': '' }); + const failure = firstFailure([ + expectResolved(root, 'https://deno.land/x/oak/mod.ts', 'main.ts', + relativeTo(root, path.join(mirror, 'deno.land/x/oak@v12.6.1/mod.ts')), 'REMOTE'), + expectResolved(root, 'https://esm.sh/preact@10.19.0', 'main.ts', + relativeTo(root, path.join(mirror, 'esm.sh/preact@10.19.0/src/index.d.ts'))), + expectResolved(root, 'https://deno.land/x/never-downloaded/mod.ts', 'main.ts', ''), + ]); + if (failure) { + return failure; + } + return fs.existsSync(path.join(mirror, 'deno.land/x/oak@v12.6.1/app.ts')) + ? null + : 'a relative import of a mirrored URL module was not copied'; + }, + }, + { + name: 'https: lockfile redirects and a caret in the URL', + proves: 'a redirect Deno recorded only in deno.lock is followed, and a URL with `^` finds the cache entry Deno hashed with the `^` as written', + rulesOut: 'partial-version CDN URLs (`esm.sh/ky@1.7`, `type-fest@^4`) left unresolved after `deno install`', + run: async (tmp) => { + const { denoDir, mirror } = freshEnvironment(tmp, 'remote-lock'); + cacheRemote(denoDir, 'https://esm.sh/ky@1.7.5', 'export default function ky() {}\n', + { 'content-type': 'application/typescript' }); + cacheRemote(denoDir, 'https://esm.sh/type-fest@^4.11.1', '', { location: 'https://esm.sh/type-fest@4.41.0' }, + '/type-fest@^4.11.1'); + cacheRemote(denoDir, 'https://esm.sh/type-fest@4.41.0', 'export type Simplify = T;\n', + { 'content-type': 'application/typescript' }); + const root = write(path.join(tmp, 'remote-lock'), { + 'deno.json': JSON.stringify({ imports: { ky: 'https://esm.sh/ky@1.7' } }), + 'deno.lock': JSON.stringify({ version: '5', redirects: { 'https://esm.sh/ky@1.7': 'https://esm.sh/ky@1.7.5' } }), + 'main.ts': '', + }); + return firstFailure([ + expectResolved(root, 'ky', 'main.ts', relativeTo(root, path.join(mirror, 'esm.sh/ky@1.7.5.ts')), 'REMOTE'), + expectResolved(root, 'https://esm.sh/type-fest@^4.11.1', 'main.ts', + relativeTo(root, path.join(mirror, 'esm.sh/type-fest@4.41.0.ts')), 'REMOTE'), + ]); + }, + }, + { + name: 'npm: the version range picks among node_modules/.deno copies', + proves: 'an npm: range the hoisted node_modules copy does not satisfy resolves into the matching node_modules/.deno/@ copy, and a package typed by its @types twin keeps the twin', + rulesOut: 'a member mapping `npm:pkg@^3` landing on the hoisted major 4, or a types-only twin rejected for its own version line', + run: async (tmp) => { + freshEnvironment(tmp, 'npm-store'); + const store = 'node_modules/.deno'; + const root = write(path.join(tmp, 'npm-store'), { + 'deno.json': JSON.stringify({ nodeModulesDir: 'manual', workspace: ['./v3'] }), + 'v3/deno.json': JSON.stringify({ name: '@x/v3', imports: { tw: 'npm:tw@^3.4.0', babel: 'npm:@babel/core@^7.28.0' } }), + 'v3/main.ts': '', + 'main.ts': '', + 'node_modules/tw/package.json': JSON.stringify({ name: 'tw', version: '4.1.0', types: './v4.d.ts' }), + 'node_modules/tw/v4.d.ts': '', + [`${store}/tw@4.1.0/node_modules/tw/package.json`]: JSON.stringify({ name: 'tw', version: '4.1.0', types: './v4.d.ts' }), + [`${store}/tw@4.1.0/node_modules/tw/v4.d.ts`]: '', + [`${store}/tw@3.4.18/node_modules/tw/package.json`]: JSON.stringify({ name: 'tw', version: '3.4.18', types: './v3.d.ts' }), + [`${store}/tw@3.4.18/node_modules/tw/v3.d.ts`]: '', + 'node_modules/@babel/core/package.json': JSON.stringify({ name: '@babel/core', version: '7.28.5', main: './lib/index.js' }), + 'node_modules/@babel/core/lib/index.js': '', + 'node_modules/@types/babel__core/package.json': JSON.stringify({ name: '@types/babel__core', version: '7.20.5', types: './index.d.ts' }), + 'node_modules/@types/babel__core/index.d.ts': '', + }); + return firstFailure([ + expectResolved(root, 'tw', 'v3/main.ts', `${store}/tw@3.4.18/node_modules/tw/v3.d.ts`, 'NODE_MODULES'), + expectResolved(root, 'npm:tw@^4', 'main.ts', 'node_modules/tw/v4.d.ts', 'NODE_MODULES'), + expectResolved(root, 'babel', 'v3/main.ts', 'node_modules/@types/babel__core/index.d.ts', 'NODE_MODULES'), + ]); + }, + }, + { + name: 'npm: a package.json dependency, and an @types twin', + proves: 'under "nodeModulesDir": "none" a bare specifier the nearest package.json declares resolves into Deno\'s npm cache, a JavaScript-only package resolves to its cached @types twin, and under "manual" (Deno\'s default beside a package.json) nothing is answered, as Deno refuses it', + rulesOut: 'Node-style members of a Deno workspace left unresolved, calls typed against a .js entry Deno never type-checks, or a cache answer where Deno demands node_modules', + run: async (tmp) => { + const { denoDir } = freshEnvironment(tmp, 'npm-pkgjson'); + const registry = path.join(denoDir, 'npm', 'registry.npmjs.org'); + write(path.join(registry, 'runner', '3.2.4'), { + 'package.json': JSON.stringify({ name: 'runner', version: '3.2.4', exports: { '.': { types: './index.d.ts' }, './config': { types: './config.d.ts' } } }), + 'index.d.ts': '', 'config.d.ts': '', + }); + write(path.join(registry, 'runner', '4.0.0'), { + 'package.json': JSON.stringify({ name: 'runner', version: '4.0.0', types: './index.d.ts' }), 'index.d.ts': '', + }); + write(path.join(registry, 'jsonly', '0.10.9'), { + 'package.json': JSON.stringify({ name: 'jsonly', version: '0.10.9', main: './main.js' }), 'main.js': '', + }); + write(path.join(registry, '@types', 'jsonly', '0.10.8'), { + 'package.json': JSON.stringify({ name: '@types/jsonly', version: '0.10.8', types: './index.d.ts' }), 'index.d.ts': '', + }); + const root = write(path.join(tmp, 'npm-pkgjson'), { + 'deno.json': JSON.stringify({ nodeModulesDir: 'none', workspace: ['./pkg'] }), + 'pkg/deno.json': JSON.stringify({ name: '@x/pkg' }), + 'manual/deno.json': '{}', + 'manual/package.json': JSON.stringify({ name: 'manual', dependencies: { runner: '^3.2.0' } }), + 'manual/main.ts': '', + 'pkg/package.json': JSON.stringify({ name: '@x/pkg', devDependencies: { runner: '^3.2.0', jsonly: 'catalog:' } }), + 'pkg/src/a.test.ts': '', + 'main.ts': '', + }); + return firstFailure([ + expectResolved(root, 'runner', 'pkg/src/a.test.ts', relativeTo(root, path.join(registry, 'runner', '3.2.4', 'index.d.ts')), 'NPM_CACHE'), + expectResolved(root, 'runner/config', 'pkg/src/a.test.ts', relativeTo(root, path.join(registry, 'runner', '3.2.4', 'config.d.ts'))), + expectResolved(root, 'jsonly', 'pkg/src/a.test.ts', relativeTo(root, path.join(registry, '@types', 'jsonly', '0.10.8', 'index.d.ts')), 'NPM_CACHE'), + // Not declared by the root's (absent) package.json: the compiler's own resolution keeps it. + expectResolved(root, 'runner', 'main.ts', ''), + expectResolved(root, 'runner', 'manual/main.ts', ''), + ]); + }, + }, + { + name: 'outside a Deno project nothing changes', + proves: 'with no deno.json above a file the resolver answers nothing, so every other TypeScript project resolves as before', + rulesOut: 'Deno rules leaking into Node projects, such as an unrelated import map or a stray mirror write', + run: async (tmp) => { + const { mirror } = freshEnvironment(tmp, 'node'); + const root = write(path.join(tmp, 'node'), { + 'package.json': JSON.stringify({ name: 'node-app' }), + 'tsconfig.json': JSON.stringify({ compilerOptions: { module: 'NodeNext' } }), + 'src/main.ts': '', + }); + for (const specifier of ['hono', 'npm:zod@3', 'jsr:@std/path', 'https://deno.land/x/oak/mod.ts', '@/a.ts']) { + if (resolveDenoSpecifier(specifier, path.join(root, 'src/main.ts')) !== undefined) { + return `${specifier} was answered in a project with no deno.json`; + } + } + return fs.existsSync(mirror) ? 'a mirror was written for a project with no deno.json' : null; + }, + }, + { + name: 'extraction: import rows, links and the vendor directory', + proves: 'ts_import carries the Deno resolution kinds and module links, unresolved jsr: rows still name their package, and vendor/ is not walked as project source', + rulesOut: 'a resolver that is right in isolation but never reaches the rows the engine reads', + run: async (tmp) => { + const { denoDir } = freshEnvironment(tmp, 'extract'); + cacheJsr(denoDir, '@std/path', '1.1.6', { '.': './mod.ts' }, { 'mod.ts': 'export function join() {}\n' }); + const root = write(path.join(tmp, 'extract'), { + 'deno.json': JSON.stringify({ workspace: ['./api', './shared'], vendor: true, imports: { '@std/path': 'jsr:@std/path@^1.0.0' } }), + 'deno.lock': JSON.stringify({ version: '5', jsr: { '@std/path@1.1.6': {} } }), + 'api/deno.json': JSON.stringify({ name: '@acme/api', imports: { '@/': './src/', 'gone': 'jsr:@std/gone@^1.0.0' } }), + 'api/src/main.ts': [ + 'import { join } from "@std/path";', + 'import { formatUser } from "@acme/shared";', + 'import { audit } from "@/audit.ts";', + 'import { lost } from "gone";', + 'export function main() { audit(formatUser(join())); lost(); }', + ].join('\n'), + 'api/src/audit.ts': 'export function audit(m: string): void {}\n', + 'shared/deno.json': JSON.stringify({ name: '@acme/shared', exports: './mod.ts' }), + 'shared/mod.ts': 'export { formatUser } from "@acme/shared/format";\n', + 'shared/format.ts': 'export function formatUser(n: string): string { return n; }\n', + 'vendor/jsr.io/@std/path/1.1.6/mod.ts': 'export function join() {}\n', + }); + // The member exports its own subpath, which the barrel above re-exports by name. + write(root, { 'shared/deno.json': JSON.stringify({ name: '@acme/shared', exports: { '.': './mod.ts', './format': './format.ts' } }) }); + clearDenoConfigCache(); + const out = path.join(tmp, 'extract-ir'); + await extractProject({ projectPath: root, versionLink: 'v1', outputDir: out, layout: 'per-language' }); + const imports = readRows(path.join(out, 'typescript', 'all-typescript-imports.csv')); + const byPath = new Map(imports.map((r) => [r.importedPath, r])); + const want: Array<[string, string, boolean, string]> = [ + ['@std/path', 'JSR_PACKAGE', false, '@std/path'], + ['@acme/shared', 'WORKSPACE_PACKAGE', true, '@acme/shared'], + ['@/audit.ts', 'PATHS_ALIAS', true, ''], + ['gone', 'UNRESOLVED', false, '@std/gone'], + ]; + for (const [specifier, kind, linked, packageName] of want) { + const row = byPath.get(specifier); + if (row === undefined) { + return `no ts_import row for ${specifier}`; + } + if (row.resolutionKind !== kind) { + return `${specifier}: resolutionKind ${row.resolutionKind}, expected ${kind}`; + } + if ((row.resolvedModuleLinkHash !== '') !== linked) { + return `${specifier}: module link ${row.resolvedModuleLinkHash === '' ? 'missing' : 'present'}, expected ${linked ? 'present' : 'missing'}`; + } + if (row.packageName !== packageName) { + return `${specifier}: packageName "${row.packageName}", expected "${packageName}"`; + } + } + const modules = readRows(path.join(out, 'typescript', 'all-typescript-modules.csv')); + if (modules.some((m) => (m.filePath ?? '').startsWith('vendor/'))) { + return 'vendor/ was walked as project source'; + } + const exports = readRows(path.join(out, 'typescript', 'all-typescript-exports.csv')); + const reExport = exports.find((e) => e.sourceSpecifier === '@acme/shared/format'); + if (reExport === undefined || reExport.resolvedSourceModuleLinkHash === '') { + return 'a barrel re-exporting a workspace member by name was not linked to the member\'s module'; + } + const strictRow = modules.find((m) => m.filePath === 'api/src/main.ts'); + if (strictRow?.moduleResolutionMode !== 'BUNDLER') { + return `a Deno file compiled under ${strictRow?.moduleResolutionMode}, expected BUNDLER`; + } + return null; + }, + }, +]; + +async function main(): Promise { + if (process.argv.includes('--list')) { + for (const c of CHECKS) console.log(`${c.name}\n proves: ${c.proves}\n rules out: ${c.rulesOut}`); + return 0; + } + // Real path: the compiler resolves through symlinks, and macOS's temp dir is one. + const tmp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'deno-tests-'))); + const saved = { DENO_DIR: process.env.DENO_DIR, AXIOMCODE_DENO_MIRROR: process.env.AXIOMCODE_DENO_MIRROR }; + let failed = 0; + try { + for (const check of CHECKS) { + let result: string | null; + try { + result = await check.run(tmp); + } catch (e) { + result = `threw: ${(e as Error).stack ?? String(e)}`; + } + console.log(`${result === null ? 'PASS' : 'FAIL'} ${check.name}`); + if (result !== null) { + console.log(` ${result}`); + failed += 1; + } + } + } finally { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; else process.env[k] = v; + } + fs.rmSync(tmp, { recursive: true, force: true }); + } + console.log(`${CHECKS.length - failed}/${CHECKS.length} Deno checks passed`); + return failed ? 1 : 0; +} + +void main().then((code) => process.exit(code)); diff --git a/parser/src/test/typescript-tests.ts b/parser/src/test/typescript-tests.ts index fd295003d..82cb7af5b 100644 --- a/parser/src/test/typescript-tests.ts +++ b/parser/src/test/typescript-tests.ts @@ -31,6 +31,7 @@ import * as os from 'os'; import * as ts from 'typescript'; import * as path from 'path'; +import { jsxTagReference } from '@/parsers/typescript/extractors/ts-expression-extractor'; import { IrCompletenessReport } from '@/parsers/typescript/extractors/ts-ir-completeness'; import { verifyRelationFileStreaming } from '@/workflows/typescript/ts-relation-writer'; import { TypeScriptProjectAnalyzer } from '@/workflows/typescript/typescript-project-analyzer'; @@ -1414,7 +1415,9 @@ function irCompleteness(): number { * The component element is a call too: `` is `Badge({...})`, a * JSX_COMPONENT_CALL site, wherever it sits -- a child, a ternary branch, the * body of a `.map` arrow. The CONTROL is the intrinsic element: `
` and - * `` name no declaration and must produce no call site at all. + * `` name no declaration and must produce no call site at all. It is + * still an element, a childless JSX_INTRINSIC_ELEMENT row, so that + * `', + ' {flag && }', + ' ', + ' ', '
', ' );', '}', @@ -1493,6 +1502,15 @@ async function jsxBraceExpressionsWalked(): Promise { } } let expected = 0; + // Expression rows by position, so an element's row can be asserted. + const exprRows = relation(outputDir, 'all-typescript-expressions.csv') + .filter((r) => (modulePath.get(r.tsModuleLinkHash ?? '') ?? '').endsWith('panel.tsx')); + const exprKindsAt = new Map(); + for (const r of exprRows) { + const key = `panel.tsx:${r.startLine}:${r.startColumn}`; + exprKindsAt.set(key, [...(exprKindsAt.get(key) ?? []), r.kind ?? '']); + } + const parents = new Set(exprRows.map((r) => r.parentExpressionHash ?? '')); let components = 0; let intrinsics = 0; const walk = (node: ts.Node): void => { @@ -1507,12 +1525,43 @@ async function jsxBraceExpressionsWalked(): Promise { failures.push(`no JSX_COMPONENT_CALL at ${key} for \`${node.getText(sf).slice(0, 44)}\`` + ` (found ${kindAt.get(key) ?? 'nothing'})`); } + // The control: a component keeps its call-shaped row, never the intrinsic kind. + const want = ts.isJsxElement(node) ? 'JSX_ELEMENT' : 'JSX_SELF_CLOSING'; + const kinds = exprKindsAt.get(key) ?? []; + if (!kinds.includes(want) || kinds.includes('JSX_INTRINSIC_ELEMENT')) { + failures.push(`component at ${key} has expression kind(s) [${kinds.join(', ')}], ` + + `expected ${want}`); + } } else { intrinsics += 1; if (emitted.has(key)) { failures.push(`intrinsic \`<${tag.getText(sf)}>\` at ${key} became a call site ` + `(${kindAt.get(key)}) -- it names no declaration`); } + // ...but it IS an element: one JSX_INTRINSIC_ELEMENT row, with no child + // (no tag reference for anything to resolve), so its attributes have an + // element to belong to, as in the JavaScript parser. + const rows = exprRows.filter((r) => `panel.tsx:${r.startLine}:${r.startColumn}` === key + && r.kind === 'JSX_INTRINSIC_ELEMENT'); + if (rows.length !== 1) { + failures.push(`intrinsic \`<${tag.getText(sf)}>\` at ${key} has ${rows.length} ` + + 'JSX_INTRINSIC_ELEMENT row(s), expected 1'); + } else if (parents.has(rows[0]!.tsExpressionUniqueHash ?? '')) { + failures.push(`intrinsic \`<${tag.getText(sf)}>\` at ${key} has a child row -- ` + + 'its tag references nothing, so nothing hangs off it'); + } + // An attribute's value is still its own rooted tree. + const attributes = ts.isJsxElement(node) ? node.openingElement.attributes : node.attributes; + for (const attribute of attributes.properties) { + if (ts.isJsxAttribute(attribute) && attribute.initializer + && ts.isJsxExpression(attribute.initializer) && attribute.initializer.expression) { + const v = sf.getLineAndCharacterOfPosition(attribute.initializer.expression.getStart(sf)); + const vkey = `panel.tsx:${v.line + 1}:${v.character + 1}`; + if ((exprKindsAt.get(vkey) ?? []).length === 0) { + failures.push(`attribute \`${attribute.getText(sf)}\` at ${vkey} has no value row`); + } + } + } } } if (ts.isCallExpression(node) || ts.isNewExpression(node)) { @@ -1528,6 +1577,33 @@ async function jsxBraceExpressionsWalked(): Promise { }; ts.forEachChild(sf, walk); + // The tag of a plain element is on its row, as written (`div`, `my-el`, `svg:circle`). The CONTROL is + // the component element, whose row keeps an empty potentialQualifiedName: its tag is the METHOD_NAME + // child that resolves, unchanged. The expectation is read off the AST. + let tagged = 0; + const walkTags = (node: ts.Node): void => { + if (ts.isJsxSelfClosingElement(node) || ts.isJsxElement(node)) { + const opening = ts.isJsxElement(node) ? node.openingElement : node; + const p = sf.getLineAndCharacterOfPosition(node.getStart(sf)); + const at = `${p.line + 1}:${p.character + 1}`; + const row = exprRows.find((r) => `${r.startLine}:${r.startColumn}` === at && (r.kind ?? '').startsWith('JSX_')); + const tag = opening.tagName.getText(sf); + const want = jsxTagReference(node) === undefined ? tag : ''; + if (row === undefined) { + failures.push(`no element row at ${at} for <${tag}>`); + } else if ((row.potentialQualifiedName ?? '') !== want) { + failures.push(`<${tag}> at ${at}: potentialQualifiedName "${row.potentialQualifiedName}", expected "${want}"`); + } else if (want !== '') { + tagged += 1; + } + } + ts.forEachChild(node, walkTags); + }; + ts.forEachChild(sf, walkTags); + if (tagged !== intrinsics) { + failures.push(`${tagged} of ${intrinsics} intrinsic element(s) carry their tag`); + } + // The edge roles that stay reserved: attributes and children are rooted trees. for (const [file, names] of [ ['all-typescript-call-sites.csv', ['callKind']], @@ -1545,7 +1621,9 @@ async function jsxBraceExpressionsWalked(): Promise { console.log(` ${expected} call/new node(s) in JSX attributes, children, spreads and a ` + `nested map; ${expected - missingCalls} emitted; ${components} component element(s) as ` + - `calls, ${intrinsics} intrinsic element(s) as none; ${RESERVED_TSX_VALUES.size} reserved ` + + `calls, ${intrinsics} intrinsic element(s) as JSX_INTRINSIC_ELEMENT rows and no call ` + + `(${tagged} with their tag); ` + + `${RESERVED_TSX_VALUES.size} reserved ` + 'value(s) still empty'); for (const f of failures.slice(0, 10)) console.log(` ${f}`); fs.rmSync(root, { recursive: true, force: true }); @@ -4630,6 +4708,46 @@ async function solutionStyleRootDelegatesToReferences(): Promise { return 0; } +/** + * A framework writes its programs' configs into a generated directory and points the + * root at them: Nuxt's `tsconfig.json` is `files: []` plus references to + * `.nuxt/tsconfig.app.json` (which includes everything under `../app`) and `.nuxt/tsconfig.server.json`. + * Each referenced config is a program, but its files are OUTSIDE its directory, so a + * program rooted at `.nuxt/` walked none of them: 2 of 63 files of a stock Nuxt 4 app + * were extracted, and nothing said the rest were missing. + */ +async function configProgramsReachOutsideTheirDirectory(): Promise { + const bundler = { target: 'ES2022', module: 'ESNext', moduleResolution: 'Bundler', strict: true }; + const { outputDir, cleanup } = await analyseProgramsInline('ts-config-programs-', { + 'tsconfig.json': JSON.stringify({ files: [], references: [ + { path: './.nuxt/tsconfig.app.json' }, { path: './.nuxt/tsconfig.server.json' }, + ] }), + '.nuxt/tsconfig.app.json': JSON.stringify({ compilerOptions: bundler, include: ['../app/**/*'] }), + '.nuxt/tsconfig.server.json': JSON.stringify({ compilerOptions: bundler, include: ['../server/**/*'] }), + 'app/lib/greet.ts': 'export function greet(n: string): string { return `hi ${n}`; }', + 'app/page.ts': 'import { greet } from "./lib/greet"; export function page(): string { return greet("a"); }', + 'server/api/hello.ts': 'export function hello(): string { return "hello"; }', + }); + const failures: string[] = []; + const modules = relation(outputDir, 'all-typescript-modules.csv'); + const governed = new Map(modules.map((r) => [r.filePath ?? '', r.tsConfigPath ?? ''])); + const want: [string, string][] = [['app/lib/greet.ts', 'tsconfig.app.json'], ['app/page.ts', 'tsconfig.app.json'], + ['server/api/hello.ts', 'tsconfig.server.json']]; + for (const [file, config] of want) { + const got = governed.get(file); + if (got === undefined) failures.push(`${file} is missing (have ${[...governed.keys()].join(', ') || 'nothing'})`); + else if (!got.endsWith(`.nuxt/${config}`)) failures.push(`${file} is governed by '${got}', want .nuxt/${config}`); + } + const skipped = relation(outputDir, 'skipped-typescript-files.csv').filter((r) => !(r.filePath ?? '').startsWith('.nuxt/')); + if (skipped.length > 0) failures.push(`skipped: ${skipped.map((r) => `${r.filePath} ${r.reason}`).join(', ')}`); + cleanup(); + if (failures.length > 0) { + return fail(failures.join('\n ')); + } + console.log(` config programs outside their directory: ${modules.length} module row(s), app and server each under its own config`); + return 0; +} + // --------------------------------------------------------------------------- // 34. strictBindCallApply is emitted RESOLVED, not as written // --------------------------------------------------------------------------- @@ -4708,6 +4826,7 @@ async function strictBindCallApplyIsResolved(): Promise { const CHECKS: Check[] = [ { name: 'strictBindCallApply is emitted resolved', proves: 'the flag is emitted as the checker resolves it — implied by strict, not by individual strict flags, explicit false winning — so a consumer can pick between the Function and CallableFunction overloads of call/apply/bind', run: strictBindCallApplyIsResolved }, { name: 'a solution-style root delegates to its references', proves: 'a root tsconfig with files: [] and references extracts the files its same-directory references claim under those configs, reaches a referenced subdirectory as a nested program, and reports a file no program claims as NO_PROGRAM_CLAIMS_FILE (#660)', run: solutionStyleRootDelegatesToReferences }, + { name: 'a referenced config claims files outside its directory', proves: 'a solution-style root whose references live in a generated directory (.nuxt/tsconfig.app.json including ../app/**) extracts every file each referenced config claims, under that config, as its own program', run: configProgramsReachOutsideTheirDirectory }, { name: 'every program is extracted, none are merged', proves: 'each program under a root reaches one flat relation set with unique keys, while the module link passes stay inside a program so two global scopes are never merged', run: everyProgramIsExtractedAndNoneAreMerged }, { name: 'streamed relations are well formed', proves: 'rows written as extraction proceeds produce one header per relation, a file for every relation, no leftover temporaries and no dropped tail', run: streamedRelationsAreWellFormed }, { name: 'streamed read-back catches a torn row', proves: 'the chunked verifier accepts a multi-MB well-formed file and still rejects short, long, truncated, U+2028-bearing and mid-file tears', run: streamedVerificationCatchesTornRows }, @@ -4721,7 +4840,7 @@ const CHECKS: Check[] = [ { name: 'tsc-adjudicated resolution', proves: 'every resolved call target equals getResolvedSignature', run: tscAdjudicatedResolution }, { name: 'type-only isolation', proves: 'no type-only construct reaches the call graph', run: typeOnlyIsolation }, { name: 'TSX reserved but empty', proves: 'the JSX attribute and child edge roles carry no rows', run: tsxReservedButEmpty }, - { name: 'JSX brace expressions are walked', proves: 'a call inside a JSX brace is an ordinary call site, a component element is a JSX_COMPONENT_CALL, an intrinsic element is none', run: jsxBraceExpressionsWalked }, + { name: 'JSX brace expressions are walked', proves: 'a call inside a JSX brace is an ordinary call site, a component element is a JSX_COMPONENT_CALL, an intrinsic element is a childless JSX_INTRINSIC_ELEMENT row carrying its tag and no call', run: jsxBraceExpressionsWalked }, { name: 'destructuring records its source', proves: 'a bound name carries the property or index it binds, so a renamed or positional binding is recoverable', run: destructuringRecordsItsSource }, { name: 'no emitted value can split a row', proves: 'no value contains a character a consumer treats as a line break, so a row cannot tear', run: noValueCanSplitARow }, { name: 'signatures link their return type', proves: 'a call through any callable shape reaches a result type, so a chain does not stop at it', run: signaturesLinkTheirReturnType }, diff --git a/parser/src/workflows/typescript/typescript-project-analyzer.ts b/parser/src/workflows/typescript/typescript-project-analyzer.ts index 3c7642595..2f719fe37 100644 --- a/parser/src/workflows/typescript/typescript-project-analyzer.ts +++ b/parser/src/workflows/typescript/typescript-project-analyzer.ts @@ -25,6 +25,8 @@ import { ModuleGraphFacts, newCompletenessAccumulator, } from '@/parsers/typescript/extractors/ts-ir-completeness'; +import { isDenoVendorDirectory } from '@/parsers/typescript/deno/deno-config'; +import { resolveDenoSpecifier } from '@/parsers/typescript/deno/deno-resolver'; import { moduleHashFor } from '@/parsers/typescript/extractors/ts-module-extractor'; import { PackageJsonResolver } from '@/parsers/javascript/package-json-resolver'; import { extractTsPackageEntries } from '@/parsers/typescript/ts-package-entry-extractor'; @@ -83,6 +85,13 @@ import { export interface SharedWriteContext { readonly writers: Map; readonly writerFor: (filename: string) => TsRelationWriter; + /** + * Every file a program in this run has already taken, normalised. A file no + * config claims can join two programs (one it imports, one that resolves an + * import into it), and a module row is a pure function of its path, so a second + * extraction would write the same primary key twice. The first program keeps it. + */ + readonly analyzedFiles: Set; } export interface TypeScriptAnalysisOptions { @@ -106,6 +115,13 @@ export interface TypeScriptAnalysisOptions { * into such a package unresolved. Stated by the caller, as JavaScript's `libraryRoot` is. */ readonly libraryRoot?: boolean; + /** + * The program is this CONFIG, not the directory `rootDir`: the files it claims, found + * by walking `rootDir`. Set by analyzePrograms for a config whose `include` reaches + * outside its own directory -- a framework's generated `.nuxt/tsconfig.app.json` + * claims `../app/**`, so rooting that program at `.nuxt/` walked none of its files. + */ + readonly programConfig?: string; } export interface TypeScriptAnalysisSummary { @@ -132,6 +148,14 @@ export interface TypeScriptAnalysisSummary { * deliberately left out. Ordered by how many files each accounts for. */ readonly nestedProgramRoots: readonly string[]; + /** + * The other programs that are a CONFIG rather than a directory: each claims files + * outside the directory it sits in, so walking that directory would find none of + * them. analyzePrograms runs each over the whole root (TypeScriptAnalysisOptions.programConfig). + */ + readonly nestedProgramConfigs: readonly string[]; + /** Every file a config other than this program's claims, by that config: what the other programs owe. */ + readonly claimedElsewhere: ReadonlyMap; readonly extractionErrors: number; readonly counts: Record; /** @@ -182,20 +206,48 @@ export class TypeScriptProjectAnalyzer { // A nested program can nest further, so roots drain from a queue rather // than one level of walking. Keyed by resolved path because two programs // can decline files to each other, which would otherwise recur forever. - const queue = [path.resolve(options.rootDir)]; - const visited = new Set(queue); + // A program is a directory, or a config whose files lie outside its directory; both + // are keyed by their resolved path, which a directory and a config never share. + const top = path.resolve(options.rootDir); + const queue: { rootDir: string; programConfig?: string }[] = [{ rootDir: top }]; + const visited = new Set([top]); + const owed = new Map(); while (queue.length > 0) { - const programRoot = queue.shift()!; - const summary = await this.analyze({ ...options, rootDir: programRoot }, shared); + const program = queue.shift()!; + const summary = await this.analyze({ ...options, ...program }, shared); summaries.push(summary); + for (const [file, config] of summary.claimedElsewhere) { + owed.set(file, config); + } for (const nested of summary.nestedProgramRoots) { const resolved = path.resolve(nested); if (!visited.has(resolved)) { visited.add(resolved); - queue.push(resolved); + queue.push({ rootDir: resolved }); + } + } + for (const config of summary.nestedProgramConfigs) { + const resolved = path.resolve(config); + if (!visited.has(resolved)) { + visited.add(resolved); + queue.push({ rootDir: top, programConfig: resolved }); } } } + // A file a config claims that no program run extracted would otherwise be absent from + // every relation with nothing saying so -- the shape that hid a whole application. + const anchor = pathAnchorFor(top, options.baseMservPath); + const hash = options.serviceVersionLink !== undefined + ? EntityUtils.generateEntityHash(ENTITY_IDENTIFIERS.SERVICE_VERSION, options.serviceVersionLink) + : options.serviceVersionLinkHash ?? ''; + const missed = [...owed].filter(([file]) => !shared.analyzedFiles.has(path.normalize(file))); + for (const [file, config] of missed) { + this.recordSkip(file, anchor, options, hash, SkippedFileReason.PROGRAM_NOT_EXTRACTED, + `${toRelative(anchor, config)} claims the file, and no program run over this root extracted it`); + } + if (missed.length > 0) { + await this.exportSkippedFilesCsv(options.outputDir); + } await this.publishShared(shared); return mergeSummaries(summaries); } @@ -206,6 +258,7 @@ export class TypeScriptProjectAnalyzer { const writers = new Map(); return { writers, + analyzedFiles: new Set(), writerFor: (filename: string): TsRelationWriter => { const existing = writers.get(filename); if (existing) { @@ -267,10 +320,22 @@ export class TypeScriptProjectAnalyzer { // each of which has its own config and its own expectations. // Every emitted path hangs off this, not off rootDir — see pathAnchorFor. const pathAnchor = pathAnchorFor(rootDir, options.baseMservPath); - const rootProgram = filesOfRootProgram(rootDir, configResolver); - const files = (rootProgram?.files ?? collectTypeScriptFiles(rootDir, excludes)).sort(); + const rootProgram = filesOfRootProgram(rootDir, configResolver, options.programConfig); + const files = (rootProgram?.files ?? collectTypeScriptFiles(rootDir, excludes)).sort() + .filter((file) => !shared?.analyzedFiles.has(path.normalize(file))); + for (const file of files) { + shared?.analyzedFiles.add(path.normalize(file)); + } const filesInOtherPrograms = rootProgram?.others.length ?? 0; - const nestedProgramRoots = programRootsOf(rootProgram?.others ?? [], configResolver); + const { roots: nestedProgramRoots, configs: nestedProgramConfigs } = + programsOf(rootProgram?.others ?? [], configResolver); + const claimedElsewhere = new Map(); + for (const file of rootProgram?.others ?? []) { + const governing = configResolver.resolve(file).configPath; + if (governing !== '') { + claimedElsewhere.set(path.normalize(file), path.resolve(governing)); + } + } // Every module hash up front, from PATHS ALONE. This is what lets a module // augmentation in file B key its declarations under file A's hash without @@ -505,7 +570,12 @@ export class TypeScriptProjectAnalyzer { // A re-export's source module may be parsed after the file that re-exports // from it, so the link is made here. Still the MODULE graph, and // ts_export.resolvedSourceModuleLinkHash is the parser's own column. - linkReExportSources(moduleGraph); + linkReExportSources(moduleGraph, (fromFilePath, specifier) => { + const deno = resolveDenoSpecifier(specifier, path.join(pathAnchor, fromFilePath)); + return deno === undefined || deno.resolvedFileName === '' + ? undefined + : toRelative(pathAnchor, deno.resolvedFileName); + }); // Reads the accumulated rows and mutates nothing. Cross-file CALL resolution // used to happen here and has been retracted: following an import to a // declaring file is `type-resolution.dl` rewritten in TypeScript. What runs @@ -566,6 +636,8 @@ export class TypeScriptProjectAnalyzer { filesAnalysed: analysed, filesInOtherPrograms, nestedProgramRoots, + nestedProgramConfigs, + claimedElsewhere, extractionErrors: this.skippedFiles.filter( (f) => f.reason === SkippedFileReason.EXTRACTION_ERROR ).length, @@ -703,7 +775,8 @@ export class TypeScriptProjectAnalyzer { */ function filesOfRootProgram( rootDir: string, - configResolver: TsConfigResolver + configResolver: TsConfigResolver, + programConfig?: string ): { readonly files: string[]; readonly others: string[]; @@ -711,7 +784,9 @@ function filesOfRootProgram( /** What the closure walk already read, so the extraction pass reads nothing twice. */ readonly texts: Map; } | undefined { - const configPath = path.join(rootDir, 'tsconfig.json'); + // A config program (programConfig) is that config's files wherever they lie under + // rootDir; a directory program is rootDir's own tsconfig.json. + const configPath = programConfig ?? path.join(rootDir, 'tsconfig.json'); if (!fs.existsSync(configPath)) { return undefined; } @@ -730,7 +805,7 @@ function filesOfRootProgram( continue; } seenReferences.add(referenced); - if (path.dirname(referenced) === path.resolve(rootDir)) { + if (path.dirname(referenced) === path.dirname(path.resolve(configPath))) { rootConfigs.add(referenced); pending.push(...(configResolver.configAt(referenced)?.references ?? [])); } @@ -743,7 +818,11 @@ function filesOfRootProgram( if (governing.configPath !== '' && rootConfigs.has(path.resolve(governing.configPath))) { claimed.push(file); } else if (governing.configPath === '') { - unclaimed.push(file); + // A file no config claims is the directory program's to pull in or report; a + // config program that also walked it would report it twice. + if (programConfig === undefined) { + unclaimed.push(file); + } } else { others.push(file); } @@ -790,8 +869,10 @@ function filesOfRootProgram( const sf = ts.createSourceFile(file, script.text, ts.ScriptTarget.Latest, false, script.scriptKind ?? (file.endsWith('.tsx') ? ts.ScriptKind.TSX : JS_JOINER.test(file) ? ts.ScriptKind.JS : ts.ScriptKind.TS)); for (const specifier of importSpecifiersOf(sf)) { - const resolved = ts.resolveModuleName(specifier, file, rootOptions, ts.sys, resolutionCache) - .resolvedModule?.resolvedFileName ?? resolveVueSpecifier(specifier, file); + const resolved = (resolveDenoSpecifier(specifier, file)?.resolvedFileName || undefined) + ?? ts.resolveModuleName(specifier, file, rootOptions, ts.sys, resolutionCache) + .resolvedModule?.resolvedFileName + ?? resolveVueSpecifier(specifier, file); if (resolved !== undefined) { keys.push(path.normalize(resolved)); } @@ -852,28 +933,43 @@ function filesOfRootProgram( } /** - * The distinct program roots that own `files`, busiest first. + * The other programs that own `files`: directories, busiest first, and configs. * * Reporting a count of excluded files tells a caller that something is missing - * without telling them what to do about it. These are the directories to point - * a further run at. + * without telling them what to do about it. These are what to point a further run at. + * A config whose files all lie under its own directory is that directory's program, as + * a package's tsconfig is. One that claims a file OUTSIDE its directory cannot be + * rooted there -- a walk of `.nuxt/` finds none of the `../app/**` files its generated + * config claims, and they were then extracted by no program at all -- so it is returned + * as a config, to be run over the whole root. */ -function programRootsOf( +function programsOf( files: readonly string[], configResolver: TsConfigResolver -): string[] { +): { roots: string[]; configs: string[] } { const counts = new Map(); + const byConfig = new Map(); for (const file of files) { const governing = configResolver.resolve(file); if (governing.configPath === '') { continue; } - const root = path.dirname(path.resolve(governing.configPath)); - counts.set(root, (counts.get(root) ?? 0) + 1); + const config = path.resolve(governing.configPath); + byConfig.set(config, [...(byConfig.get(config) ?? []), file]); + } + const configs: string[] = []; + for (const [config, owned] of byConfig) { + const dir = path.dirname(config); + if (owned.some((f) => !path.resolve(f).startsWith(dir + path.sep))) { + configs.push(config); + } else { + counts.set(dir, (counts.get(dir) ?? 0) + owned.length); + } } - return [...counts.entries()] + const roots = [...counts.entries()] .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) .map(([root]) => root); + return { roots, configs: configs.sort() }; } /** Every module specifier a file imports, re-exports, or imports dynamically. */ @@ -960,7 +1056,7 @@ function collectTypeScriptFiles(dir: string, excludes: ReadonlySet, only const full = path.join(current, entry.name); if (entry.isDirectory()) { if (!excludes.has(entry.name) && !entry.name.startsWith('.') && !isGitIgnoredDir(full) - && !isGeneratedOutputDirectory(current, entry.name)) { + && !isGeneratedOutputDirectory(current, entry.name) && !isDenoVendorDirectory(current, entry.name)) { walk(full); } continue; @@ -1037,6 +1133,8 @@ function mergeSummaries( // declined, so the last program's figure is what no program claims at all. filesInOtherPrograms: last?.filesInOtherPrograms ?? 0, nestedProgramRoots: [], + nestedProgramConfigs: [], + claimedElsewhere: new Map(), extractionErrors: sum((s) => s.extractionErrors), // Row counts are read from the SHARED writers, which already span every // program. Summing the per-program counts would multiply every relation, diff --git a/plugins/axiomcode/.claude-plugin/plugin.json b/plugins/axiomcode/.claude-plugin/plugin.json index 917704f8c..7e4ba77b7 100644 --- a/plugins/axiomcode/.claude-plugin/plugin.json +++ b/plugins/axiomcode/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "axiomcode", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", - "version": "0.2.0", + "version": "0.2.1", "author": { "name": "AxiomCode" } diff --git a/plugins/axiomcode/.codex-plugin/plugin.json b/plugins/axiomcode/.codex-plugin/plugin.json index 82d1aa414..f7780f510 100644 --- a/plugins/axiomcode/.codex-plugin/plugin.json +++ b/plugins/axiomcode/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.2.0", + "version": "0.2.1", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", "author": { "name": "AxiomCode", "url": "https://github.com/AxiomCodeAI/axiomcodegraph" }, "homepage": "https://github.com/AxiomCodeAI/axiomcodegraph", diff --git a/plugins/axiomcode/.cursor-plugin/plugin.json b/plugins/axiomcode/.cursor-plugin/plugin.json index 06402050e..ce3908644 100644 --- a/plugins/axiomcode/.cursor-plugin/plugin.json +++ b/plugins/axiomcode/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.2.0", + "version": "0.2.1", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed. Java, TypeScript, Python, JavaScript.", "author": { "name": "AxiomCode" }, "homepage": "https://github.com/AxiomCodeAI/axiomcodegraph", diff --git a/plugins/axiomcode/hooks/_host.py b/plugins/axiomcode/hooks/_host.py index e133112a2..5ae1b7335 100644 --- a/plugins/axiomcode/hooks/_host.py +++ b/plugins/axiomcode/hooks/_host.py @@ -34,10 +34,22 @@ GEMINI = None # the Gemini event name this hook was called for, once read() has seen one -def read(): - """the event from stdin, with Cursor's or Gemini's names mapped onto the ones the hooks use""" +def orphaned(): + """ax_orphan.orphaned for the install these hooks belong to: '' while it is current""" + try: + sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'skills', 'axiomcode', 'scripts')) + import ax_orphan + return ax_orphan.orphaned() + except Exception: + return '' + +def read(orphan_ok=False): + """the event from stdin, with Cursor's or Gemini's names mapped onto the ones the hooks use. A hook of an ORPHANED + older install (ax_orphan.orphaned: an agent session that outlived an upgrade) exits here, silent, so it writes nothing + into a repository's .axiomcode; only the refresh hook (orphan_ok) goes on, to say at a session start and on a prompt that the session wants a restart""" global GEMINI ev = json.load(sys.stdin) + if not orphan_ok and orphaned(): sys.exit(0) if isinstance(ev, dict) and ev.get('hook_event_name') in GEMINI_EVENTS: GEMINI = ev['hook_event_name'] ev['hook_event_name'] = GEMINI_EVENTS[GEMINI] diff --git a/plugins/axiomcode/hooks/refresh.py b/plugins/axiomcode/hooks/refresh.py index e5798d22d..87bf297e5 100644 --- a/plugins/axiomcode/hooks/refresh.py +++ b/plugins/axiomcode/hooks/refresh.py @@ -14,7 +14,7 @@ import _host, _where try: - ev = _host.read() + ev = _host.read(orphan_ok=True) # an orphaned install's hook only says so (ax_fresh.hook_kick) except Exception: sys.exit(0) cwd = ev.get('cwd') or os.getcwd() diff --git a/plugins/axiomcode/mcp/server.py b/plugins/axiomcode/mcp/server.py index 27c4c3574..b1ebc5e62 100755 --- a/plugins/axiomcode/mcp/server.py +++ b/plugins/axiomcode/mcp/server.py @@ -164,10 +164,26 @@ async def call_tool(self, name, arguments, *a, **k): # the SDK: refused as a # the flags whose next argument is their value, as the dispatcher skips them when it looks for the repository VALUED = {'--in', '--from', '--budget', '--seeds', '--depth', '--limit', '--tests-in', '--kind', '--range', '--old', '--new', '--file', '--page', '--page-budget', '--out'} +def orphaned(root=None): + """ax_orphan.orphaned for the install this server runs now (plugin_root): '' while it is current. An install whose + scripts are gone cannot be imported, so that is said here""" + root = root or plugin_root() + try: return scripts_module('ax_orphan').orphaned(root) + except Exception: + if os.path.isfile(os.path.join(root, 'skills', 'axiomcode', 'scripts', 'ax_fresh.py')): return '' + return (f"this is an orphaned older axiomcode (its install directory {root} is gone: a newer version was installed in " + "its place), still running in an agent session started before the upgrade; it writes nothing into the " + "repository's .axiomcode and rebuilds nothing. Restart the agent session so that only the installed axiomcode runs") + def _timer(interval): import time while True: time.sleep(max(1.0, min(60.0, interval / 3))) + # AN ORPHANED SERVER STOPS (ax_orphan.orphaned): its install was replaced and no newer one is recorded to run in its + # place, so all it could still do is rebuild graphs with the older version, the turn-taking an upgrade started + o = orphaned() + if o: + sys.stderr.write(f"axiomcode mcp: {o}. This server stops.\n"); sys.stderr.flush(); os._exit(0) for repo in list(SEEN): try: fresh = scripts_module('ax_fresh') @@ -183,6 +199,9 @@ def run(args, cwd=None, timeout=900): env = dict(os.environ, AXIOMCODE_BUILD_NOWAIT='1', AXIOMCODE_SURFACE='mcp') if args and args[0] != 'index' else None root = plugin_root() ax = os.path.join(root, 'skills', 'axiomcode', 'scripts', 'axiomcode') + # an orphaned older install answers with the restart and runs nothing: its verbs would write into .axiomcode + o = orphaned(root) + if o: return f"axiomcode: {o}." note = stale_note(root) head = (note + '\n') if note else '' try: @@ -272,6 +291,9 @@ def tests() -> str: if __name__ == '__main__': # catch up on whatever changed while no session was running (#1305): started, never waited on + o = orphaned() + if o: + sys.stderr.write(f"axiomcode mcp: {o}. Not starting.\n"); sys.exit(1) try: scripts_module('ax_fresh').hook_kick(os.getcwd(), 'the MCP server starting') if os.path.isdir(os.path.join(os.getcwd(), '.axiomcode')): SEEN.add(os.path.realpath(os.getcwd())) diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index a4f480f51..e01093337 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -102,7 +102,8 @@ once on `index` and kept by every rebuild after it: axiomcode index --library .venv/lib/python3.12/site-packages/requests,/deps/ir/jdk # or name them `auto` finds what the project depends on, in every language: a Python package its source imports, in its virtual -environment (`.venv`, `venv`, `$VIRTUAL_ENV`); a JavaScript / TypeScript package it imports, under `node_modules`; a +environment (`.venv`, `venv`, `$VIRTUAL_ENV`); a JavaScript / TypeScript package it imports, under `node_modules`; in a +Deno project, every `jsr:`, `npm:` and URL module it imports, from Deno's cache (`deno install` fetches them); a Java dependency `pom.xml` or `build.gradle` declares, through its `-sources.jar` in the Maven repository or Gradle's cache, or its class jar decompiled when it ships no sources (needs Vineflower: `mvn dependency:get -Dartifact=org.vineflower:vineflower:1.10.1`); a NuGet package a `.csproj` references, decompiled from its assembly diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py index 5aa67cc24..98764b89a 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py @@ -112,7 +112,7 @@ # C#: a call the language writes for a statement: `using` -> Dispose, `foreach` -> GetEnumerator, `{ a, b }` -> Add 'using_dispose': 'call', 'foreach_enumerator': 'call', 'collection_add': 'call', # the callable is named, not called at that line — it runs when whoever took it runs it - 'ref': 'method-ref', + 'ref': 'method-ref', 'METHOD_REFERENCE': 'method-ref', # a declaration handed to a decorator, which is what wires most framework handlers up 'DECORATOR_APPLICATION': 'decorator', 'DECORATOR_ATTRIBUTE': 'decorator', 'DECORATOR_BARE': 'decorator', 'DECORATOR_CALL': 'decorator', diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py index e1fc01047..2e09be9e8 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py @@ -71,7 +71,9 @@ 'web': WEB_EXT} NAMES = { 'java': ('lombok.config',), - 'typescript': ('package.json',), + # a Deno project's config and lockfile decide what its imports resolve to (its import map, workspace members and + # pinned jsr: / npm: versions), so an edit to either changes the graph as an edit to a source file does + 'typescript': ('package.json', 'deno.json', 'deno.jsonc', 'deno.lock'), 'javascript': ('package.json',), 'python': ('pyproject.toml', 'setup.cfg', 'setup.py'), 'csharp': ('global.json', 'Directory.Build.props'), @@ -258,7 +260,7 @@ def snapshot(repo, lang, src): DEP_FILES = {'requirements.txt', 'requirements-dev.txt', 'requirements-test.txt', 'pyproject.toml', 'setup.cfg', 'setup.py', 'Pipfile', 'Pipfile.lock', 'poetry.lock', 'uv.lock', 'package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', 'pom.xml', 'build.gradle', 'build.gradle.kts', 'gradle.lockfile', 'Directory.Packages.props', - 'packages.lock.json', 'project.assets.json', '.package-lock.json'} + 'packages.lock.json', 'project.assets.json', '.package-lock.json', 'deno.json', 'deno.jsonc', 'deno.lock'} DEP_PRUNE = {'.git', '.axiomcode', '__pycache__', '.venv', 'venv', 'env', '.env', 'dist', 'build', 'target', 'bin', '.gradle'} def dep_snapshot(repo): @@ -652,10 +654,14 @@ def newer_build(repo, t=None): IMPACT_VERSION is not re-exported either (rewarm): that would record the older version over the newer one's""" return built_by_state(repo, t)[1] +TWO_VERSIONS = ("two axiomcode versions are running over this repository (an agent session started before an upgrade keeps " + "the older version's hooks and MCP server); restart the agent session after upgrading the engine or the plugin") + def newer_note(n): - """what an answer from a graph a newer axiomcode built says""" - return (f"graph refresh: {n}; answering from it as is, and this older axiomcode does not rebuild it; rows in files " - "edited since are marked. Update this axiomcode, or `AXIOMCODE_REINDEX=1 axiomcode index` rebuilds it with this one") + """what an answer from a graph a newer axiomcode built says: most often the older of TWO VERSIONS RUNNING at once (an + agent session that outlived an upgrade), which a restart ends, not a stale graph""" + return (f"graph refresh: {n}: {TWO_VERSIONS}. This older axiomcode answers from that graph as it is and does not rebuild " + "it; rows in files edited since are marked. Update it, or `AXIOMCODE_REINDEX=1 axiomcode index` rebuilds the graph with this one") def engine_change(repo, t=None): """'' when the graph was built by the axiomcode that would build it now, else what differs, written @@ -676,6 +682,54 @@ def export_behind(t): if _vnum(have) and _vnum(now) and _vnum(have) > _vnum(now): return '' return have +# ── TWO VERSIONS TAKING TURNS: REBUILT ONCE, NOT ON EVERY TURN ───────────────────────────────────────────────────────── +# NEVER A DOWNGRADE (newer_build) holds only when the older axiomcode knows it: one from before it (or one whose version +# numbers tie with this one's) rebuilds the graph with its own engine, this one rebuilt it back, and every answer between +# came from whichever had won last (measured: IMPACT_VERSION 41 -> 76 and back, a full rebuild on each turn). So a build +# that replaces a graph another axiomcode built records that graph's maker (displace); when a graph by that same maker +# turns up again AFTER that, both are still running, and this one does not rebuild it again in the background: it says +# that two versions are running and that a restart of the agent session ends it (contested). `axiomcode index` still +# rebuilds it: asked for by name, it is not a turn. +DISPLACED_KEEP = 8 + +def build_key(b): + """a short identity of what built a graph (its file table's built_by): every axiomcode that writes it differently differs""" + if not isinstance(b, dict): return 'unrecorded' + keep = {k: b.get(k) for k in ('impact', 'rules', 'index', 'engine_version', 'engine_hash')} + return hashlib.sha1(json.dumps(keep, sort_keys=True).encode()).hexdigest()[:16] + +def displace(repo, t): + """a build of this axiomcode's is about to replace the graph table t describes, which another axiomcode built""" + if not t: return + d = dict(read_state(repo).get('displaced') or {}) + d[build_key(t.get('built_by'))] = time.time() + write_state(repo, displaced=dict(sorted(d.items(), key=lambda kv: kv[1])[-DISPLACED_KEEP:])) + +def _maker(b): + if not isinstance(b, dict): return 'an axiomcode that did not record its version' + return f"axiomcode {b.get('engine_version') or '?'} (IMPACT_VERSION {b.get('impact') or '?'})" + +def contested(repo, t=None, older=None): + """'' unless the graph (an older axiomcode's: `older` is engine_change's line) was built by an axiomcode this one had + already rebuilt it over, after that rebuild: then one line naming both, that two versions are running, and that this + one does not rebuild it again in the background""" + t = t if t is not None else load_table(repo) + older = engine_change(repo, t) if older is None else older + if not t or not older: return '' + when = (read_state(repo).get('displaced') or {}).get(build_key(t.get('built_by'))) + built = t.get('built') + if built is None: + try: built = os.path.getmtime(table_path(repo)) + except OSError: built = 0 + if not when or float(built) <= float(when): return '' + eng = current_engine(repo); mine = f"IMPACT_VERSION {plugin_id()[1]}" + if eng: + try: mine = f"{json.load(open(os.path.join(eng, 'package.json'))).get('version', '?')} ({mine})" + except (OSError, ValueError): pass + return (f"this graph was rebuilt by {_maker(t.get('built_by'))} after this axiomcode {mine} had rebuilt it: {TWO_VERSIONS}. " + "This one does not rebuild it again in the background, so the two stop rebuilding it in turn; once only one runs, " + "`axiomcode index` rebuilds it once") + def _seen_hash(repo, e): """the content hash of engine e: an engine whose files moved (a reinstall, a new checkout) but whose bytes did not is hashed once, and its hash kept beside the graph for its stat signature, so later queries only stat it""" @@ -695,12 +749,16 @@ def read_state(repo): except (OSError, ValueError): return {} def write_state(repo, **kv): + if orphaned(): return # an orphaned older install writes nothing (orphaned) st = read_state(repo); st.update(kv) tmp = state_path(repo) + f'.{os.getpid()}' try: json.dump(st, open(tmp, 'w')); os.replace(tmp, state_path(repo)) except OSError: pass +# ── AN ORPHANED INSTALL WRITES NOTHING: ax_orphan.orphaned, the one check every path that writes unasked makes ────────── +from ax_orphan import ORPHAN_MARK, install_dir, orphaned + def has_graph(repo): """a graph was built here: graph.sqlite exists, or is a pointer whose target is gone. A broken pointer is a graph to REPAIR, not a repository without one: taken for the latter, the refresher stopped for good and the next query @@ -717,7 +775,7 @@ def relink(repo, locked=False, say=True): interrupted". A pointer that leaves this .axiomcode/out is re-pointed at the same graph here (the copy's own), and says which case it was; one with no graph here to take is removed, so the query builds one. Returns what it did.""" out = out_dir(repo); p = os.path.join(out, 'graph.sqlite') - if not os.path.islink(p) or (not locked and building(repo)): return '' # a running build owns the pointer + if not os.path.islink(p) or (not locked and building(repo)) or orphaned(): return '' # a running build owns the pointer try: t = os.readlink(p) except OSError: return '' if not os.path.isabs(t): return '' # relative: this out/'s own; dangling, it is an interrupted build to repair @@ -963,6 +1021,8 @@ def status(repo): return dict(state='stale', changed=[unbuilt_row(repo)], added=[], removed=[], **nw) if unbuilt(repo) else dict(state='fresh', **nw) d = dict(state='building' if busy else 'stale', changed=changed, added=added, removed=removed, **nw) if eng: d['engine'] = eng + two = contested(repo, t, eng) if eng else '' + if two: d['contested'] = two if busy and not (changed or added or removed): d.update(pending(repo)) if not busy and failed_on(st, c, eng): d['failed'] = st.get('failed_log', ''); d['failed_reason'] = st.get('failed_reason', '') return d @@ -983,7 +1043,7 @@ def behind(s): return bool(edited(s) or s.get('engine')) def enabled(repo): - return not os.environ.get('AXIOMCODE_NO_REFRESH') and not os.environ.get('AXIOMCODE_GRAPH') and has_graph(repo) + return not os.environ.get('AXIOMCODE_NO_REFRESH') and not os.environ.get('AXIOMCODE_GRAPH') and has_graph(repo) and not orphaned() def refresh_off(repo): """the refresher is switched off (AXIOMCODE_NO_REFRESH) over a graph of this repository's own: nothing rebuilds it, @@ -1071,7 +1131,10 @@ def hook_kick(repo, trigger, session='', event='', tool=''): """what a hook (and the MCP server's timer) runs in place of kick: '' after starting the refresher as kick does, or, for a graph another axiomcode built (engine_change), nothing started and one line to say so, once per session and difference ('' when it was said already). A graph a newer axiomcode built is never rebuilt anyway (newer_build). - `event` and `tool` name the hook event and the tool it followed, for batching edits (batch_held)""" + `event` and `tool` name the hook event and the tool it followed, for batching edits (batch_held). An orphaned older + install (orphaned) starts and writes nothing, and says so at a session's start and on a prompt""" + o = orphaned() + if o: return f"graph refresh: {o}" if event in ('SessionStart', 'UserPromptSubmit') and has_graph(repo) else '' if not enabled(repo): return '' held = engine_change(repo) if not held: @@ -1080,6 +1143,8 @@ def hook_kick(repo, trigger, session='', event='', tool=''): key = f"{session}|{held}" if read_state(repo).get('hook_held') == key: return '' write_state(repo, hook_held=key) + two = contested(repo, None, held) + if two: return f"graph refresh: {two}" return (f"graph refresh: {held}; the hooks do not rebuild a graph another axiomcode built, so it is kept as it is. " f"`axiomcode index` rebuilds it with engine {engine_label(repo)}, and so does a query, which says so on its first " f"line; {NO_REFRESH_HOW}") @@ -1189,6 +1254,7 @@ def rewarm(repo, t): except OSError: pass def worker(repo): + if orphaned(): return 0 # an orphaned older install builds nothing (orphaned) lock = os.path.join(repo, '.axiomcode', 'refresh.lock') fd = os.open(lock, os.O_RDWR | os.O_CREAT, 0o644) if not _flock(fd, False): return 0 # single flight @@ -1219,6 +1285,12 @@ def worker(repo): # BUILT BY ANOTHER AXIOMCODE: rebuilt with no file changed, once per worker. A rebuild that leaves the same # difference behind (a build that could not record what built it) is not repeated eng = engine_change(repo, t) + two = contested(repo, t, eng) if eng else '' + if two: + # TWO VERSIONS TAKING TURNS: this one rebuilt it once already, and the other rebuilt it back + write_state(repo, state='contested', checked=time.time(), contested=two, checked_by=os.environ.get('AXIOMCODE_REFRESH_TRIGGER', '')) + print(f"{time.strftime('%H:%M:%S')} refresh: {two}", flush=True) + return 0 if eng and eng == engine_done: eng = '' if (c is None or not any(c)) and not eng and not base_moved(repo): if export_behind(t): rewarm(repo, t) @@ -1232,6 +1304,7 @@ def worker(repo): why = (f"{n} file(s) changed" if n else eng if eng else f"HEAD moved to {(head(repo) or '')[:10]}") + \ (f"; {eng}" if n and eng else '') + f", found by {os.environ.get('AXIOMCODE_REFRESH_TRIGGER') or 'an edit'}" engine_done = eng + if eng: displace(repo, t) env = rebuild_env(t, AXIOMCODE_BACKGROUND='1', AXIOMCODE_REFRESH_REASON=why) slot = take_slot(repo) # queued behind the machine's other background builds t0 = time.time(); write_state(repo, state='building', started=t0, files=sum(len(x) for x in c), edits=0) @@ -1281,7 +1354,7 @@ def wait(repo, seconds): if s['state'] == 'unknown': kick(repo, 'a query'); return s # a graph from before the file table: its first refresh records one # a FIRST build's other languages are not waited for: there is no graph of theirs to refresh, only a compile # that can take an hour (#1555). A rebuild's are, for the bounded while any refresh is. - if s['state'] in ('fresh', 'no graph') or s.get('failed') or s.get('first') or s.get('newer'): return s + if s['state'] in ('fresh', 'no graph') or s.get('failed') or s.get('first') or s.get('newer') or s.get('contested'): return s if s['state'] == 'stale': kick(repo, 'a query') # idempotent: a no-op while a worker holds its lock if time.time() >= end: return s time.sleep(0.5) @@ -1292,6 +1365,8 @@ def wait_baseline(repo, seconds, hook=False): hook: asked by a hook, which never rebuilds a graph another axiomcode built (hook_kick)""" if not enabled(repo) or not base_moved(repo): return '' if hook and engine_change(repo): return '' + two = contested(repo) + if two: return f"graph refresh: HEAD moved since the baseline was set ({head(repo)[:10]}), and {two}" if newer_build(repo): return (f"graph refresh: HEAD moved since the baseline was set ({head(repo)[:10]}), and the graph was built by a newer " "axiomcode, which this one does not rebuild: the baseline stays where it was, so this answer also counts what the " @@ -1350,6 +1425,9 @@ def note(s, marked=None, named=None, off=False): if s.get('state') not in ('stale', 'building'): return first if s.get('engine'): # BUILT BY ANOTHER AXIOMCODE: every row may differ from what this version answers, so none is marked; the line says so + if s.get('contested'): + return (first + '\n' if first else '') + f"graph refresh: {s['contested']}; this answer is from that graph, " \ + "and may lack what this version finds" + (f"; it predates edits to {', '.join(edited(s)[:5])}" if edited(s) else '') if off: says = "refresh is OFF (--no-refresh, or AXIOMCODE_NO_REFRESH is set), so nothing rebuilds it; `axiomcode index` does" elif s.get('failed'): says = "the rebuild FAILED" + (f" — {s['failed_reason']}" if s.get('failed_reason') else '') + f" (see {s['failed']})" else: says = "rebuilding in the background; --fresh waits for the rebuild" @@ -1536,7 +1614,7 @@ def wait_fresh(repo, seconds, say=False, files=()): while True: s = status(repo) if (s['state'] in ('fresh', 'no graph', 'unknown') or not behind(s)) and not _waits_on(s, files): return True - if s.get('failed') or s.get('newer'): return False # nothing here rebuilds a newer axiomcode's graph + if s.get('failed') or s.get('newer') or s.get('contested'): return False # nothing here rebuilds a newer axiomcode's graph if s['state'] == 'stale': kick(repo, 'a query') now = time.time() if now >= end: return False @@ -1573,6 +1651,11 @@ def with_note(n): # the answer as it is, then # REFRESH SWITCHED OFF IS NOT "UP TO DATE". With AXIOMCODE_NO_REFRESH nothing rebuilds the graph, which is exactly when # an answer from it most needs to say which edits it predates: a "nothing named X" for an X an edit just added read # as X not existing. It is checked, marked and noted as below; it only never kicks a refresh or waits for one. + o = orphaned() + if o: + # AN ORPHANED OLDER INSTALL answers nothing of its own: its verbs would write their caches, or a first build, into + # .axiomcode with the older version's rules (orphaned) + print(f"graph refresh: {o}.", flush=True); return 0 off = not enabled(repo) and refresh_off(repo) if not enabled(repo) and not off: # the refresh switched off still leaves a build that is solving the repository's other languages (#1555); a @@ -1592,7 +1675,7 @@ def with_note(n): # the answer as it is, then if not note(s): passthrough() return with_note(note(s)) # a graph a newer axiomcode built is never rebuilt here: no refresh is started or waited for, as with refresh off - hold = off or bool(s.get('newer')) + hold = off or bool(s.get('newer')) or bool(s.get('contested')) started = not s.get('failed') and not hold and kick(repo, 'a query') # a refresher already running (a hook's, the timer's, another query's) replaces this graph all the same: said too running = not started and not s.get('failed') and not hold and refresher_running(repo) @@ -1631,6 +1714,7 @@ def with_note(n): # the answer as it is, then obj['freshness'] = dict(state='off' if off else s['state'], edited=edited(s), rows_marked=n, **({'built_by': s['engine']} if s.get('engine') else {}), **({'newer': s['newer']} if s.get('newer') else {}), + **({'contested': s['contested']} if s.get('contested') else {}), **({'built_with': s['built_with']} if s.get('built_with') else {}), **({'failed': s['failed']} if s.get('failed') else {}), **({'named_in_edits': [dict(name=a, file=b) for a, b in missed]} if missed else {})) @@ -1698,7 +1782,7 @@ def main(argv): # the files are the graph's; the axiomcode that built it must be the one building now (axiomcode-build passes # the engine it found as AXIOMCODE_ENGINE), or the graph is rebuilt, saying why eng = engine_change(repo, t) - if eng: print(f"{eng}; rebuilding", flush=True); return 1 + if eng: displace(repo, t); print(f"{eng}; rebuilding", flush=True); return 1 return 0 if cmd == 'status': s = status(repo) @@ -1721,7 +1805,7 @@ def main(argv): # what axiomcode-build asks before it rebuilds: exit 0, saying why, when the graph here was built by a newer # axiomcode, which this one must not rebuild (newer_build); 1 otherwise n = newer_build(repo) - if n: print(f"{n}: kept, not rebuilt by this older axiomcode; update it, or AXIOMCODE_REINDEX=1 rebuilds the graph with this one") + if n: print(f"{n}: kept, not rebuilt by this older axiomcode; {TWO_VERSIONS}. Update it, or AXIOMCODE_REINDEX=1 rebuilds the graph with this one") return 0 if n else 1 if cmd == 'baseline': n = wait_baseline(repo, float(argv[3]) if len(argv) > 3 else 30) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py index 622c61c88..8f01b85bf 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py @@ -70,7 +70,7 @@ def python_sites(repo): return [] -def discover(repo): +def discover(repo, engine=None): found, own = [], {d for d in os.listdir(repo) if os.path.isdir(os.path.join(repo, d))} src = os.path.join(repo, 'src') if os.path.isdir(src): @@ -105,6 +105,7 @@ def discover(repo): if os.path.isdir(d) and not os.path.islink(d): found.append(d) note(f'auto: {len(found) - before} JavaScript / TypeScript package(s) the project imports, from node_modules') + found += deno_libraries(repo, engine) found += java_sources(repo) found += csharp_sources(repo) if not found: @@ -113,6 +114,40 @@ def discover(repo): return found +def has_deno_config(repo): + for root, dirs, files in os.walk(repo): + dirs[:] = [x for x in dirs if x not in SKIP and not x.startswith('.') and x != 'vendor'] + if 'deno.json' in files or 'deno.jsonc' in files: + return True + return False + + +def deno_libraries(repo, engine): + """A Deno project's dependencies: its mirror of the jsr: and URL modules it imports (copied from Deno's cache, + laid out as vendor/ is) and every npm: package directory, as the parser's own resolver finds them.""" + if not engine or not has_deno_config(repo): + return [] + cli = os.path.join(engine, 'parser', 'dist', 'deno-libraries.js') + if not os.path.isfile(cli): + note(f'auto: a Deno project, but {cli} is missing (parser not built); its jsr:, npm: and URL imports are not staged') + return [] + try: + out = subprocess.run(['node', cli, repo], capture_output=True, text=True, timeout=600) + roots = json.loads(out.stdout.strip().splitlines()[-1]) if out.returncode == 0 and out.stdout.strip() else None + except (OSError, ValueError, subprocess.TimeoutExpired): + roots = None + if roots is None: + note('auto: Deno dependency discovery failed; its jsr:, npm: and URL imports are not staged') + return [] + found = [d for d in roots.get('mirrors', []) + roots.get('packages', []) if os.path.isdir(d)] + note(f"auto: {len(roots.get('mirrors', []))} Deno mirror(s) of jsr: and URL modules, " + f"{len(roots.get('packages', []))} npm package(s) the Deno project imports") + if roots.get('unresolved'): + note(f"auto: {roots['unresolved']} Deno dependency import(s) resolve to nothing; if Deno has not downloaded " + f"them (or its cache was cleared), run `deno install` and index again") + return found + + def maven_repo(repo): cfg = os.path.join(repo, '.mvn', 'maven.config') if os.path.isfile(cfg): @@ -318,7 +353,7 @@ def main(): cache = CACHE out, seen = [], set() for e in [x for x in spec.split(',') if x]: - entries = discover(repo) if e == 'auto' else [e if os.path.isabs(e) else os.path.join(repo, e)] + entries = discover(repo, engine) if e == 'auto' else [e if os.path.isabs(e) else os.path.join(repo, e)] for d in entries: if not os.path.isdir(d): note(f'not a directory: {d}; not staged'); continue diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_orphan.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_orphan.py new file mode 100644 index 000000000..b9148d809 --- /dev/null +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_orphan.py @@ -0,0 +1,46 @@ +"""ax_orphan.py: is the axiomcode running here an orphaned older install? One check, for every long-running path. + +An agent session started before an upgrade keeps running the OLD version's MCP server and hooks. A plugin manager that +keeps one directory per version marks the one it replaced with `.orphaned_at` (and deletes its files later); a process +still running out of it is an older axiomcode, and what it wrote into a repository's .axiomcode was the older version's: +it rebuilt the graph back to its own engine after the new one had rebuilt it, every rebuild undoing the last, and it +recreated .axiomcode (its refresh.log, refresh.json) while the user was deleting it, so `rm -rf .axiomcode` failed with +"Directory not empty". So every path that writes there without being asked to (the refresher, the hooks, the MCP server +and its timer, a query's background build) asks orphaned() first, and an orphaned install writes nothing and starts +nothing: the MCP server stops, a hook and a query say so instead. + +Its own module, with nothing but os and json, because every hook asks it before anything else: importing ax_fresh for it +cost a hook 20-100 ms on every tool call.""" +import json, os + +H = os.path.dirname(os.path.abspath(__file__)) +ORPHAN_MARK = '.orphaned_at' + +def install_dir(scripts=None): + """the plugin directory a scripts directory belongs to (/skills/axiomcode/scripts), these scripts' by default""" + return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(scripts or H)))) + +def _install_version(root): + try: return str(json.load(open(os.path.join(root, '.claude-plugin', 'plugin.json'), encoding='utf-8')).get('version') or '') + except (OSError, ValueError, AttributeError): pass + b = os.path.basename(os.path.normpath(root)) # a versioned plugin cache: /// + return b if all(x.isdigit() for x in b.split('.')) else '' + +def orphaned(root=None): + """'' while the install is current, else one line saying this is an orphaned older axiomcode and the agent session + is to be restarted. root: the plugin directory to judge (the MCP server passes the one it resolved); by default the + one these scripts belong to. Orphaned: the plugin directory carries ORPHAN_MARK, or its scripts are gone""" + if root: + root = os.path.abspath(root) + gone = not os.path.isfile(os.path.join(root, 'skills', 'axiomcode', 'scripts', 'ax_fresh.py')) + else: + # these scripts' own file is what is checked for "gone": an engine that lays them out differently is never orphaned + root = install_dir(); gone = not os.path.isfile(os.path.join(H, 'ax_fresh.py')) + if os.path.lexists(os.path.join(root, ORPHAN_MARK)): + why = f"{root} is marked {ORPHAN_MARK}: a newer version was installed in its place" + elif gone: why = f"its install directory {root} is gone: a newer version was installed in its place" + else: return '' + v = _install_version(root) + return (f"this is an orphaned older axiomcode{' ' + v if v else ''} ({why}), still running in an agent session started " + "before the upgrade; it writes nothing into the repository's .axiomcode and rebuilds nothing. Restart the agent " + "session so that only the installed axiomcode runs") diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py index 0167e2c42..299232f86 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py @@ -49,7 +49,7 @@ MAX_SITE_SPAN = 200 # a "call" spanning a whole file is a parse artefact, not a registration -def registrations(q, site_file=None): +def registrations(q, site_file=None, with_own_lines=False): """[(file, line, kind, key, why)] — every line that hands a declaration to something outside the graph. `kind` is always "route"; `key` is the string the framework dispatches on, VERBATIM and never normalised @@ -57,9 +57,11 @@ def registrations(q, site_file=None): client (`supertest.get('/orders/1')`, `TestClient(app).post('/orders')`) be joined to the handler it reaches: the two ends spell the same string and nothing else connects them. Normalising it belongs in the join, not here — `:id`, `{order_id}` and `` are three frameworks' spellings of one hole. + + `with_own_lines`: (rows, route_site_lines(q, site_file)) — the same walk computes both. """ if not _has(q, 'call_sites'): - return [] + return ([], {}) if with_own_lines else [] sf = site_file or (lambda x: x) lits = {} if _has(q, 'literals'): @@ -83,7 +85,7 @@ def registrations(q, site_file=None): # the cap and joins two unrelated tests. The DEPENDENT row is still true and still printed — a test that mounts a # handler does depend on it — so only the key is withheld. tf = {f for (f,) in q("SELECT DISTINCT file FROM symbols WHERE is_test = 1 AND file IS NOT NULL")} if _has(q, 'symbols') else set() - out, verbs, _own = _route_links(q, sf, lits, tf) + out, verbs, own = _route_links(q, sf, lits, tf) # `own` does not depend on tf, which only withholds a key # `router.route('/').get(h)` is two route-shaped calls on one line, and `route` said nothing about the method: the # line's HTTP verbs name it, in the order written. A chain registering two (`.get(a).post(b)`) names both, because # a line is all a reference records and it cannot say which argument list the handler sat in @@ -92,7 +94,8 @@ def registrations(q, site_file=None): if k == 'route' and vs: m = re.match(r'registered as a \w+ route (".*?") here', w) if m: out[(f, l)] = (k, key, f"registered as a {'/'.join(sorted(vs, key=vs.get))} route {m.group(1)} here — the router calls it, no call site does") - return sorted((ref_at.get((f, l), ''), f, l, k, key, w) for (f, l), (k, key, w) in out.items()) + rows = sorted((ref_at.get((f, l), ''), f, l, k, key, w) for (f, l), (k, key, w) in out.items()) + return (rows, own) if with_own_lines else rows def route_site_lines(q, site_file=None): @@ -289,6 +292,7 @@ class _Handlers: def __init__(self, q, read, sf): self.q, self.read, self.sf, self.names, self.consts, self.held = q, read, sf, {}, {}, {} + self.tables = None # const_value_tables, read on the first const asked about if _has(q, 'symbols'): for n, k, f, l in q("SELECT name, kind, file, line FROM symbols WHERE name IS NOT NULL"): self.names.setdefault(n, {}).setdefault(k or '', set()).add(f) @@ -310,7 +314,8 @@ def holds_fn(self, n, files=None): # a module a const requires may export the handler itself: `const h = require('./h'); app.get('/x', h)` required = any(re.search(rf'(? 0"""): callee_at.setdefault((sf(fp), l, col), set()).add(m) + return fns, rets, callee_at + + +def const_values(q, code, decls, site_file=None, tables=None): + """For each (file, line, name) const declaration: ({(file, line)} that hold a function, {(file, line, wrapper)} the + client callable its initializer calls, {(file, line, name)} the function it is an alias of). `tables` is + const_value_tables(q, site_file), when the caller already has it.""" + callable_, wrap, alias = set(), set(), set() + if not decls or code is None: + return callable_, wrap, alias + fns, rets, callee_at = tables if tables is not None else const_value_tables(q, site_file) def call_value(text, i, f, a, starts, depth=0): """(callable?, wrappers) for the expression at text[i:]""" diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build index 7a52d595c..28ec84d31 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build @@ -316,14 +316,16 @@ if [ -f "$OUT/corrupt" ]; then echo "rebuilding: $(head -1 "$OUT/corrupt")"; fi if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && NEWER="$(AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" newer "$REPO")"; then echo "$NEWER"; exit 0 fi -if [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then +# AXIOMCODE_REINDEX=1 IS A REBUILD, as every message that names it promises ("rebuilds it with this one", "to force"): +# it skips both up-to-date answers below, which otherwise said "graph up to date" whenever the files matched the table. +if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then if [ "$(cat "$OUT/stamp")" != "$STAMP" ]; then echo "$STAMP" > "$OUT/stamp"; INDEXED_TREE="$OLD_INDEXED"; set_base "$OLD_INDEXED"; PREV=""; keep_base_graph; commit_base; fi set_base_files has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO" [ -f "$REPO/.axiomcode/engine" ] || printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine" || true echo "graph up to date (${LANGS//,/, }) at $OUT/graph.sqlite"; exit 0 fi -if [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then +if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO" python3 "$H/axiomcode-impact" --warm "$REPO" || true # export the graph's facts now (~9 s on 1,373 files): the first # query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once. diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index 2c6b86689..83b0a0f53 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -264,8 +264,17 @@ class Impact: triple = py or file.endswith(('.java', '.kt', '.scala', '.groovy')) # a Java text block is """ too js = file.endswith(('.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx')) prev = '' # the last code character seen + # only these characters can open a comment, a string or a regex literal: a run of any others is code, copied whole + special = self._CODE_SPECIAL_PY if py else self._CODE_SPECIAL while i < n: c = t[i] + if c not in '/"\'`#': + m = special.search(t, i); j = m.start() if m else n + out.append(t[i:j]) + k = j - 1 # `prev`: the run's last non-space, or its last newline + while k >= i and t[k].isspace() and t[k] != '\n': k -= 1 + if k >= i: prev = t[k] if not t[k].isspace() else '\n' + i = j; continue if js and c == '/' and i + 1 < n and t[i + 1] not in '/*' and prev in '(,=:[!&|?{};+-*%~^<>' + '\n': j = i + 1 # a regex literal: /…/flags, never a division while j < n and t[j] not in '/\n': @@ -292,8 +301,9 @@ class Impact: if not c.isspace(): prev = c elif c == '\n': prev = '\n' i += 1; continue - out.append(''.join('\n' if ch == '\n' else ' ' for ch in t[i:j])); prev = 'x'; i = j + out.append(self._BLANK.sub(' ', t[i:j])); prev = 'x'; i = j self.src[('code', file)] = ''.join(out).split('\n'); return self.src[('code', file)] + _CODE_SPECIAL = re.compile(r'[/"\'`]'); _CODE_SPECIAL_PY = re.compile(r'[/"\'`#]'); _BLANK = re.compile(r'[^\n]') # ── targets ────────────────────────────────────────────────────────────────────────────────────────────────── @@ -1843,13 +1853,18 @@ class Impact: # `Accessors = ...`): if the declaration's own source names a generator, it generates the same members gen_table = {d: set(ws) for d, ws in GENERATED.items()} used = {r[1].split('.')[-1] for r in g.q("SELECT owner_id, name FROM decorations")} if g.has('decorations') else set() - for d in used - set(gen_table): + # the first declaration of each such name with a file and a line, in the symbol table's order: found in one pass + # rather than one pass over every symbol per decorator name + wanted = used - set(gen_table); first_decl = {} + if wanted: for i, sy in g.sym.items(): - if sy['name'] != d or not sy.get('file') or not sy.get('line'): continue + if sy['name'] in wanted and sy['name'] not in first_decl and sy.get('file') and sy.get('line'): first_decl[sy['name']] = sy + for d in wanted: + sy = first_decl.get(d) + if sy is not None: body = '\n'.join(self.code(sy['file'])[sy['line'] - 1: (sy['end_line'] or sy['line']) + 1]) hit = {w for k, ws in GENERATED.items() if re.search(rf'\b{re.escape(k)}\b', body) for w in ws} if hit: gen_table[d] = hit - break # A BASE CLASS BUILT BY A FACTORY: `Base = declarative_base()`, then `class Customer(Base)`. The written # base name is `Base`, which discriminates nothing, and the alias rule cannot help because an assignment # is not a declaration with a body to read. What identifies it is the assignment itself, in the source the @@ -1863,6 +1878,7 @@ class Impact: pat = re.compile(rf'^\s*{re.escape(b)}\s*(?::[^=]+)?=\s*(\w+)\s*\(') for (fl,) in g.q("SELECT DISTINCT file FROM type_refs WHERE name = ? AND file IS NOT NULL", b): for ln in (self.code(fl) or []): + if b not in ln: continue # the pattern starts with the name itself m = pat.match(ln) if m and m.group(1) in GENERATED: gen_table[b] = set(GENERATED[m.group(1)]); break @@ -1877,9 +1893,10 @@ class Impact: W('cert_tier', sorted({(t, ax_edges.direct_cert(t), ax_edges.direct_why(t)) for t in {r[0] for r in g.q("SELECT DISTINCT tier FROM call_edges")} | {ax_edges.STUB_TIER, 'overload'}})) # a call written inside a mock's stub or verification keeps its row and loses its tier: "stub" (ax_edges.stub_sites) - stubs = ax_edges.stub_sites(lambda s, p: g.q(s, *p)) if g.has('call_sites') else set() + stubs = g.stub_sites() # a chained route link's edge sits on the line of its own verb and path, where `registration` labels it - link_line = ax_registration.route_site_lines(g.q, g.site_file) + # the registrations come out of the same walk over the route calls, so they are kept for `registration` below + regs, link_line = ax_registration.registrations(g.q, g.site_file, with_own_lines=True) # LEADS A LINK REJECTED (ax_links.py): a site's by-name match or set member someone read and found wrong is not walked; # the engine's row stays in graph.sqlite. `rejected` feeds the by-name rules, the set members leave `calls` here. import ax_links @@ -1978,7 +1995,7 @@ class Impact: L = self.code(r['file']); text = L[r['line'] - 1] if r['line'] <= len(L) else '' for qn in re.findall(rf'([A-Za-z_$][\w$]*)\s*\.\s*{re.escape(r["name"])}\b', text): quals.append((r['file'], r['line'], r['name'], qn)) W('ref', refs); W('qualifier', sorted(set(quals))) - regs = ax_registration.registrations(g.q, g.site_file) # computed once: reg_key_fact below reads it too + # `regs`, computed once above with link_line: reg_key_fact below reads it too W('registration', regs) trs = []; trf = set() # a reference on a module-level type alias's own lines belongs to the alias, not the module (#784) @@ -2339,7 +2356,19 @@ class Impact: # These are edges no call site carries — without them a change to a key or a bean has no visible consumer at all. cfg = []; keys = set(); beans = []; inj = []; factories = [] if g.has('ext_config_affects_method'): - for r in g.q("SELECT a.c0, a.c1, a.c2 FROM ext_config_affects_method a JOIN symbols s ON s.method_id = a.c1"): cfg.append((ckey(r[0]), r[1], r[2] or 'bound')) + # the rows whose method the index names, once per symbol row that names it: the join + # `ext_config_affects_method a JOIN symbols s ON s.method_id = a.c1`, in a's order, without probing the + # symbols index once per row (millions of rows over a few keys) + per_method = collections.Counter(m for (m,) in g.q("SELECT method_id FROM symbols WHERE method_id IS NOT NULL")) + ck = {} + for r in g.q("SELECT c0, c1, c2 FROM ext_config_affects_method"): + n = per_method.get(r[1], 0) + if not n: continue + k = ck.get(r[0]) + if k is None: k = ck[r[0]] = ckey(r[0]) + row = (k, r[1], r[2] or 'bound') + cfg.append(row) + for _ in range(n - 1): cfg.append(row) keys |= {ckey(r[0]) for r in g.q("SELECT DISTINCT c0 FROM ext_config_affects_method")} for t in ('ext_config_binding', 'ext_config_key_ref', 'ext_config_unresolved'): if g.has(t): keys |= {ckey(r[0]) for r in g.q(f"SELECT DISTINCT c0 FROM {t}")} diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index index 2751c4430..4bac96bce 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index @@ -20,16 +20,17 @@ to --src). The language's IR is read through the ADAPTER table below — header positions — mirroring graph/bundle/languages.ts. When the IR is absent the index degrades: symbols come from methods/types alone, refs/literals/comments are empty, and index_meta says so. """ -import csv, os, re, sqlite3, sys, time, glob, collections, functools -csv.field_size_limit(10**9) -INDEX_VERSION = '9' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; +import os, re, sqlite3, sys, time, glob, collections, functools +INDEX_VERSION = '10' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; # v5: JavaScript fields owned by their class, computed-key members named by their key, anonymous class expressions named by their binding; # v6: TypeScript class-property arrows named after their field; # v7: data keys of module-level const objects declared, Object.freeze seen through; # v8: literals carry numbers and booleans with a kind column, and the name of the constant # a scalar initializes — a spec's numbers were the one thing the graph could not answer; # v9: the web graph (HTML + CSS) is indexed: pages, stylesheets, .classes, #ids, custom properties (var(--x)), - # @keyframes/@font-face/@layer/@container, selectors and inline scripts are symbols); the query + # @keyframes/@font-face/@layer/@container, selectors and inline scripts are symbols; + # v10: the IR's RFC 4180 quoting is decoded, so a literal, comment, decoration or name holding a `"` + # is stored as written (`a "b" c`, not `a ""b"" c`), and C# verbatim and raw strings are literals); the query # frontend re-indexes an older graph when the IR is still there REPO = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].startswith('-') else os.environ.get('AXIOMCODE_REPO') or '.') @@ -193,7 +194,7 @@ A = { # name elsewhere is a by-name reader whatever its qualifier says, so `_r.Limit` was listed under AuditOptions.Limit memberName=dict(role='edgeRole', value='MEMBER_NAME', parent='parentExpressionHash', id='csExpressionUniqueHash'), # the literal TYPE column is literalKind here, not literalType as in java/typescript/python - litKinds={'LITERAL'}, litType=('literalKind', {'STRING': 'string', 'INTEGER': 'number', 'REAL': 'number', 'BOOLEAN': 'bool'}), litValue='literalValue', litLinkRoles={'ROOT'}), + litKinds={'LITERAL'}, litType=('literalKind', {'STRING': 'string', 'VERBATIM_STRING': 'string', 'RAW_STRING': 'string', 'INTEGER': 'number', 'REAL': 'number', 'BOOLEAN': 'bool'}), litValue='literalValue', litLinkRoles={'ROOT'}), comments=dict(file='all-csharp-comments.csv', text='commentText', kind='commentKind', line='startLine', fileVia=('modules', 'csModuleLinkHash')), # type references carry no module link of their own. The owner is a type only for a base-list reference; for # `M()`, `new T()`, a parameter, a return or a field it is the expression, method, parameter or field, so @@ -273,13 +274,76 @@ def is_test_file(fp): def rows(name): p = os.path.join(IR, name) if not HAVE_IR or not os.path.exists(p): return iter(()) - f = open(p, newline='', encoding='utf-8', errors='replace') - return csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE) + return ir_rows(p) + +def ir_rows(p): + """The rows of one IR file as dicts keyed by its header, its fields DECODED the way the bundle's reader + (graph/bundle/csv.ts readRfc4180) decodes them: the parser writes RFC 4180 over a tab, so a field that STARTS with + a double quote is quoted, `""` inside it is one quote, and the closing quote ends it (what follows the close, up + to the tab, is kept as written); a quote anywhere else in a field is an ordinary character. The writer + (EntityUtils.escapeTsv) turns a newline, a CR and a tab into the two-character escapes BEFORE it quotes, and quotes + only for a `"`, so a quoted field it wrote never holds a tab or a line break and a row is one line. Reading + QUOTE_NONE stored `Order "A-1"` as `Order ""A-1""`, so `impact` said the string was in no literal (#1835). + The rest is csv.DictReader(f, delimiter='\\t') over open(p, newline=''), at about half its cost (the expressions + file is most of this script's time on a large repository): a line ends at \\n, \\r or \\r\\n, a blank line is + skipped, a short row has None for its missing columns and a long one keeps the rest as a list under the key None. + A line with no quote in it is split on tabs and nothing else: that is nearly every line, and a test for a quote + that OPENS a field cost more than the decoding it saved. A quoted field left open at the end of a line, which this + writer does not produce, takes the line break and the next line, as the bundle's reader does. + tests/index_reader.py holds this to both readers.""" + with open(p, newline='', encoding='utf-8', errors='replace') as f: + def fields(line): + # a line holding a quote: split on tabs, then decode each field that opens with one, rejoined across the tabs + # (and, if it is still open at the end of the line, the line breaks) it holds + out = []; parts = line.split('\t'); i = 0 + while True: + if i == len(parts): return out + s = parts[i]; i += 1 + if s[:1] != '"': out.append(s); continue + buf = []; k = 1 + while True: + j = s.find('"', k) + if j >= 0: + buf.append(s[k:j]) + if s[j + 1:j + 2] == '"': buf.append('"'); k = j + 2; continue + buf.append(s[j + 1:]); break # the closing quote; the rest of the field as written + buf.append(s[k:]); k = 0 + if i < len(parts): buf.append('\t'); s = parts[i]; i += 1; continue + buf.append(line_end[0]); nxt = f.readline() # still open at the end of the line: the break is text + if not nxt: break + body = nxt.rstrip('\r\n'); line_end[0] = nxt[len(body):] + parts = body.split('\t'); s = parts[0]; i = 1 + out.append(''.join(buf)) + def decoded(line, raw): + line_end[0] = raw[len(line):] + v = fields(line) + if v[-1][-1:] == '\r': v[-1] = v[-1][:-1] # as the bundle's reader drops a CR before the newline + return v + line_end = [''] + head = raw = f.readline() + if not head: return + if head[-1:] == '\n': head = head[:-1] + if head[-1:] == '\r': head = head[:-1] + keys = (head.split('\t') if '"' not in head else decoded(head, raw)) if head else [] + n = len(keys) + for raw in f: + line = raw + if line[-1:] == '\n': line = line[:-1] + if line[-1:] == '\r': line = line[:-1] + if not line: continue + v = line.split('\t') if '"' not in line else decoded(line, raw) + d = dict(zip(keys, v)) + if len(v) != n: + if len(v) > n: d[None] = v[n:] + else: + for k in keys[len(v):]: d[k] = None + yield d # an older augment step left `symbols` as a VIEW; a name is dropped as whichever it is +dropped = False # whether this run frees pages: it decides the VACUUM at the end for name in ('symbols', 'refs', 'literals', 'comments', 'nesting', 'type_refs', 'decorations', 'skipped', 'index_meta', 'paths', 'callers', 'callees', 'source', 'sites'): kind = c.execute("SELECT type FROM sqlite_master WHERE name=?", (name,)).fetchone() - if kind: c.execute(f"DROP {kind[0].upper()} IF EXISTS {name}") + if kind: c.execute(f"DROP {kind[0].upper()} IF EXISTS {name}"); dropped = True c.executescript(''' DROP TABLE IF EXISTS symbols; DROP TABLE IF EXISTS refs; DROP TABLE IF EXISTS literals; DROP TABLE IF EXISTS comments; DROP TABLE IF EXISTS nesting; DROP TABLE IF EXISTS type_refs; DROP TABLE IF EXISTS decorations; DROP TABLE IF EXISTS skipped; DROP TABLE IF EXISTS index_meta; DROP TABLE IF EXISTS paths; @@ -673,6 +737,20 @@ for name, od, v, k in data_keys: c.executemany("INSERT INTO symbols VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", sym) # ── references, literals, comments ─────────────────────────────────────────────────────────────── +def cs_string_text(v): + """The text of a C# string token, as ir_rows hands it over (the IR's RFC 4180 quoting already undone): `"x"` is x + with its escapes as written, as Java's and Python's are; a verbatim `@"x"` is x with a doubled quote made one, the + language's own escape; a raw string is x, closed by as many quotes as opened it (three or more: a regular string + cannot open with three, `""` is the whole empty string); a `$` is dropped. Before ir_rows decoded the IR, a C# + literal holding a quote was un-doubled here by hand; on decoded text that is a second decoding, which reads the + raw `\"\"\"\"a \"\" b\"\"\"\"` as `"a " b"`, and a verbatim string's doubled quote was never made one.""" + p = 0 + while p < len(v) and v[p] in '@$': p += 1 + t = v[p:] + q = len(t) - len(t.lstrip('"')) + if q >= 3 and len(t) >= 2 * q and t.endswith('"' * q): return t[q:-q] + if len(t) >= 2 and t[0] == '"' == t[-1]: return t[1:-1].replace('""', '"') if '@' in v[:p] else t[1:-1] + return v e = A['expr']; refs = []; lits = [] namecol = e['name'] if isinstance(e['name'], str) else dict(x.split(':') for x in e['name']) mname = e.get('memberName'); child_ek = {} @@ -701,11 +779,8 @@ for r in rows(e['file']): kind = e['litType'][1][r.get(e['litType'][0])] # 'string' | 'number' | 'bool' across every language v = (r.get(e['litValue']) or '') if kind == 'string': - # C# keeps the source token, and the IR writer RFC4180-quotes a field holding a double quote, so `"Fee"` arrived - # as `"""Fee"""` and was stored `""Fee""`: no key, route or member name written in a C# string ever matched - if LANG == 'csharp' and len(v) > 1 and v[0] == '"' == v[-1] and '""' in v: v = v[1:-1].replace('""', '"') - if LANG == 'csharp' and len(v) > 2 and v[0] in '@$' and v[-1] == '"': v = v.lstrip('@$') # @"verbatim", $"interpolated" - if len(v) >= 2 and v[0] in '\'"`' and v[-1] == v[0]: v = v[1:-1] # JavaScript keeps the quotes + if LANG == 'csharp': v = cs_string_text(v) # C# keeps the source token: "x", @"x", """x""" + elif len(v) >= 2 and v[0] in '\'"`' and v[-1] == v[0]: v = v[1:-1] # JavaScript keeps the quotes if v: lits.append((v[:200], file_of(r, e), int(r.get(e['line']) or 0), kind, r.get('edgeRole', ''))) # a scalar's NAME: `MAX_ITEMS = 5` — the one literal on a const/variable/field declaration's line IS its value, # so the row gets the symbol's display and "what is MAX_ITEMS" has an answer with a source location. Strictly: @@ -741,13 +816,6 @@ if tr: c.executemany("INSERT INTO type_refs VALUES (?,?,?,?,?)", ((r.get(tr['name'], '').split('.')[-1].split('<')[0], file_of(r, tr), int(r.get(tr['line']) or 0), r.get(tr['context'], ''), r.get(tr['ownerKind'], '')) for r in rows(tr['file']) if r.get(tr['name']))) except (KeyError, ValueError): pass # decorations: @Annotation(args) / @decorator(args) on a method or type — a route, a bean name, a validator, a marker -def unquoted(v): - """The IR is tab-separated, but the writer still RFC4180-quotes a field that contains a double quote and the - reader is QUOTE_NONE, so `@router.post("/orders")` arrives as `"@router.post(""/orders"")"`. Every decoration - whose argument is a double-quoted string is in that shape — which is exactly the decorations that carry a KEY - (a route path, a command name, a topic, a validated field), so everything downstream that reads a key out of - the text read the quoting instead: `@exporter("csv")` yielded `@exporter(` and `)` and never `csv`.""" - return v[1:-1].replace('""', '"') if len(v) > 1 and v[0] == '"' == v[-1] and '""' in v else v dec = A.get('decorations') if dec: args = collections.defaultdict(list) @@ -758,18 +826,29 @@ if dec: for r in rows(dec['file']): nm = r.get(dec['name'], '') if not nm: continue - text = unquoted(r.get(dec['text'], '')) if dec.get('text') else ('@' + nm + ('(' + ', '.join(args.get(r.get(dec['id'], ''), [])) + ')' if args.get(r.get(dec['id'], '')) else '')) + # the text as written (ir_rows undoes the writer's RFC 4180 quoting, so `@exporter("csv")` arrives as that) + text = r.get(dec['text'], '') if dec.get('text') else ('@' + nm + ('(' + ', '.join(args.get(r.get(dec['id'], ''), [])) + ')' if args.get(r.get(dec['id'], '')) else '')) try: drows.append((r.get(dec['owner'], ''), nm, text[:200], file_of(r, dec), int(r.get(dec['line']) or 0))) except ValueError: pass c.executemany("INSERT INTO decorations VALUES (?,?,?,?,?)", drows) c.executemany("INSERT INTO skipped VALUES (?,?)", ((rel(r.get('filePath') or r.get('file') or ''), r.get('reason') or r.get('skipReason') or '') for r in rows(A['skipped']))) # ── views: the joins written once ──────────────────────────────────────────────────────────────── +# Who calls a method and what it calls are looked up by call_edges(callee_method_id) and call_edges(caller_id); the +# fields a caller reads by field_access(caller_id): via_base_rows asks per single-target call site, and without that +# index each ask scanned the whole table (65,797 rows × 5,162 sites on an 8,619-file repository = 24 s of a 171 s warm +# → 1.4 s). The bundle indexes the same columns (idx__), but these are made anyway, and LAST: the graph +# has no ANALYZE statistics, so SQLite picks among equal-cost indexes by creation order, and with the bundle's alone +# `caller_id=? AND access=?` planned on idx_field_access_access and `caller_id=? AND ... AND tier=?` on +# idx_call_edges_tier. Skipping them made the impact warm 1.5-1.9x slower on large Java repositories (warm 79 s → 54 s on ~8k files). +c.executescript(''' +CREATE INDEX IF NOT EXISTS ce_callee ON call_edges(callee_method_id); CREATE INDEX IF NOT EXISTS ce_caller ON call_edges(caller_id); +CREATE INDEX IF NOT EXISTS fa_caller ON field_access(caller_id); +''') c.executescript(''' CREATE INDEX IF NOT EXISTS symbols_name ON symbols(name); CREATE INDEX IF NOT EXISTS symbols_display ON symbols(display); CREATE INDEX IF NOT EXISTS symbols_file ON symbols(file, line); CREATE INDEX IF NOT EXISTS symbols_mid ON symbols(method_id); CREATE INDEX IF NOT EXISTS symbols_tid ON symbols(type_id); CREATE INDEX IF NOT EXISTS refs_name ON refs(name); CREATE INDEX IF NOT EXISTS type_refs_name ON type_refs(name); CREATE INDEX IF NOT EXISTS decorations_owner ON decorations(owner_id); CREATE INDEX IF NOT EXISTS literals_value ON literals(value); CREATE INDEX IF NOT EXISTS comments_file ON comments(file); -CREATE INDEX IF NOT EXISTS ce_callee ON call_edges(callee_method_id); CREATE INDEX IF NOT EXISTS ce_caller ON call_edges(caller_id); -- lookups by id and by file + line: without them each per-site or per-test query scans the whole table CREATE INDEX IF NOT EXISTS symbols_id ON symbols(id); CREATE INDEX IF NOT EXISTS refs_file_line ON refs(file, line); CREATE INDEX IF NOT EXISTS literals_file_line ON literals(file, line); @@ -777,9 +856,6 @@ CREATE INDEX IF NOT EXISTS literals_file_line ON literals(file, line); -- the parts of a partial type (type_parts): without these each ask scanned type_refs / symbols, 1.2 s of a 1.8 s impact -- on a 1,300-file Java repository → 0.5-0.75 s, answers unchanged CREATE INDEX IF NOT EXISTS type_refs_file_line ON type_refs(file, line); CREATE INDEX IF NOT EXISTS symbols_qn ON symbols(qualified_name); --- the fields a caller reads: via_base_rows asks per single-target call site, and without the index each ask --- scanned the whole table (65,797 rows × 5,162 sites on an 8,619-file repository = 24 s of a 171 s warm → 1.4 s) -CREATE INDEX IF NOT EXISTS fa_caller ON field_access(caller_id); -- who calls a method: one row per (site, target), with the engine's tier. caller_id is symbols.id: a -- Java call written in a field initializer has the TYPE as its caller, and that row is kept, not dropped CREATE VIEW callers AS @@ -804,7 +880,12 @@ meta = dict(language=LANG, repo=REPO, source_dir=SRC, ir=IR if HAVE_IR else '', symbols=n('symbols'), refs=n('refs'), type_refs=n('type_refs'), decorations=n('decorations'), literals=n('literals'), comments=n('comments'), nested_types=n('nesting'), skipped_files=n('skipped')) c.executemany("INSERT INTO index_meta VALUES (?,?)", [(k, str(v)) for k, v in meta.items()]) con.commit() -con.execute("VACUUM") # the tables above are dropped and recreated on every run; without this the file only grows +# The tables above are dropped and recreated on every run, so a re-index leaves their old pages free and the file only +# grows without a VACUUM. A graph the bundle has just written has nothing to reclaim: no table of this script existed +# in it, and nothing freed a page (freelist_count). Rewriting it anyway cost 4.2 s of a 12 s index on a ~1,300-file +# Python repository, for the same rows. +if dropped or c.execute("PRAGMA freelist_count").fetchone()[0] > 0: + con.execute("VACUUM") con.close() # THE ASSERTED LINKS (ax_links.py, axiomcode-links.tsv) are applied to every graph as it is indexed, after the engine's # export and before the graph is published, so a rebuild keeps them and no query ever sees a graph without them: each is diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path index 78e76b363..63adb2970 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path @@ -293,6 +293,13 @@ class G: self.con.executemany("INSERT INTO decorations VALUES (?,?,?,?,?)", out) self.decorations_synthesised = len(out) def q(self, s, *p): return self.con.execute(s, p).fetchall() + def stub_sites(self): + """ax_edges.stub_sites of this graph, read once per connection: both exports (path's and impact's) ask for it. + The set is shared — a caller must not change it.""" + if getattr(self, '_stubs_con', None) is not self.con: + self._stubs = ax_edges.stub_sites(lambda s, p: self.q(s, *p)) if self.has('call_sites') else set() + self._stubs_con = self.con + return self._stubs def has(self, t): return bool(self.q("SELECT 1 FROM sqlite_master WHERE name=?", t)) def loc(self, i): r = self.sym.get(i) or self.all_sym.get(i) @@ -1054,7 +1061,7 @@ class G: # …except a call written inside a mocking library's stub or verification (`when(repo.find(1))`, # `mock.Setup(r => r.Find(1))`): the receiver is a mock, so the real body never runs there, and walked as a call # it made every test that stubs a method a `sound` route to it. impact lists those sites as "stubs it". - stubs = ax_edges.stub_sites(lambda s, p: self.q(s, *p)) if self.has('call_sites') else set() + stubs = self.stub_sites() # leads an asserted-links rejection takes out (ax_links.suppressed): a set member every site of which was rejected import ax_links _rej, rej_edges = ax_links.suppressed(self.q, self.site_file) diff --git a/skills/axiomcode/SKILL.md b/skills/axiomcode/SKILL.md index ae84f0056..65d012940 100644 --- a/skills/axiomcode/SKILL.md +++ b/skills/axiomcode/SKILL.md @@ -1,7 +1,7 @@ --- name: axiomcode description: >- - Use for any why, what or where question about code — how a codebase works, where something lives, who calls it, what a change to it breaks, which tests cover an edit. Also use when resolving an issue or bug report, which names a symptom rather than a file. Examples: "How does X work?", "Where do I change Y?", "What calls this?", "What breaks if I change Z?", "Which tests do I run?", "Fix this issue". Search with grep as usual; ask the graph for what grep cannot know — which declaration a call reaches and the callers that never spell the name. Answers come from a resolved call graph, so they include callers that never spell the name — through an interface, an override, a callback, dependency injection or a config key — and every place comes with the code of the function it sits in. Call the MCP tools directly, no need to load this skill first: impact(name) for who calls it and what a change reaches (with no name: your uncommitted edits), path(start, end) for how A reaches B, tests() for the tests your edits reach, context(task, source=True) for how something works as a step-by-step call flow with each step's code. Only when those tools are not in your list, the same from the shell: `axiomcode impact `, `axiomcode path `, `axiomcode tests`, `axiomcode context "" --source`. Java, TypeScript, Python, JavaScript, C#. + Use for any why, what or where question about code — how a codebase works, where something lives, who calls it, what a change to it breaks, which tests cover an edit. Also use when resolving an issue or bug report, which names a symptom rather than a file. Examples: "How does X work?", "Where do I change Y?", "What calls this?", "What breaks if I change Z?", "Which tests do I run?", "Fix this issue". Search with grep as usual; ask the graph for what grep cannot know — which declaration a call reaches and the callers that never spell the name. Answers come from a resolved call graph, so they include callers that never spell the name — through an interface, an override, a callback, dependency injection or a config key — and every place comes with the code of the function it sits in. Call the MCP tools directly, no need to load this skill first: impact(name) for who calls it and what a change reaches (with no name: your uncommitted edits), path(start, end) for how A reaches B, tests() for the tests your edits reach, context(task, source=True) for how something works as a step-by-step call flow with each step's code. Only when those tools are not in your list, the same from the shell: `axiomcode impact `, `axiomcode path `, `axiomcode tests`, `axiomcode context "" --source`. Java, TypeScript, Python, JavaScript, C#, and HTML + CSS (impact .class, #id, a custom property as var(…), page.html, sheet.css, page.html:LINE; each language its own graph). --- # axiomcode @@ -102,7 +102,8 @@ once on `index` and kept by every rebuild after it: axiomcode index --library .venv/lib/python3.12/site-packages/requests,/deps/ir/jdk # or name them `auto` finds what the project depends on, in every language: a Python package its source imports, in its virtual -environment (`.venv`, `venv`, `$VIRTUAL_ENV`); a JavaScript / TypeScript package it imports, under `node_modules`; a +environment (`.venv`, `venv`, `$VIRTUAL_ENV`); a JavaScript / TypeScript package it imports, under `node_modules`; in a +Deno project, every `jsr:`, `npm:` and URL module it imports, from Deno's cache (`deno install` fetches them); a Java dependency `pom.xml` or `build.gradle` declares, through its `-sources.jar` in the Maven repository or Gradle's cache, or its class jar decompiled when it ships no sources (needs Vineflower: `mvn dependency:get -Dartifact=org.vineflower:vineflower:1.10.1`); a NuGet package a `.csproj` references, decompiled from its assembly diff --git a/tests/README.md b/tests/README.md index 06150d2fd..a0056b56e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -26,6 +26,11 @@ One check needs no graph and is its own script: signature edit is reported before it lands with nothing upstream did; `--range` from a branch left behind its remote reads from the remote's fork (indexes a small project) + python3 tests/index_reader.py axiomcode-index's IR reader decodes the IR's RFC 4180 quoting as csv.DictReader and + the bundle's reader do (a literal, decoration or name holding a `"`, a C# raw + string), and agrees with them on quoted tabs and newlines, CRLF, a lone CR, a + short or long row and invalid UTF-8; pass IR directories to compare them too + (no graph needed; the bundle half runs node on dist/bundle/csv.js) python3 tests/refresh.py the graph refreshes itself after an edit in every language: a query sees the edit, `changed` answers the same before and after, a burst costs one rebuild and queries during it answer (#1305; builds real graphs, needs the engine) @@ -68,6 +73,11 @@ One check needs no graph and is its own script: and the refresh is expected within the budget, never on a rules compile; --fresh waits and MCP takes fresh=true; the file table prunes exactly what each parser skips, so an edit under out/ or build/ is seen (#1594, #1595; no engine) + python3 tests/two_versions.py two axiomcode versions over one repository: an older one's graph is rebuilt + once, not again each time the older one rebuilds it back, and every answer says + two versions are running and to restart the agent session; an install marked + .orphaned_at, or whose files are gone, writes nothing under .axiomcode from its + hooks, timer, refresher, query or MCP server (no engine) python3 tests/refresh_races.py what a refresh after an edit can leave behind, each with a control: a broken graph pointer is repaired and keeps the baseline; a query mid-build waits for it and starts no build; a failed build never leaves the pointer dangling; a failed diff --git a/tests/cases/csharp/quoted-literal-text/case.json b/tests/cases/csharp/quoted-literal-text/case.json new file mode 100644 index 000000000..c2ae33141 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/case.json @@ -0,0 +1,28 @@ +{"lang": "csharp", "src": "src", + "checks": [ + {"why": "a member read on a string literal is named by the member: the IR quotes the expression (RFC 4180) and the index kept a stray quote, `Length\"` (#1835)", + "run": ["impact", "Length"], "expect_error": true, + "want": ["[by name] Orders.LabelWidth"]}, + {"why": "a raw string is a literal, and its value is the text between the quotes that open and close it", + "run": ["impact", "Orders.RawKey"], + "want": ["const Orders.RawKey = 'order.created'"]}, + {"why": "a raw string holding a quote keeps it, once", + "run": ["impact", "Orders.Raw"], + "want": ["const Orders.Raw = 'raw \"x\" here'"]}, + {"why": "a raw string opened by four quotes may hold two, and both are kept: un-doubling quotes by hand on the decoded IR read it as `\"a \" b\"`", + "run": ["impact", "Orders.Raw4"], + "want": ["const Orders.Raw4 = 'a \"\" b'"]}, + {"why": "a verbatim string's doubled quote is one quote", + "run": ["impact", "Orders.Verbatim"], + "want": ["const Orders.Verbatim = 'say \"hi\" now'"]}, + {"why": "control: a regular string holding an escaped quote is printed as written, in single quotes", + "run": ["impact", "Orders.Key"], + "want": ["const Orders.Key = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "control: a string literal holding an escaped quote is found", + "run": ["impact", "\"Order \\\"Foo\\\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "Orders.Plain"], + "want": ["const Orders.Plain = 'a plain value'"]} + ]} diff --git a/tests/cases/csharp/quoted-literal-text/src/Orders.cs b/tests/cases/csharp/quoted-literal-text/src/Orders.cs new file mode 100644 index 000000000..474829f26 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/Orders.cs @@ -0,0 +1,20 @@ +namespace App; + +public class Orders +{ + public const string Key = "a \"b\" c"; + public const string Plain = "a plain value"; + public const string Verbatim = @"say ""hi"" now"; + public const string RawKey = """order.created"""; + public const string Raw = """raw "x" here"""; + public const string Raw4 = """"a "" b""""; + + string Send(string subject) => subject; + + [Route("/orders/{code}")] + public string NotifyLate() => Send("Order \"Foo\" Is Late"); + + public string Resolve() => Send("The order has shipped"); + + public int LabelWidth() => "header".Length; +} diff --git a/tests/cases/csharp/quoted-literal-text/src/Orders.csproj b/tests/cases/csharp/quoted-literal-text/src/Orders.csproj new file mode 100644 index 000000000..d3b2f0306 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/Orders.csproj @@ -0,0 +1 @@ +net8.0 diff --git a/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs b/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs new file mode 100644 index 000000000..5c155c401 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs @@ -0,0 +1,8 @@ +namespace App; + +[System.AttributeUsage(System.AttributeTargets.Method)] +public class RouteAttribute : System.Attribute +{ + public RouteAttribute(string path) { Path = path; } + public string Path { get; } +} diff --git a/tests/cases/java/quoted-literal-text/case.json b/tests/cases/java/quoted-literal-text/case.json new file mode 100644 index 000000000..b771009ca --- /dev/null +++ b/tests/cases/java/quoted-literal-text/case.json @@ -0,0 +1,24 @@ +{"lang": "java", "src": "src", + "checks": [ + {"why": "a string literal holding an escaped double quote is found: the IR quotes such a field (RFC 4180) and the index stored it doubled, so `impact` said no literal held it (#1835)", + "run": ["impact", "\"Order \\\"Foo\\\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "a constant's value holding a quote is printed as written, in single quotes, not with each quote doubled", + "run": ["impact", "Orders.KEY"], + "want": ["const Orders.KEY = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "an annotation argument holding a quote is found as written: the decoration was stored with every quote doubled", + "run": ["impact", "\"/say \\\"hi\\\"\""], + "want": ["(0 literal site(s), 1 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "Orders.PLAIN"], + "want": ["const Orders.PLAIN = 'a plain value'"]}, + {"why": "control: a literal with no quote in it is found as before", + "run": ["impact", "\"The order has shipped\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "control: an annotation's quoted path still binds the method and registers it under the path", + "run": ["impact", "Orders.notifyLate"], + "want": ["registered by @Route \"/orders/{code}\""]} + ]} diff --git a/tests/cases/java/quoted-literal-text/src/app/Orders.java b/tests/cases/java/quoted-literal-text/src/app/Orders.java new file mode 100644 index 000000000..f8c987bfd --- /dev/null +++ b/tests/cases/java/quoted-literal-text/src/app/Orders.java @@ -0,0 +1,20 @@ +package app; + +public class Orders { + static final String KEY = "a \"b\" c"; + static final String PLAIN = "a plain value"; + + String send(String subject) { + return subject; + } + + @Route("/orders/{code}") + public String notifyLate() { + return send("Order \"Foo\" Is Late"); + } + + @Route("/say \"hi\"") + public String greet() { + return send("The order has shipped"); + } +} diff --git a/tests/cases/java/quoted-literal-text/src/app/Route.java b/tests/cases/java/quoted-literal-text/src/app/Route.java new file mode 100644 index 000000000..4bc1364fd --- /dev/null +++ b/tests/cases/java/quoted-literal-text/src/app/Route.java @@ -0,0 +1,10 @@ +package app; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +public @interface Route { + String value() default ""; + String name() default ""; +} diff --git a/tests/cases/python/method-reference-reaches-its-target/case.json b/tests/cases/python/method-reference-reaches-its-target/case.json new file mode 100644 index 000000000..d96498e7c --- /dev/null +++ b/tests/cases/python/method-reference-reaches-its-target/case.json @@ -0,0 +1,230 @@ +{ + "lang": "python", + "src": ".", + "checks": [ + { + "why": "a bound method handed to an executor as a VALUE (`loop.run_in_executor(None, svc.cancel, oid)`) is called by the executor, by no call site the graph sees. Its receiver is typed exactly as `svc.cancel()` would be, so the function that queues it reaches it, labelled as handed over, never as a call", + "run": [ + "path", + "executor_svc", + "OrderService.cancel" + ], + "want": [ + "executor_svc → OrderService.cancel: 1 of 1 target(s) reached" + ], + "avoid": [ + "the two are independent in this graph" + ] + }, + { + "why": "every resolved shape that hands cancel over is a dependent of it: a positional argument, a keyword argument, functools.partial, a receiver the function constructed itself", + "run": [ + "impact", + "OrderService.cancel", + "--limit", + "50" + ], + "want": [ + "executor_svc", + "keyword_register", + "partial_cancel", + "constructed_receiver", + "handed over as a value" + ], + "avoid": [ + "untyped_receiver", + "by_string" + ] + }, + { + "why": "control: a same-named cancel on an unrelated class gains none of OrderService.cancel's hand-offs", + "run": [ + "impact", + "LedgerService.cancel", + "--limit", + "50" + ], + "avoid": [ + "executor_svc", + "keyword_register", + "partial_cancel", + "constructed_receiver", + "handed over as a value" + ] + }, + { + "why": "and the closure continues from the hand-off: the test that drives executor_svc reaches cancel", + "run": [ + "impact", + "OrderService.cancel", + "--tests" + ], + "want": [ + "test_queue.py" + ], + "avoid": [ + "test_ledger.py" + ] + }, + { + "why": "a bound method passed as a sort key, and one read inside a lambda: the lambda is the reader", + "run": [ + "impact", + "OrderService.rank", + "--limit", + "50" + ], + "want": [ + "sort_key", + "lambda_holder" + ], + "avoid": [ + "ledger_rank" + ] + }, + { + "why": "control: a same-named method on an UNRELATED class keeps its own reader and gains none of OrderService's", + "run": [ + "impact", + "LedgerService.rank", + "--limit", + "50" + ], + "want": [ + "ledger_rank" + ], + "avoid": [ + "sort_key", + "lambda_holder" + ] + }, + { + "why": "map() over a bound method and over the method read off its CLASS (an unbound reference)", + "run": [ + "impact", + "OrderService.convert", + "--limit", + "50" + ], + "want": [ + "map_convert", + "unbound_reference" + ] + }, + { + "why": "threading.Thread(target=...), a list of callbacks, a signal handler, a view_func keyword", + "run": [ + "path", + "thread_target", + "OrderService.run_job" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a callback stored in a list literal", + "run": [ + "path", + "callback_list", + "OrderService.on_event" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a signal handler", + "run": [ + "path", + "signal_connect", + "OrderService.handle" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a view function registered by keyword", + "run": [ + "path", + "url_rule", + "OrderService.index" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "self-references: submitted to the instance's own pool, used as a sort key, and returned (an escaping reference)", + "run": [ + "path", + "Worker.submit_bound_self", + "Worker._do_work" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a self method as a sort key", + "run": [ + "path", + "Worker.sort_by_self_key", + "Worker._rank" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a returned bound method escapes to whoever called escapes()", + "run": [ + "path", + "Worker.escapes", + "Worker._handle" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a plain module function handed to a pool is now the engine's resolved reference, not a name match", + "run": [ + "path", + "submit_plain_function", + "top_level_job" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "control: an UNTYPED receiver's reference resolves to nothing, exactly like `thing.cancel()`, so there is no edge to OrderService.cancel", + "run": [ + "path", + "untyped_receiver", + "OrderService.cancel" + ], + "expect_error": true, + "avoid": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "control: the method's name in a string, an attribute read OFF the method object (`svc.audit.__name__`), str() of it, and an identity test on it do not run it", + "run": [ + "impact", + "OrderService.audit", + "--limit", + "50" + ], + "avoid": [ + "by_string", + "reads_its_name", + "prints_it", + "compares_it" + ] + } + ] +} diff --git a/tests/cases/python/method-reference-reaches-its-target/orders.py b/tests/cases/python/method-reference-reaches-its-target/orders.py new file mode 100644 index 000000000..44a34d5f0 --- /dev/null +++ b/tests/cases/python/method-reference-reaches-its-target/orders.py @@ -0,0 +1,163 @@ +"""Methods and functions handed over as REFERENCES: named as a value, called later by whoever holds them.""" +import functools +import signal +import threading +from concurrent.futures import ThreadPoolExecutor + + +class OrderService: + def cancel(self, oid): + return oid + + def rank(self, x): + return x + + def convert(self, row): + return row + + def on_event(self, e): + return e + + def handle(self, sig, frame): + return sig + + def run_job(self): + return 1 + + def index(self): + return "ok" + + def audit(self, oid): + return oid + + +class LedgerService: + """Same method names as OrderService, and no relation to it.""" + + def cancel(self, oid): + return oid + + def rank(self, x): + return x + + +def top_level_job(item): + return item + + +class Worker: + def __init__(self): + self.pool = ThreadPoolExecutor() + + def _do_work(self, item): + return item + + def _rank(self, x): + return x + + def _handle(self, e): + return e + + def submit_bound_self(self, item): + return self.pool.submit(self._do_work, item) + + def sort_by_self_key(self, xs): + return sorted(xs, key=self._rank) + + def escapes(self): + return self._handle + + +def submit_plain_function(pool, item): + return pool.submit(top_level_job, item) + + +def executor_svc(loop, svc: OrderService, oid): + return loop.run_in_executor(None, svc.cancel, oid) + + +def thread_target(svc: OrderService): + t = threading.Thread(target=svc.run_job) + t.start() + return t + + +def callback_list(handler: OrderService): + callbacks = [handler.on_event] + return callbacks + + +def register(handler): + return handler + + +def keyword_register(svc: OrderService): + return register(handler=svc.cancel) + + +def sort_key(svc: OrderService, xs): + return sorted(xs, key=svc.rank) + + +def map_convert(svc: OrderService, rows): + return list(map(svc.convert, rows)) + + +def unbound_reference(rows): + return list(map(OrderService.convert, rows)) + + +def partial_cancel(svc: OrderService, oid): + return functools.partial(svc.cancel, oid) + + +def signal_connect(svc: OrderService): + signal.signal(signal.SIGTERM, svc.handle) + + +def url_rule(app, views: OrderService): + app.add_url_rule("/x", view_func=views.index) + + +def constructed_receiver(oid): + svc = OrderService() + return functools.partial(svc.cancel, oid) + + +def lambda_holder(svc: OrderService, xs): + return sorted(xs, key=lambda x: svc.rank) + + +# ── near-miss controls ─────────────────────────────────────────────────────── +def untyped_receiver(executor, thing, oid): + """`thing` has no type: `thing.cancel` may be any class's cancel, so it resolves to none.""" + return executor.submit(thing.cancel, oid) + + +def ledger_rank(ledger: LedgerService, xs): + """LedgerService.rank, not OrderService.rank.""" + return sorted(xs, key=ledger.rank) + + +def by_string(svc: OrderService): + """The method's NAME in a string is not a reference to it.""" + return getattr_name("audit") + + +def getattr_name(n): + return n + + +def reads_its_name(svc: OrderService): + """Reading an attribute OF the method object does not run it.""" + return svc.audit.__name__ + + +def prints_it(svc: OrderService): + """str() of the method object runs the function object's __str__, not the method.""" + return str(svc.audit) + + +def compares_it(svc: OrderService, cb): + """An identity test on the method object does not run it.""" + return cb is svc.audit diff --git a/tests/cases/python/method-reference-reaches-its-target/tests/test_ledger.py b/tests/cases/python/method-reference-reaches-its-target/tests/test_ledger.py new file mode 100644 index 000000000..5ba83160c --- /dev/null +++ b/tests/cases/python/method-reference-reaches-its-target/tests/test_ledger.py @@ -0,0 +1,5 @@ +from orders import LedgerService, ledger_rank + + +def test_ledger_rank(): + assert ledger_rank(LedgerService(), [2, 1]) == [1, 2] diff --git a/tests/cases/python/method-reference-reaches-its-target/tests/test_queue.py b/tests/cases/python/method-reference-reaches-its-target/tests/test_queue.py new file mode 100644 index 000000000..8d8de3896 --- /dev/null +++ b/tests/cases/python/method-reference-reaches-its-target/tests/test_queue.py @@ -0,0 +1,9 @@ +from unittest import mock + +from orders import OrderService, executor_svc + + +def test_queues_the_cancel(): + loop = mock.Mock() + executor_svc(loop, OrderService(), "o-1") + loop.run_in_executor.assert_called_once() diff --git a/tests/cases/python/quoted-literal-text/case.json b/tests/cases/python/quoted-literal-text/case.json new file mode 100644 index 000000000..04c622f2d --- /dev/null +++ b/tests/cases/python/quoted-literal-text/case.json @@ -0,0 +1,23 @@ +{"lang": "python", "src": "src", + "checks": [ + {"why": "a string literal holding a double quote is found: the IR quotes such a field (RFC 4180) and the index stored it doubled, so `impact` said no literal held it (#1835)", + "run": ["impact", "\"Order \"Foo\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "a constant's value holding a quote is printed as written, in single quotes, not with each quote doubled", + "run": ["impact", "KEY"], + "want": ["const KEY = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "PLAIN"], + "want": ["const PLAIN = 'a plain value'"]}, + {"why": "control: a literal with no quote in it is found as before", + "run": ["impact", "\"The order has shipped\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "a decorator's quoted path reads cleanly: it binds the function and registers it under the path", + "run": ["impact", "\"/orders/{code}\""], + "want": ["1 decoration site(s)", "binds to it through @route"]}, + {"why": "a decorator path written as a triple-quoted string is read as the path", + "run": ["impact", "notify_triple"], + "want": ["registered by @route \"/triple/{code}\""]} + ]} diff --git a/tests/cases/python/quoted-literal-text/src/app/__init__.py b/tests/cases/python/quoted-literal-text/src/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cases/python/quoted-literal-text/src/app/orders.py b/tests/cases/python/quoted-literal-text/src/app/orders.py new file mode 100644 index 000000000..85a5aadbf --- /dev/null +++ b/tests/cases/python/quoted-literal-text/src/app/orders.py @@ -0,0 +1,22 @@ +KEY = "a \"b\" c" +PLAIN = "a plain value" + + +def route(path): + def deco(f): + return f + return deco + + +def send(subject): + return subject + + +@route("/orders/{code}") +def notify(): + return send('Order "Foo" Is Late') + + +@route("""/triple/{code}""") +def notify_triple(): + return send("The order has shipped") diff --git a/tests/cases/typescript/namespace-reexport/case.json b/tests/cases/typescript/namespace-reexport/case.json new file mode 100644 index 000000000..226beb24f --- /dev/null +++ b/tests/cases/typescript/namespace-reexport/case.json @@ -0,0 +1,6 @@ +{"lang": "typescript", "src": "src", + "checks": [ + {"why": "a namespace import re-exported by name (export { z }, export { z as zz }, export default z) binds the importer's name to that namespace, so z.alpha() reaches the declaration", "run": ["impact", "ns/impl#alpha"], "want": ["src/app.ts:5"], "avoid": ["calls a method of this name", "src/app.ts:10"]}, + {"why": "the alias and the default forms bind the same namespace", "run": ["impact", "beta"], "want": ["src/app.ts:6", "src/app.ts:11"], "avoid": ["calls a method of this name"]}, + {"why": "export * as core is one name whose value is the namespace, and it survives a further export * barrel", "run": ["impact", "delta"], "want": ["src/app.ts:8", "src/app.ts:9"], "avoid": ["calls a method of this name"]}, + {"why": "and it is not a star: impl's alpha is not exported at the top level of the barrel, so the bare alpha() is other.ts's alone", "run": ["impact", "ns/other#alpha"], "want": ["src/app.ts:10"], "avoid": ["calls a method of this name"]}]} diff --git a/tests/cases/typescript/namespace-reexport/src/app.ts b/tests/cases/typescript/namespace-reexport/src/app.ts new file mode 100644 index 000000000..100a36c4a --- /dev/null +++ b/tests/cases/typescript/namespace-reexport/src/app.ts @@ -0,0 +1,11 @@ +import zdef, { z, zz, core } from "./ns/index.ts"; +import { z as zOuter, alpha } from "./ns/outer.ts"; +import * as direct from "./ns/impl.ts"; + +export function viaNamed(): number { return z.alpha(); } +export function viaAlias(): number { return zz.beta(); } +export function viaDefault(): number { return zdef.gamma(); } +export function viaStarAs(): number { return core.delta(); } +export function viaStarChain(): number { return zOuter.delta(); } +export function topLevelAlpha(): number { return alpha(); } +export function controlDirect(): number { return direct.beta(); } diff --git a/tests/cases/typescript/namespace-reexport/src/ns/impl.ts b/tests/cases/typescript/namespace-reexport/src/ns/impl.ts new file mode 100644 index 000000000..cad5ceee0 --- /dev/null +++ b/tests/cases/typescript/namespace-reexport/src/ns/impl.ts @@ -0,0 +1,4 @@ +export function alpha(): number { return 1; } +export function beta(): number { return 2; } +export function gamma(): number { return 3; } +export function delta(): number { return 4; } diff --git a/tests/cases/typescript/namespace-reexport/src/ns/index.ts b/tests/cases/typescript/namespace-reexport/src/ns/index.ts new file mode 100644 index 000000000..e13bec0ba --- /dev/null +++ b/tests/cases/typescript/namespace-reexport/src/ns/index.ts @@ -0,0 +1,5 @@ +import * as z from "./impl.ts"; +export { z }; +export { z as zz }; +export default z; +export * as core from "./impl.ts"; diff --git a/tests/cases/typescript/namespace-reexport/src/ns/other.ts b/tests/cases/typescript/namespace-reexport/src/ns/other.ts new file mode 100644 index 000000000..21e3537be --- /dev/null +++ b/tests/cases/typescript/namespace-reexport/src/ns/other.ts @@ -0,0 +1 @@ +export function alpha(): number { return 10; } diff --git a/tests/cases/typescript/namespace-reexport/src/ns/outer.ts b/tests/cases/typescript/namespace-reexport/src/ns/outer.ts new file mode 100644 index 000000000..455f01fce --- /dev/null +++ b/tests/cases/typescript/namespace-reexport/src/ns/outer.ts @@ -0,0 +1,2 @@ +export * from "./index.ts"; +export * from "./other.ts"; diff --git a/tests/cases/typescript/namespace-reexport/src/tsconfig.json b/tests/cases/typescript/namespace-reexport/src/tsconfig.json new file mode 100644 index 000000000..c7d0def2a --- /dev/null +++ b/tests/cases/typescript/namespace-reexport/src/tsconfig.json @@ -0,0 +1 @@ +{"compilerOptions":{"allowImportingTsExtensions":true,"noEmit":true,"module":"ESNext","moduleResolution":"bundler"}} diff --git a/tests/index_reader.py b/tests/index_reader.py new file mode 100644 index 000000000..3f6508eac --- /dev/null +++ b/tests/index_reader.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""tests/index_reader.py: axiomcode-index reads the IR as the bundle does, row for row and key for key. + +axiomcode-index reads every IR row through ir_rows(), a line splitter that replaced csv.DictReader because the +DictReader was most of the index's time on a large repository. The index's tables are made of these dicts, so any +difference is a different graph. The parser writes RFC 4180 over a tab (EntityUtils.escapeTsv quotes a field that +holds a `"` and doubles the quote), and ir_rows decodes it: read QUOTE_NONE, as it once was, the literal +`Order "Foo"` was stored `Order ""Foo""` and `impact` said no literal held it (#1835). + +Each fixture is read three ways, and they must agree: + - csv.DictReader(f, delimiter='\\t') over open(p, newline=''), RFC 4180 quoting on: the same list of dicts; + - the bundle's own reader, graph/bundle/csv.ts readRfc4180 (the built dist/bundle/csv.js, run with node): its rows, + zipped with the header, must give the same dicts. A lone CR is left out of this half on purpose: ir_rows keeps + the csv module's rule that it ends a line, and the bundle splits on \\n only. A blank line is no row in either + (the bundle yields one empty field for it, which is dropped here); + - for the shapes the bug was about, the decoded value itself, written out, so two readers agreeing on a wrong value + is not a pass: a literal with quotes, a decoration with a quoted path, a C# raw string `\"\"\"x\"\"\"`, a C# + member access on a string (`"x".Length`), a quote inside a field, and the control, a field with no quote. +The other shapes are where a line splitter and the csv module could part: a quoted field holding a tab or a newline, +CRLF, a lone CR, a blank line, a short row (None for the missing columns), a long row (the rest under the key None), +a repeated header name, a blank header, an empty file, a header with no rows, no final newline, bytes that are not +UTF-8, and the Unicode line separators that str.splitlines() would split on and the csv module does not. + + python3 tests/index_reader.py [IR_DIR ...] each IR_DIR: every *.csv in it is compared with csv.DictReader too +""" +import ast, csv, glob, itertools, json, os, shutil, subprocess, sys, tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCRIPT = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode-index') +BUNDLE_CSV = os.path.join(ROOT, 'dist', 'bundle', 'csv.js') # graph/bundle/csv.ts, built +csv.field_size_limit(10**9) +MISSING = object() + + +def load_ir_rows(): + """ir_rows() out of the script without running it (the script indexes a graph at import).""" + tree = ast.parse(open(SCRIPT, encoding='utf-8').read()) + fn = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'ir_rows'] + if not fn: + sys.exit('FAIL: axiomcode-index defines no ir_rows()') + ns = {} + exec(compile(ast.Module(body=fn, type_ignores=[]), SCRIPT, 'exec'), ns) + return ns['ir_rows'] + + +def dictreader(p): + with open(p, newline='', encoding='utf-8', errors='replace') as f: + return list(csv.DictReader(f, delimiter='\t')) + + +BUNDLE_JS = r''' +const { readRfc4180 } = require(process.argv[1]); +(async () => { + const out = {}; + for (const p of process.argv.slice(2)) { out[p] = []; for await (const r of readRfc4180(p)) out[p].push(r); } + process.stdout.write(JSON.stringify(out)); +})(); +''' + + +def bundle_rows(paths): + """{path: the bundle reader's rows}, or None when node or the built reader is not there.""" + node = shutil.which('node') + if not node or not os.path.exists(BUNDLE_CSV): + return None + r = subprocess.run([node, '-e', BUNDLE_JS, BUNDLE_CSV] + paths, capture_output=True, text=True, encoding='utf-8') + if r.returncode: + sys.exit(f'FAIL: the bundle reader did not run: {r.stderr[-400:]}') + return json.loads(r.stdout) + + +def as_dicts(rows): + """The bundle's rows as ir_rows' dicts: the first row is the header, a blank line (one empty field) is no row.""" + if not rows: + return [] + keys = rows[0] if rows[0] != [''] else [] + n, out = len(keys), [] + for v in rows[1:]: + if v == ['']: + continue + d = dict(zip(keys, v)) + if len(v) > n: + d[None] = v[n:] + for k in keys[len(v):]: + d[k] = None + out.append(d) + return out + + +FIXTURES = { + 'plain': b'a\tb\tc\n1\t2\t3\n4\t5\t6\n', + 'quoted-tab': b'a\tb\tc\n"x\ty"\t2\t3\n1\t"p\tq"\t3\n', + 'quoted-newline': b'a\tb\tc\n"line one\nline two"\t2\t3\n4\t5\t6\n', + 'quoted-crlf-inside': b'a\tb\n"x\r\ny"\tz\n', + 'doubled-quotes': b'a\tb\n"@r.post(""/orders"")"\t1\n"""Fee"""\t2\n', + 'crlf': b'a\tb\tc\r\n1\t2\t3\r\n4\t5\t6\r\n', + 'lone-cr': b'a\tb\r1\t2\r3\t4\r', + 'mixed-endings': b'a\tb\n1\t2\r\n3\t4\r5\t6\n', + 'blank-lines': b'a\tb\n\n1\t2\n\r\n\n3\t4\n\n', + 'short-row': b'a\tb\tc\n1\t2\n1\n', + 'long-row': b'a\tb\n1\t2\t3\t4\n', + 'trailing-tab': b'a\tb\n1\t\n\t\n', + 'repeated-header': b'id\tname\tid\n1\tx\t2\n3\n', + 'blank-header': b'\n1\t2\n3\n', + 'empty': b'', + 'header-only': b'a\tb\n', + 'header-only-no-newline': b'a\tb', + 'no-final-newline': b'a\tb\n1\t2\n3\t4', + 'not-utf8': b'a\tb\n\xff\xfe\t\xc3\x28\n', + 'unicode-separators': 'a\tb\nx y\tz\u0085w\n1\u000b2\t3\u000c4\n'.encode('utf-8'), + 'spaces': b'a\tb\n 1 \t 2 \n', + # the quoting EntityUtils.escapeTsv writes, as the parser wrote it for each shape + 'quoted-literal': b'value\tkind\n"Order ""Foo"" Is Late"\tstring\n', + 'quoted-decoration': b'name\ttext\nRequestMapping\t"@RequestMapping(""/a/{b}"")"\nRequestMapping\t"""/a/{b}"""\n', + 'csharp-raw-string': b'value\tkind\n"""""""x"""""""\tstring\n', + 'csharp-string-member': b'name\tkind\n"""x"".Length"\tMEMBER_ACCESS\n', + 'plain-control': b'value\tkind\nThe order has shipped\tstring\n', + # what RFC 4180 says about a quote that does not open a field, and about text after the closing quote + 'quote-inside-field': b'a\tb\tc\nsay "hi"\tx"\ty"z"\n', + 'text-after-close': b'a\tb\n"ab"cd\t"x""y"z\n', + 'empty-quoted': b'a\tb\tc\n""\t""""\t\n', + 'quoted-crlf-row': b'a\tb\r\n"x ""y"""\t2\r\n', + 'quoted-header': b'"a"\t"b ""c"""\n1\t2\n', + 'quoted-open-at-eof': b'a\tb\n"x\ty\n', +} + +# the decoded values, written out +DECODED = { + 'quoted-literal': [{'value': 'Order "Foo" Is Late', 'kind': 'string'}], + 'quoted-decoration': [{'name': 'RequestMapping', 'text': '@RequestMapping("/a/{b}")'}, + {'name': 'RequestMapping', 'text': '"/a/{b}"'}], + 'csharp-raw-string': [{'value': '"""x"""', 'kind': 'string'}], + 'csharp-string-member': [{'name': '"x".Length', 'kind': 'MEMBER_ACCESS'}], + 'plain-control': [{'value': 'The order has shipped', 'kind': 'string'}], + 'quote-inside-field': [{'a': 'say "hi"', 'b': 'x"', 'c': 'y"z"'}], + 'quoted-open-at-eof': [{'a': 'x\ty\n', 'b': None}], + 'text-after-close': [{'a': 'abcd', 'b': 'x"yz'}], + 'quoted-header': [{'a': '1', 'b "c"': '2'}], +} +# a lone CR ends an ir_rows line and is text to the bundle's reader (the parser never writes one: escapeTsv escapes it) +NOT_BUNDLE = {'lone-cr', 'mixed-endings'} + + +def main(argv): + ir_rows = load_ir_rows() + fails = checks = 0 + rows_compared = bundle_checked = 0 + with tempfile.TemporaryDirectory() as w: + paths = {} + for name, data in FIXTURES.items(): + p = paths[name] = os.path.join(w, name + '.csv') + open(p, 'wb').write(data) + bundle = bundle_rows(list(paths.values())) + for name, p in paths.items(): + want, got = dictreader(p), list(ir_rows(p)) + checks += 1; rows_compared += len(want) + if want != got: + fails += 1 + print(f'FAIL {name}: csv.DictReader {want!r}\n ir_rows {got!r}') + if name in DECODED: + checks += 1 + if got != DECODED[name]: + fails += 1; print(f'FAIL {name}: decoded {got!r}\n want {DECODED[name]!r}') + if bundle is not None and name not in NOT_BUNDLE: + checks += 1; bundle_checked += 1 + b = as_dicts(bundle[p]) + if b != got: + fails += 1; print(f'FAIL {name}: bundle readRfc4180 {b!r}\n ir_rows {got!r}') + # not vacuous: the fixtures hold rows, and the shapes the reader special-cases are among them + p = paths['short-row'] + checks += 1 + if list(ir_rows(p)) != [{'a': '1', 'b': '2', 'c': None}, {'a': '1', 'b': None, 'c': None}]: + fails += 1; print(f'FAIL short-row: {list(ir_rows(p))!r}') + p = paths['long-row'] + checks += 1 + if list(ir_rows(p)) != [{'a': '1', 'b': '2', None: ['3', '4']}]: + fails += 1; print(f'FAIL long-row: {list(ir_rows(p))!r}') + # a real IR: every file of every directory named on the command line + files = 0 + for d in argv: + for p in sorted(glob.glob(os.path.join(d, '*.csv'))): + files += 1; checks += 1 + n = 0 + with open(p, newline='', encoding='utf-8', errors='replace') as f: # streamed: an IR file can be 10^6 rows + for a, b in itertools.zip_longest(csv.DictReader(f, delimiter='\t'), ir_rows(p), fillvalue=MISSING): + n += 1 + if a != b: + fails += 1; print(f'FAIL {p} row {n}: csv.DictReader {a!r}\n ir_rows {b!r}'); break + rows_compared += n + if rows_compared < 20: + fails += 1; print(f'FAIL only {rows_compared} rows compared') + if bundle is None: + print('index_reader: the bundle half was NOT run (no node on PATH, or dist/bundle/csv.js is not built)') + elif bundle_checked < len(FIXTURES) - len(NOT_BUNDLE): + fails += 1; print(f'FAIL the bundle reader was compared on only {bundle_checked} fixtures') + print(f"index_reader: {'ok' if not fails else 'FAILED'} ({checks - fails}/{checks} checks, {rows_compared} rows," + f" {len(FIXTURES)} fixtures, {bundle_checked} against the bundle reader{f', {files} IR files' if files else ''})") + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tests/refresh.py b/tests/refresh.py index 2a02b7c6a..405029879 100644 --- a/tests/refresh.py +++ b/tests/refresh.py @@ -3,7 +3,7 @@ Each language gets a throwaway git repository with a real graph. Then, per language: - up to date `index` with nothing changed does not rebuild + up to date `index` with nothing changed does not rebuild; with AXIOMCODE_REINDEX=1 it does query an edit adds a function; a query verb, which starts the refresher and waits for it, finds it baseline `changed` answers the same before and after the refresh absorbed the edit: the refresh moves the graph, not the baseline edits are measured against. The answer must name the edit (a pass on two @@ -110,6 +110,13 @@ def check(ok, why, detail=''): check(m.get('refresh_reason') == 'axiomcode index' and m.get('refreshed_at', '').endswith('Z'), f'{lang}: the graph records when and why it was built', json.dumps(m)) t0 = time.time(); again = sh(repo, AX, 'index', '.', '--lang', lang, env=env); took = time.time() - t0 check('graph up to date' in again.stdout, f'{lang}: `index` with nothing changed does not rebuild ({took:.1f}s)', again.stdout + again.stderr) + # AXIOMCODE_REINDEX=1 rebuilds with nothing changed, as SKILL.md and the build's own messages promise; it used to + # answer "graph up to date" whenever the files matched the table. The control is the check above: the same + # `index` without it does not rebuild + built = meta().get('refreshed_at') + rx = sh(repo, AX, 'index', '.', '--lang', lang, env=dict(env, AXIOMCODE_REINDEX='1')) + check(rx.returncode == 0 and 'graph up to date' not in rx.stdout and 'building' in rx.stdout and meta().get('refreshed_at') != built, + f'{lang}: AXIOMCODE_REINDEX=1 `index` rebuilds although nothing changed', rx.stdout + rx.stderr) # ── built by an older axiomcode ──────────────────────────────────────────────────────────────────── # no file changed, but the table says another engine built the graph (what an update leaves behind): not up diff --git a/tests/two_versions.py b/tests/two_versions.py new file mode 100644 index 000000000..f36c86f59 --- /dev/null +++ b/tests/two_versions.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""tests/two_versions.py: two axiomcode versions over one repository rebuild its graph once, not on every turn, say that +two versions are running, and an orphaned older install writes nothing into .axiomcode. + +An agent session started before an engine or plugin upgrade keeps the old version's MCP server and hooks running beside +the new ones. The old one rebuilt the graph with its engine, the new one rebuilt it back, and so on, a full rebuild on +every turn; the answers said "stale graph"; and the old one's refresher recreated .axiomcode while the user deleted it. + + turns this axiomcode rebuilds a graph an older one built, once; when that older one rebuilds it again afterwards, + this one does not rebuild it in the background again (refresher, hook, query), and the status, the query's + note and the hook all say two versions are running and to restart the agent session. Controls: an older + graph met for the first time is rebuilt; a rebuild that failed (the older graph not rebuilt since) is retried; + `index` (uptodate) still rebuilds a contested graph, asked for by name + newer an older axiomcode over a graph a newer one built keeps it, and says the two-versions case (note and the + `newer` command), not only "built by a newer axiomcode" + orphan an install whose plugin directory carries .orphaned_at, or whose files are gone, writes nothing under + .axiomcode from its refresh hook (session start, an edit), its other hooks, its timer (hook_kick), its + refresher, a query, or its MCP server, and says it is an orphaned older version to restart; the MCP server + refuses to start. Control: the same copy without the mark does write (the refresher starts on an edit) + +No engine: a stand-in engine directory, and a stand-in build that records what built the graph, as freshness.py does. + + python3 tests/two_versions.py [-v] +""" +import json, os, shutil, subprocess, sys, tempfile, time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import freshness # its stand-in engine and repository +from freshness import ax_fresh, fake_engine, fake_repo, check, RESULTS, SCRIPTS, ROOT, DRIVER + +PLUGIN = os.path.join(ROOT, 'plugins', 'axiomcode') + +# the stand-in build: records that it ran, and writes the file table as this axiomcode's build would (built_by, built) +BUILD = r""" +import json, os, sys, time +sys.path.insert(0, os.environ['TV_SCRIPTS']); import ax_fresh +repo = sys.argv[2] +open(os.path.join(repo, 'builds'), 'a').write('x\n') +tp = ax_fresh.table_path(repo); t = json.load(open(tp)) +t.update(built=time.time(), files=ax_fresh.snapshot(repo, 'python', repo), built_by=ax_fresh.built_by(ax_fresh.current_engine(repo), 'python')) +json.dump(t, open(tp, 'w')) +""" + + +def tree(d): + """every path under d with its size and mtime: what a process wrote there""" + out = {} + for root, dirs, files in os.walk(d): + for f in dirs + files: + p = os.path.join(root, f) + try: st = os.lstat(p); out[os.path.relpath(p, d)] = (st.st_size, st.st_mtime_ns) + except OSError: pass + return out + + +def diff(a, b): + return sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k)) + + +def turns_checks(work, env): + fb = os.path.join(work, 'build.py'); open(fb, 'w').write(BUILD) + runner = os.path.join(work, 'runner.sh'); open(runner, 'w').write(f'#!/bin/sh\nexec "{sys.executable}" "{fb}" "$@"\n'); os.chmod(runner, 0o755) + wenv = dict(env, AXIOMCODE_BASH=runner, TV_SCRIPTS=SCRIPTS) + + def builds(repo): + try: return len(open(os.path.join(repo, 'builds')).read().split()) + except OSError: return 0 + + def worker(repo): + subprocess.run([sys.executable, os.path.join(SCRIPTS, 'ax_fresh.py'), 'worker', repo], env=wenv, capture_output=True, text=True, timeout=60) + return builds(repo) + + other = dict(engine_version='0.9.0', engine_hash='0' * 40, engine_stat='0' * 40) + + def older_turn(repo): + """the older axiomcode rebuilds the graph with its own engine: the table records it, built now""" + tp = ax_fresh.table_path(repo); t = json.load(open(tp)) + t['built_by'].update(other); t['built'] = time.time(); json.dump(t, open(tp, 'w')) + + repo = fake_repo(work, 'turns'); os.remove(os.path.join(repo, '.axiomcode/refresh.json')) + older_turn(repo) + s = ax_fresh.status(repo) + check("turns: control: an older axiomcode's graph met for the first time is stale and not contested", + s.get('engine', '').startswith('graph built by an older axiomcode') and not s.get('contested'), s) + n1 = worker(repo) + check("turns: control: the refresher rebuilds an older axiomcode's graph once", n1 == 1, n1) + check("turns: after this one's rebuild the graph is current", ax_fresh.status(repo).get('state') == 'fresh', ax_fresh.status(repo)) + rounds = [] + for i in range(3): + time.sleep(0.02); older_turn(repo) # the older one, still running, rebuilds it back + rounds.append(worker(repo)) + check(f"turns: the older one rebuilding it back three times costs no further rebuild here (builds after each turn: {rounds})", + rounds == [1, 1, 1], rounds) + s = ax_fresh.status(repo) + two = s.get('contested', '') + check("turns: the status names the two-versions case: both versions, and restart the agent session", + 'two axiomcode versions are running' in two and 'axiomcode 0.9.0' in two and 'axiomcode 1.0.0' in two + and 'restart the agent session' in two and ax_fresh.read_state(repo).get('state') == 'contested', s) + # a query answers from it, starts nothing, waits for nothing, and its note is the two-versions line, not a stale-graph one + driver = os.path.join(work, 'driver.py'); open(driver, 'w').write(DRIVER.replace("ax_fresh.kick = lambda *a, **k: True", + "ax_fresh.kick = lambda *a, **k: open(os.path.join(sys.argv[2], 'kicked'), 'w').close() or True")) + t0 = time.time() + r = subprocess.run([sys.executable, driver, SCRIPTS, repo, 'impact', '--', sys.executable, '-c', "print('rows')"], + capture_output=True, text=True, env=dict(env, AXIOMCODE_FRESH_WAIT='30')) + took = time.time() - t0 + check(f"turns: a query answers at once ({took:.1f}s), starts no rebuild, and says two versions are running rather than 'rebuilding'", + r.stdout.strip() == 'rows' and 'two axiomcode versions are running' in r.stderr and 'rebuilding in the background' not in r.stderr + and 'queued' not in r.stderr and 'background rebuild' not in r.stdout and not os.path.exists(os.path.join(repo, 'kicked')) and took < 10, (r.stdout, r.stderr)) + h = ax_fresh.hook_kick(repo, 'the PostToolUse hook', session='s9', event='UserPromptSubmit') + check("turns: the hook says the two-versions case once", 'two axiomcode versions are running' in h + and ax_fresh.hook_kick(repo, 'the PostToolUse hook', session='s9', event='UserPromptSubmit') == '', h) + time.sleep(0.02); older_turn(repo) # the older one's graph, whichever way the turns above went + u = subprocess.run([sys.executable, os.path.join(SCRIPTS, 'ax_fresh.py'), 'uptodate', repo, 'python', '', ''], capture_output=True, text=True, env=env) + check("turns: control: `index` (uptodate) still rebuilds a contested graph, asked for by name", u.returncode == 1 and 'rebuilding' in u.stdout, u.stdout) + + # control: a rebuild that FAILED leaves the older graph as it was (not rebuilt since): that is no contest, it is retried + repo = fake_repo(work, 'failed'); os.remove(os.path.join(repo, '.axiomcode/refresh.json')) + older_turn(repo); ax_fresh.displace(repo, ax_fresh.load_table(repo)) + check("turns: control: an older graph not rebuilt since this one displaced it (a failed rebuild) is not contested, and is rebuilt", + not ax_fresh.status(repo).get('contested') and worker(repo) == 1, ax_fresh.status(repo)) + + +def newer_checks(work, env): + repo = fake_repo(work, 'newer'); os.remove(os.path.join(repo, '.axiomcode/refresh.json')) + tp = ax_fresh.table_path(repo); t = json.load(open(tp)); t['built_by'].update(engine_version='1.0.1'); json.dump(t, open(tp, 'w')) + s = ax_fresh.status(repo) + note = ax_fresh.note(s) + check("newer: an older axiomcode over a newer one's graph says two versions are running and to restart, and that it does not rebuild", + s.get('newer') and 'engine 1.0.1, this one is 1.0.0' in note and 'two axiomcode versions are running' in note + and 'restart the agent session' in note and 'does not rebuild' in note, note) + n = subprocess.run([sys.executable, os.path.join(SCRIPTS, 'ax_fresh.py'), 'newer', repo], capture_output=True, text=True, env=env) + check("newer: `index` is told the two-versions case too", n.returncode == 0 and 'two axiomcode versions are running' in n.stdout, n.stdout) + + +def orphan_checks(work, env): + fb = os.path.join(work, 'build-orphan.py'); open(fb, 'w').write("import os, sys; open(os.path.join(sys.argv[2], 'builds'), 'a').write('x\\n')\n") + runner = os.path.join(work, 'runner-orphan.sh'); open(runner, 'w').write(f'#!/bin/sh\nexec "{sys.executable}" "{fb}" "$@"\n'); os.chmod(runner, 0o755) + cfg = os.path.join(work, 'agent-config'); os.makedirs(cfg) + # TMPDIR: the hooks remember a session's repositories there, and a session-start hook refreshes them too + oenv = dict(env, AXIOMCODE_BASH=runner, AXIOMCODE_REFRESH_DEBOUNCE='0.05', CLAUDE_CONFIG_DIR=cfg, TMPDIR=os.path.join(work, 'tmp')) + os.makedirs(oenv['TMPDIR']) + for k in ('AXIOMCODE_PLUGIN_ROOT', 'AXIOMCODE_NO_REFRESH'): oenv.pop(k, None) + + # this checkout's plugin version, which the orphan message names (it moves with every release) + VERSION = json.load(open(os.path.join(PLUGIN, '.claude-plugin', 'plugin.json'), encoding='utf-8'))['version'] + + def install(name, mark): + """a copy of this plugin in a versioned plugin cache: ///""" + d = os.path.join(work, 'cache', name, 'axiomcode', VERSION) + shutil.copytree(PLUGIN, d, ignore=shutil.ignore_patterns('__pycache__')) + if mark: open(os.path.join(d, '.orphaned_at'), 'w').write('1791880805485\n') + return d + + def edited_repo(name): + repo = fake_repo(work, name); os.remove(os.path.join(repo, '.axiomcode/refresh.json')) + open(os.path.join(repo, 'shop/api.py'), 'a').write('\ndef audit(items):\n return total(items)\n') + return repo + + def hook(plugin, repo, event, **extra): + ev = dict(hook_event_name=event, cwd=repo, session_id='two-versions-' + os.path.basename(repo), **extra) + return subprocess.run([sys.executable, os.path.join(plugin, 'hooks', 'refresh.py')], input=json.dumps(ev), capture_output=True, text=True, env=oenv, timeout=30) + + def everything(plugin, repo): + """every path by which an install writes into .axiomcode without being asked to; returns what the hooks printed""" + said = hook(plugin, repo, 'SessionStart').stdout + hook(plugin, repo, 'PostToolUse', tool_name='Edit', tool_input=dict(file_path=os.path.join(repo, 'shop/api.py'))) + for h in ('changes.py', 'orient.py', 'assist.py'): + subprocess.run([sys.executable, os.path.join(plugin, 'hooks', h)], input=json.dumps(dict(hook_event_name='UserPromptSubmit', cwd=repo, session_id='two-versions-' + os.path.basename(repo), + prompt='how does run reach total')), capture_output=True, text=True, env=oenv, timeout=60) + scripts = os.path.join(plugin, 'skills', 'axiomcode', 'scripts') + code = ("import sys; sys.path.insert(0, sys.argv[1]); import ax_fresh; ax_fresh.hook_kick(sys.argv[2], 'the timer'); " + "ax_fresh.kick(sys.argv[2]); ax_fresh.worker(sys.argv[2])") + subprocess.run([sys.executable, '-c', code, scripts, repo], capture_output=True, text=True, env=oenv, timeout=60) + q = subprocess.run([sys.executable, os.path.join(scripts, 'ax_fresh.py'), 'query', repo, 'impact', '--', sys.executable, '-c', "print('rows')"], + capture_output=True, text=True, env=oenv, timeout=60) + time.sleep(1.0) # a detached refresher, had one been started, has written by now + return said, q.stdout + + # control first: the same copy, not orphaned, does write on an edit (else "wrote nothing" below would prove nothing) + live = install('live', mark=False); repo = edited_repo('orphan-control') + before = tree(os.path.join(repo, '.axiomcode')); everything(live, repo); time.sleep(1.5) + wrote = diff(before, tree(os.path.join(repo, '.axiomcode'))) + check("orphan: control: the same install, not orphaned, writes under .axiomcode after an edit (the refresher starts)", + 'refresh.log' in wrote and 'refresh.json' in wrote, wrote) + + plugin = install('marked', mark=True); repo = edited_repo('orphan-marked') + before = tree(os.path.join(repo, '.axiomcode')); said, q = everything(plugin, repo) + wrote = diff(before, tree(os.path.join(repo, '.axiomcode'))) + check("orphan: an install marked .orphaned_at writes nothing under .axiomcode (hooks, timer, refresher, query)", wrote == [] and not + os.path.exists(os.path.join(repo, 'builds')), wrote) + check("orphan: its session-start hook says it is an orphaned older version and to restart the agent session", + f'orphaned older axiomcode {VERSION}' in said and 'Restart the agent session' in said, said) + check("orphan: its query answers with the restart, not from a build of its own", 'orphaned older axiomcode' in q and 'rows' not in q, q) + # the MCP server: a tool call answers with the restart and writes nothing; started anew it refuses to start + code = ("import os, sys; sys.path.insert(0, os.path.join(sys.argv[1], 'mcp')); import server; " + "print(server.run(['impact', 'run', sys.argv[2]]))") + before = tree(os.path.join(repo, '.axiomcode')) + r = subprocess.run([sys.executable, '-c', code, plugin, repo], capture_output=True, text=True, cwd=repo, timeout=60, + env=dict(oenv, AXIOMCODE_PLUGIN_ROOT=plugin)) + check("orphan: its MCP server's tool call answers that it is an orphaned older version, running nothing", + 'orphaned older axiomcode' in r.stdout and 'Restart the agent session' in r.stdout and diff(before, tree(os.path.join(repo, '.axiomcode'))) == [], + (r.stdout, r.stderr[-400:])) + r = subprocess.run([sys.executable, os.path.join(plugin, 'mcp', 'server.py')], capture_output=True, text=True, cwd=repo, timeout=60, + stdin=subprocess.DEVNULL, env=dict(oenv, AXIOMCODE_PLUGIN_ROOT=plugin)) + check("orphan: its MCP server, started, says so and does not start, writing nothing", + r.returncode != 0 and 'orphaned older axiomcode' in r.stderr and 'Not starting' in r.stderr + and diff(before, tree(os.path.join(repo, '.axiomcode'))) == [], (r.returncode, r.stderr[-400:])) + + # the install directory deleted under a running process (the plugin manager's later step): nothing is written either + plugin = install('gone', mark=False); repo = edited_repo('orphan-gone') + scripts = os.path.join(plugin, 'skills', 'axiomcode', 'scripts') + before = tree(os.path.join(repo, '.axiomcode')) + code = ("import shutil, sys; sys.path.insert(0, sys.argv[1]); import ax_fresh; shutil.rmtree(sys.argv[3]); " + "print(ax_fresh.orphaned()); ax_fresh.hook_kick(sys.argv[2], 'the timer'); ax_fresh.kick(sys.argv[2]); ax_fresh.worker(sys.argv[2])") + r = subprocess.run([sys.executable, '-c', code, scripts, repo, plugin], capture_output=True, text=True, env=oenv, timeout=60) + time.sleep(1.0) + wrote = diff(before, tree(os.path.join(repo, '.axiomcode'))) + check("orphan: a running process whose install directory was deleted says so and writes nothing under .axiomcode", + 'is gone' in r.stdout and 'Restart the agent session' in r.stdout and wrote == [], (r.stdout, r.stderr[-400:], wrote)) + check("orphan: control: this checkout (no mark, files present) is not orphaned", ax_fresh.orphaned() == '', ax_fresh.orphaned()) + + +def main(): + work = tempfile.mkdtemp(prefix='axiomcode-two-versions-'); saved = os.environ.get('AXIOMCODE_ENGINE') + try: + e1 = fake_engine(os.path.join(work, 'e1'), version='1.0.0'); os.environ['AXIOMCODE_ENGINE'] = e1 + env = {k: v for k, v in os.environ.items() if k not in ('AXIOMCODE_NO_REFRESH', 'AXIOMCODE_FRESH', 'AXIOMCODE_GRAPH')} + env.update(AXIOMCODE_REFRESH_DEBOUNCE='0.05', AXIOMCODE_REFRESH_SLOTS=os.path.join(work, 'slots')) + for part in (turns_checks, newer_checks, orphan_checks): + try: part(work, env) + except Exception as e: # a missing helper is a failure of that part, not of the run + check(f"{part.__name__}: ran to the end", False, f"{type(e).__name__}: {e}") + finally: + if saved is None: os.environ.pop('AXIOMCODE_ENGINE', None) + else: os.environ['AXIOMCODE_ENGINE'] = saved + shutil.rmtree(work, ignore_errors=True) + bad = [n for n, ok in RESULTS if not ok] + print(f"\n{len(RESULTS) - len(bad)} passed, {len(bad)} failed") + return 1 if bad or not RESULTS else 0 + + +if __name__ == '__main__': + sys.exit(main())