From 9cdf41e02e6212279a5075df325cf963e925e896 Mon Sep 17 00:00:00 2001 From: swapnil Date: Sun, 11 Oct 2026 12:36:58 -0700 Subject: [PATCH 01/18] =?UTF-8?q?perf:=20faster=20graph=20export=20?= =?UTF-8?q?=E2=80=94=20bundle=20and=20impact=20facts,=20same=20answers=20(?= =?UTF-8?q?#1933)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bundle (raw relations + IR -> graph.sqlite): - the IR reader splits a quote-free line on tabs and keeps the character reader for lines a quote touches; rows the caller drops are skipped before they are split (call-site and expression passes) - keyed (WITHOUT ROWID) tables are loaded in key order (stable sort of a copy) - 16 KiB pages (AXIOM_BUNDLE_PAGE_SIZE overrides) - the c0 index only on ext relations the query layer reads (EXT_QUERIED); bundle-test fails when a script names one missing from it - AXIOM_BUNDLE_PROFILE=1 prints per-stage times Facts export (axiomcode-impact --warm): - const_values reads its whole-graph tables once, not once per const - gen_table finds each decorator's first declaration in one pass - Impact.code() copies runs of plain code whole - the route walk and stub_sites run once for both exports - config rows are filtered by a count of symbols rows instead of a join Every table row, every exported fact and every impact/path --json answer is byte-identical before and after on both measured projects. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- graph/bundle/build.ts | 34 ++++- graph/bundle/catalog.ts | 15 +++ graph/bundle/cli.ts | 6 +- graph/bundle/csv.ts | 127 ++++++++++++++---- graph/bundle/write.ts | 41 +++++- graph/test/tools/bundle-test.sh | 26 +++- .../axiomcode/scripts/ax_registration.py | 50 +++++-- .../skills/axiomcode/scripts/axiomcode-impact | 45 +++++-- .../skills/axiomcode/scripts/axiomcode-path | 9 +- 9 files changed, 294 insertions(+), 59 deletions(-) diff --git a/graph/bundle/build.ts b/graph/bundle/build.ts index 84c9fcb9e..d066ead75 100644 --- a/graph/bundle/build.ts +++ b/graph/bundle/build.ts @@ -10,9 +10,19 @@ import * as fs from 'fs'; import * as path from 'path'; -import { Header, readHeader, readRaw, readRfc4180 } from '@/bundle/csv'; +import { Header, type Pick, readHeader, readRaw, readRfc4180 } from '@/bundle/csv'; import type { LanguageAdapter, RawSource } from '@/bundle/languages'; +/** AXIOM_BUNDLE_PROFILE=1: one stderr line per stage with its wall time, for measuring the bundle. */ +const PROFILE = !!process.env.AXIOM_BUNDLE_PROFILE; +let lapAt = Date.now(); +export function lap(what: string): void { + if (!PROFILE) return; + const now = Date.now(); + process.stderr.write(` [bundle-profile] ${((now - lapAt) / 1000).toFixed(2)}s ${what}\n`); + lapAt = now; +} + export type Cell = string | number | null; export type Row = Cell[]; @@ -114,9 +124,10 @@ async function libSource(inp: BuildInputs, file: string): Promise { +async function* rowsOf(src: EntitySource, pick?: Pick): AsyncGenerator { let first = src.hasHeaderRow; - for await (const r of readRfc4180(src.path)) { + // the reader never filters a file's first row: here that is the header, and a headerless file is read whole + for await (const r of readRfc4180(src.path, src.hasHeaderRow ? pick : undefined)) { if (first) { first = false; continue; } yield r; } @@ -136,6 +147,7 @@ export async function buildCore(inp: BuildInputs): Promise { const rawFieldAccess = A.raw.fieldAccess ? await readSource(rawDir, A.raw.fieldAccess) : []; const rawTypeUse = A.raw.typeUse ? await readSource(rawDir, A.raw.typeUse) : []; const rawConfigBinding = A.raw.configBinding ? await readSource(rawDir, A.raw.configBinding) : []; + lap('read raw relations'); log(` raw: ${rawEdges.length} edge rows, ${rawAncestors.length} ancestor rows, ${rawOverrides.length} override rows`); // call_edges: split the raw ToMethod into a method key or a label by provenance @@ -179,6 +191,7 @@ export async function buildCore(inp: BuildInputs): Promise { return [ref!, nul(type), context!, int(depth) ?? 0, ownerKind!, owner!, nul(enclMethod), nul(enclType), nul(prov), tier!]; }); + lap('transform raw rows (call_edges, field_access, type_use)'); // ── 2. which ids the bundle must name ───────────────────────────────────── const wantMethods = new Set(); const wantTypes = new Set(); @@ -211,6 +224,7 @@ export async function buildCore(inp: BuildInputs): Promise { if (r[2] === 'field' && r[3] !== null) wantFields.add(r[3] as string); } + lap('collect wanted ids'); // ── 3. client entities (all of them — the client is the subject) ────────── // The IR spells visibility differently per entity file (PUBLIC on methods/types, PUBLIC_ACCESS on // fields); normalise to one vocabulary so a rule can match on it. '' when the language has none. @@ -367,6 +381,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } + lap('read client modules/methods/types/fields IR'); // ── 4. library entities — only the referenced ones ──────────────────────── const missingMethods = new Set([...wantMethods].filter((id) => !methods.has(id))); const missingTypesEarly = [...wantTypes].some((id) => !types.has(id)); @@ -447,6 +462,7 @@ export async function buildCore(inp: BuildInputs): Promise { if (FI?.ownerQualifiedNameFromType) fillOwnerNames(fields, 3, 4); // A library module's file path is not staged; a library site never occurs (edges start in the client). + lap('read library entities + generated/external'); // ── 5. call sites: name + position for every site the edges mention ─────── // row: id, caller_id, kind, callee_name, file_path, start_line, start_column, end_line, end_column const sites = new Map(); @@ -466,7 +482,7 @@ export async function buildCore(inp: BuildInputs): Promise { const h = src.header; const ce = h.col(C.expressionId), cn = h.col(C.calleeName), cl = h.col(C.startLine), cc = h.col(C.startColumn); const cel = C.endLine ? h.col(C.endLine) : -1; - for await (const r of rowsOf(src)) { + for await (const r of rowsOf(src, { column: ce, has: (id) => sites.has(id) })) { const row = sites.get(r[ce] ?? ''); if (!row) continue; fill(row, 3, nul(r[cn])); fill(row, 4, fileOf(C.fileVia, h, r)); @@ -475,6 +491,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } } + lap('position sites from call-site IR'); { const E = A.ir.expressions; const src = await clientSource(inp.clientIrDir, E.file); @@ -483,7 +500,7 @@ export async function buildCore(inp: BuildInputs): Promise { const ci = h.col(E.id), ck = h.col(E.kind), cl = h.col(E.startLine), cc = h.col(E.startColumn), cel = h.col(E.endLine), cec = h.col(E.endColumn); const cname = E.calleeName ? h.col(E.calleeName.column) : -1; const nameKinds = new Set(E.calleeName?.kinds ?? []); - for await (const r of rowsOf(src)) { + for await (const r of rowsOf(src, { column: ci, has: (id) => sites.has(id) || fieldSites.has(id) })) { const id = r[ci] ?? ''; // A FIELD ACCESS site is an expression too, and this is the one pass over a table that // can be GB-scale, so both are positioned here rather than streaming it twice. @@ -501,6 +518,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } } + lap('position sites from expressions IR'); if (A.ir.decorators) { const D = A.ir.decorators; const src = await clientSource(inp.clientIrDir, D.file); @@ -572,6 +590,7 @@ export async function buildCore(inp: BuildInputs): Promise { const unplaced = [...sites.values()].filter((r) => r[5] === null).length; if (unplaced > 0) log(` ! ${unplaced} of ${sites.size} call sites have no position in the IR`); + lap('position sites from decorators/locals/field sites + fallbacks'); // ── 5b. what the parser did NOT read ────────────────────────────────────── // A file the parser skipped has no row in any table above, and until #1148 nothing in the // bundle said so: a consumer could not tell a file with no calls from a file that was @@ -599,9 +618,10 @@ export async function buildCore(inp: BuildInputs): Promise { } if (skipped.length > 0) log(` skipped: ${skipped.length} file(s) the parser did not read`); + lap('read skipped'); // ── 6. assemble ─────────────────────────────────────────────────────────── const run: Row[] = Object.entries(inp.meta).map(([k, v]) => [k, v]); - return { + const core = { run, methods: [...methods.values()], types: [...types.values()], @@ -619,6 +639,8 @@ export async function buildCore(inp: BuildInputs): Promise { type_use, skipped: dedupe(skipped), }; + lap('assemble + dedupe'); + return core; } function dedupe(rows: Row[]): Row[] { diff --git a/graph/bundle/catalog.ts b/graph/bundle/catalog.ts index 3276e855b..9e216e7d6 100644 --- a/graph/bundle/catalog.ts +++ b/graph/bundle/catalog.ts @@ -16,6 +16,21 @@ export interface ExtRelation { description: string; } +/** + * The ext relations the query layer reads (plugins/, hooks/: impact, path, context, test-impact, the MCP server). + * Only these get the c0 index: an index on a relation nothing queries is time and disk at every build, and the + * schema promises the table, not an index on it. Not a filter on what is loaded — every manifest relation is still + * an ext table. graph/test/tools/ext-queried-test.sh fails when a script names an ext relation missing here. + */ +export const EXT_QUERIED: ReadonlySet = new Set([ + 'bean_def', 'call_runs_edge', 'call_site_unresolved', 'config_affects_method', 'config_binding', 'config_key_ref', + 'config_unresolved', 'ctor_implicit_type', 'decorated_name_target', 'di_edge', 'fn_value_call', 'framework_edge', + 'inject_point', 'lib_callback_edge', 'lib_callback_site', 'library_receiver', 'method_returns_method', + 'method_returns_param', 'persistence_field', 'remote_edge', 'remote_unsent', 'remote_unserved', 'return_value', + 'state_call_alloc', 'state_call_open', 'state_gate', 'state_gate_alloc', 'state_world_of_gated', + 'type_base_unresolved', 'unresolved_value_callee', +]); + /** relationfile per line; comments and blanks skipped. */ export function readManifest(langDir: string): { relation: string; file: string }[] { const p = path.join(langDir, 'souffle', 'export_manifest.tsv'); diff --git a/graph/bundle/cli.ts b/graph/bundle/cli.ts index ad975fad2..36a26c0c0 100644 --- a/graph/bundle/cli.ts +++ b/graph/bundle/cli.ts @@ -17,7 +17,7 @@ import * as path from 'path'; process.removeAllListeners('warning'); process.on('warning', (w) => { if (w.name !== 'ExperimentalWarning') console.error(w.stack ?? w.message); }); -import { buildCore } from '@/bundle/build'; +import { buildCore, lap } from '@/bundle/build'; import { catalogExt, readLibMap } from '@/bundle/catalog'; import { adapterFor } from '@/bundle/languages'; import { renderSchemaMarkdown, SCHEMA_VERSION } from '@/bundle/schema'; @@ -91,7 +91,9 @@ export async function main(argv = process.argv.slice(2)): Promise { const dbPath = path.join(a.out!, 'graph.sqlite'); if (haveSqlite) { - await writeSqlite({ dbPath, language: adapter.language, core, ext: catalogExt(langDir), rawDir: a.raw!, log }); + const ext = catalogExt(langDir); + lap('lift ext docs (catalogExt)'); + await writeSqlite({ dbPath, language: adapter.language, core, ext, rawDir: a.raw!, log }); log(`▶ wrote ${dbPath}${a.debug ? ' (+ csv/*.csv, --debug)' : ''}`); } else { fs.rmSync(dbPath, { force: true }); diff --git a/graph/bundle/csv.ts b/graph/bundle/csv.ts index 43fc7ef97..befee56b5 100644 --- a/graph/bundle/csv.ts +++ b/graph/bundle/csv.ts @@ -14,43 +14,120 @@ import * as fs from 'fs'; const TAB = '\t'; -/** One RFC 4180 (tab-delimited) file as an async stream of rows. A zero-byte file yields nothing. */ -export async function* readRfc4180(path: string): AsyncGenerator { +/** One RFC 4180 (tab-delimited) file as an async stream of rows. A zero-byte file yields nothing. + * + * A LINE WITH NO QUOTE IN IT IS A WHOLE ROW, and nearly every line is one, so it is split on tabs as it + * stands: a quote is the only thing that can make a field hold a tab or a newline. Only a line that holds + * a quote, and the lines a quoted field carries it across, goes through the character-by-character reader + * (`Rfc4180Row`), which is the definition. Reading every character that way cost most of the bundle's time + * on a GB-scale expression table. */ +export async function* readRfc4180(path: string, pick?: Pick): AsyncGenerator { const stream = fs.createReadStream(path, { encoding: 'utf8', highWaterMark: 1 << 20 }); - let field = ''; - let row: string[] = []; - let quoted = false; // inside a quoted field - let afterQuote = false; // just saw a quote inside a quoted field — next char decides - let atFieldStart = true; + const slow = new Rfc4180Row(); + let rest = ''; + let first = true; // the first row, a header in a headered file, is never filtered for await (const chunk of stream as AsyncIterable) { - for (let i = 0; i < chunk.length; i++) { + const text = rest.length > 0 ? rest + chunk : chunk; + let at = 0; + let quote = text.indexOf('"'); // the next quote at or after `at`, -1 when the chunk holds no more + for (;;) { + const nl = text.indexOf('\n', at); + if (nl < 0) break; + if (quote >= 0 && quote < at) quote = text.indexOf('"', at); + if (slow.open || (quote >= 0 && quote < nl)) { + // the line holds a quote, or a quoted field opened on an earlier line runs on into it + const row = slow.feed(text, at, nl + 1); + if (row) yield row; + } else if (first || !pick || pick.has(fieldAt(text, at, nl, pick.column))) { + yield splitLine(text, at, nl); + } + first = false; + at = nl + 1; + } + rest = text.slice(at); + } + // final row without a trailing newline + if (slow.open || rest.indexOf('"') >= 0) { + slow.feed(rest, 0, rest.length); + const row = slow.end(); + if (row) yield row; + } else if (rest.length > 0 && (first || !pick || pick.has(fieldAt(rest, 0, rest.length, pick.column)))) { + yield splitLine(rest, 0, rest.length); + } +} + +/** Rows the caller will drop, skipped before they are split: a row is read only when `has` holds for its + * `column`th field ('' when the row is shorter, as `row[column] ?? ''` reads it). A row a quote touches is + * always returned, so the caller must still test every row it gets; this only saves the split of the rest. */ +export interface Pick { column: number; has: (value: string) => boolean } + +/** The `col`th tab-separated field of text[a, b), a line holding no quote; a trailing \r on the last field dropped. */ +function fieldAt(text: string, a: number, b: number, col: number): string { + let start = a; + for (let i = 0; i < col; i++) { + const t = text.indexOf(TAB, start); + if (t < 0 || t >= b) return ''; + start = t + 1; + } + const t = text.indexOf(TAB, start); + if (t >= 0 && t < b) return text.slice(start, t); + const v = text.slice(start, b); + return v.endsWith('\r') ? v.slice(0, -1) : v; +} + +/** text[a, b) — one line holding no quote, without its newline — as a row: split on tabs, a trailing \r dropped. */ +function splitLine(text: string, a: number, b: number): string[] { + const line = text.slice(a, b); + const row = line.split(TAB); + const last = row[row.length - 1]!; + if (last.endsWith('\r')) row[row.length - 1] = last.slice(0, -1); + return row; +} + +/** The RFC 4180 reader, a character at a time: fed whole lines, it returns a row when one ends on a newline outside quotes. */ +class Rfc4180Row { + private field = ''; + private row: string[] = []; + private quoted = false; // inside a quoted field + private afterQuote = false; // just saw a quote inside a quoted field — next char decides + private atFieldStart = true; + /** a row has begun and not ended: a quoted field crossed a newline */ + get open(): boolean { return this.quoted || this.afterQuote || this.row.length > 0 || this.field.length > 0 || !this.atFieldStart; } + feed(chunk: string, from: number, to: number): string[] | null { + let out: string[] | null = null; + for (let i = from; i < to; i++) { const ch = chunk[i]!; - if (quoted) { - if (afterQuote) { - afterQuote = false; - if (ch === '"') { field += '"'; continue; } // doubled quote → literal quote - quoted = false; // the closing quote — fall through to unquoted handling + if (this.quoted) { + if (this.afterQuote) { + this.afterQuote = false; + if (ch === '"') { this.field += '"'; continue; } // doubled quote → literal quote + this.quoted = false; // the closing quote — fall through to unquoted handling } else { - if (ch === '"') afterQuote = true; else field += ch; + if (ch === '"') this.afterQuote = true; else this.field += ch; continue; } } - if (ch === '"' && atFieldStart) { quoted = true; atFieldStart = false; continue; } - atFieldStart = false; - if (ch === TAB) { row.push(field); field = ''; atFieldStart = true; continue; } + if (ch === '"' && this.atFieldStart) { this.quoted = true; this.atFieldStart = false; continue; } + this.atFieldStart = false; + if (ch === TAB) { this.row.push(this.field); this.field = ''; this.atFieldStart = true; continue; } if (ch === '\n') { - row.push(field.endsWith('\r') ? field.slice(0, -1) : field); - field = ''; atFieldStart = true; - yield row; row = []; + this.row.push(this.field.endsWith('\r') ? this.field.slice(0, -1) : this.field); + this.field = ''; this.atFieldStart = true; + out = this.row; this.row = []; continue; } - field += ch; + this.field += ch; } + return out; } - // final row without a trailing newline - if (field.length > 0 || row.length > 0) { - row.push(field.endsWith('\r') ? field.slice(0, -1) : field); - yield row; + /** end of input: the row in progress, if any */ + end(): string[] | null { + if (this.field.length > 0 || this.row.length > 0) { + this.row.push(this.field.endsWith('\r') ? this.field.slice(0, -1) : this.field); + const r = this.row; this.row = []; this.field = ''; + return r; + } + return null; } } diff --git a/graph/bundle/write.ts b/graph/bundle/write.ts index d00899fee..6898777b5 100644 --- a/graph/bundle/write.ts +++ b/graph/bundle/write.ts @@ -7,8 +7,8 @@ import * as fs from 'fs'; import * as path from 'path'; import { readRaw, writeTable } from '@/bundle/csv'; -import type { CoreTables, Row } from '@/bundle/build'; -import type { ExtRelation } from '@/bundle/catalog'; +import { lap, type CoreTables, type Row } from '@/bundle/build'; +import { EXT_QUERIED, type ExtRelation } from '@/bundle/catalog'; import { CATALOG_TABLES, CORE_TABLES, GUIDE, LANGUAGES, NOTES, QUERIES, SCHEMA_VERSION, VOCAB, type Language, type TableSpec } from '@/bundle/schema'; export function writeCoreCsv(graphDir: string, core: CoreTables, log: (s: string) => void): void { @@ -48,6 +48,7 @@ export interface SqliteInputs { // VALUES list amortizes both; 128 rows per statement keeps the flattened argument array well // under sqlite's bind limit at the widest table. const BATCH = Math.max(1, Number(process.env.AXIOM_BUNDLE_BATCH ?? 128) || 128); +const PAGE_SIZE = [4096, 8192, 16384, 32768, 65536].includes(Number(process.env.AXIOM_BUNDLE_PAGE_SIZE)) ? Number(process.env.AXIOM_BUNDLE_PAGE_SIZE) : 16384; type DbLike = { prepare(sql: string): { run(...a: (string | number | null)[]): unknown } }; function batchInsert(db: DbLike, table: string, arity: number, verb: string): { push(r: (string | number | null)[]): void; flush(): void } { const one = `(${Array.from({ length: arity }, () => '?').join(', ')})`; @@ -72,12 +73,35 @@ function batchInsert(db: DbLike, table: string, arity: number, verb: string): { }; } +/** + * A keyed table's rows in key order, for loading. A WITHOUT ROWID table is stored in key order whatever order its + * rows arrive in, so this changes nothing stored; but rows arriving in hash order land all over the B-tree and + * split its pages again and again, while rows in key order are appended. The sort is stable, so of two rows with + * one key the first is still the one INSERT OR IGNORE keeps. A copy is sorted: the caller's array is not reordered. + */ +function inKeyOrder(t: TableSpec, rows: Row[]): Row[] { + const keys = t.columns.flatMap((c, i) => (c.key ? [i] : [])); + if (keys.length === 0 || rows.length < 2) return rows; + const str = (v: Row[number] | undefined) => (v === null || v === undefined ? '' : String(v)); + return rows.slice().sort((a, b) => { + for (const i of keys) { + const x = str(a[i]), y = str(b[i]); + if (x !== y) return x < y ? -1 : 1; + } + return 0; + }); +} + export async function writeSqlite(inp: SqliteInputs): Promise { // Loaded lazily so the CSV path works on a Node without the module. const { DatabaseSync } = await import('node:sqlite'); fs.rmSync(inp.dbPath, { force: true }); fs.rmSync(inp.dbPath + '-journal', { force: true }); const db = new DatabaseSync(inp.dbPath); + // 16 KiB PAGES, set before the first table (AXIOM_BUNDLE_PAGE_SIZE overrides). The bundle is a bulk load of a file + // that reaches GBs, and at SQLite's default 4 KiB every table and index B-tree has four times the pages to split + // and write; the queries over the graph read it no slower. The page size changes no row and no answer. + db.exec(`PRAGMA page_size = ${PAGE_SIZE};`); db.exec('PRAGMA journal_mode = OFF; PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;'); try { // ── core ──────────────────────────────────────────────────────────────── @@ -86,11 +110,13 @@ export async function writeSqlite(inp: SqliteInputs): Promise { db.exec(createSql(t)); const rows = (inp.core as unknown as Record)[t.name] ?? []; const ins = batchInsert(db, t.name, t.columns.length, 'INSERT OR IGNORE'); - for (const r of rows) ins.push(r as (string | number | null)[]); + for (const r of inKeyOrder(t, rows)) ins.push(r as (string | number | null)[]); ins.flush(); inp.log(` sqlite ${t.name}: ${rows.length} rows`); + lap(`insert ${t.name} (${rows.length} rows)`); } db.exec('COMMIT;'); + lap('commit core'); // ── ext ───────────────────────────────────────────────────────────────── db.exec('BEGIN;'); @@ -107,8 +133,10 @@ export async function writeSqlite(inp: SqliteInputs): Promise { } ins.flush(); if (n > 0) inp.log(` sqlite ${name}: ${n} rows`); + lap(`insert ${name} (${n} rows)`); } db.exec('COMMIT;'); + lap('commit ext'); // ── catalog ───────────────────────────────────────────────────────────── db.exec('BEGIN;'); @@ -165,14 +193,17 @@ export async function writeSqlite(inp: SqliteInputs): Promise { const query = db.prepare('INSERT INTO schema_queries VALUES (?, ?, ?, ?)'); for (const q of QUERIES) query.run(q.name, q.question, q.params, q.sql); db.exec('COMMIT;'); + lap('catalog + vocab'); // ── indexes, after the load ───────────────────────────────────────────── db.exec('BEGIN;'); for (const t of CORE_TABLES) { - for (const cc of t.columns) if (cc.indexed) db.exec(`CREATE INDEX idx_${t.name}_${cc.name} ON ${t.name}(${cc.name});`); + for (const cc of t.columns) if (cc.indexed) { db.exec(`CREATE INDEX idx_${t.name}_${cc.name} ON ${t.name}(${cc.name});`); lap(`index ${t.name}.${cc.name}`); } } - for (const e of inp.ext) if (e.arity > 0) db.exec(`CREATE INDEX idx_ext_${e.relation}_c0 ON ext_${e.relation}(c0);`); + // an ext table the tools query keeps its c0 index; the rest are loaded and left unindexed (catalog.ts EXT_QUERIED) + for (const e of inp.ext) if (e.arity > 0 && EXT_QUERIED.has(e.relation)) { db.exec(`CREATE INDEX idx_ext_${e.relation}_c0 ON ext_${e.relation}(c0);`); lap(`index ext_${e.relation}.c0`); } db.exec('COMMIT;'); + lap('commit indexes'); db.exec(`PRAGMA user_version = ${Number(SCHEMA_VERSION)};`); } finally { db.close(); diff --git a/graph/test/tools/bundle-test.sh b/graph/test/tools/bundle-test.sh index 06ead93d8..8f3245bdf 100755 --- a/graph/test/tools/bundle-test.sh +++ b/graph/test/tools/bundle-test.sh @@ -310,6 +310,30 @@ if ! diff -q <(BUNDLE --print-schema) "$ROOT/graph/bundle/SCHEMA.md" >/dev/null; bad "graph/bundle/SCHEMA.md is stale — run: npm run schema-doc" fi +# 8. every ext relation a script queries is in EXT_QUERIED, so it keeps its c0 index (catalog.ts). The others are +# loaded unindexed: a new query on one of them would still answer, only slower, which no other check would see. +missing="$(python3 - "$ROOT" <<'PY' +import os, re, sys +root = sys.argv[1] +rels = set() +for lang in os.listdir(os.path.join(root, 'graph')): + m = os.path.join(root, 'graph', lang, 'souffle', 'export_manifest.tsv') + if os.path.isfile(m): + rels |= {l.split('\t')[0].strip() for l in open(m) if l.strip() and not l.lstrip().startswith('#')} +cat = open(os.path.join(root, 'graph', 'bundle', 'catalog.ts')).read() +queried = set(re.findall(r"'(\w+)'", cat[cat.index('EXT_QUERIED'):cat.index(']);', cat.index('EXT_QUERIED'))])) +used = set() +for d in ('plugins', 'hooks', 'bin'): + for dp, dns, fns in os.walk(os.path.join(root, d)): + dns[:] = [x for x in dns if x not in ('node_modules', 'dist', '.git')] + for fn in fns: + try: text = open(os.path.join(dp, fn), errors='replace').read() + except OSError: continue + used |= {r for r in re.findall(r'\bext_(\w+)', text) if r in rels} +print(' '.join(sorted(used - queried))) +PY +)" +[ -z "$missing" ] || bad "a script queries ext relation(s) missing from EXT_QUERIED in graph/bundle/catalog.ts (no c0 index): $missing" # ── the DEFAULT writes the database and nothing else ──────────────────────── # The deliverable is graph.sqlite. csv/*.csv carries the same core tables, so writing @@ -323,7 +347,7 @@ if [ -d "$d/csv" ] && [ -n "$(ls -A "$d/csv" 2>/dev/null)" ]; then fi if [ "$fail" -eq 0 ]; then - echo "bundle: ok (3 languages$( [ "$HAVE_SQLITE" = 1 ] && echo ', one schema across them, canonical queries run' || echo ', csv only — no sqlite3 CLI'), schema doc current)" + echo "bundle: ok (3 languages$( [ "$HAVE_SQLITE" = 1 ] && echo ', one schema across them, canonical queries run' || echo ', csv only — no sqlite3 CLI'), schema doc current, queried ext relations indexed)" else echo "bundle: $fail failure(s)"; exit 1 fi diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py index 0167e2c42..299232f86 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py @@ -49,7 +49,7 @@ MAX_SITE_SPAN = 200 # a "call" spanning a whole file is a parse artefact, not a registration -def registrations(q, site_file=None): +def registrations(q, site_file=None, with_own_lines=False): """[(file, line, kind, key, why)] — every line that hands a declaration to something outside the graph. `kind` is always "route"; `key` is the string the framework dispatches on, VERBATIM and never normalised @@ -57,9 +57,11 @@ def registrations(q, site_file=None): client (`supertest.get('/orders/1')`, `TestClient(app).post('/orders')`) be joined to the handler it reaches: the two ends spell the same string and nothing else connects them. Normalising it belongs in the join, not here — `:id`, `{order_id}` and `` are three frameworks' spellings of one hole. + + `with_own_lines`: (rows, route_site_lines(q, site_file)) — the same walk computes both. """ if not _has(q, 'call_sites'): - return [] + return ([], {}) if with_own_lines else [] sf = site_file or (lambda x: x) lits = {} if _has(q, 'literals'): @@ -83,7 +85,7 @@ def registrations(q, site_file=None): # the cap and joins two unrelated tests. The DEPENDENT row is still true and still printed — a test that mounts a # handler does depend on it — so only the key is withheld. tf = {f for (f,) in q("SELECT DISTINCT file FROM symbols WHERE is_test = 1 AND file IS NOT NULL")} if _has(q, 'symbols') else set() - out, verbs, _own = _route_links(q, sf, lits, tf) + out, verbs, own = _route_links(q, sf, lits, tf) # `own` does not depend on tf, which only withholds a key # `router.route('/').get(h)` is two route-shaped calls on one line, and `route` said nothing about the method: the # line's HTTP verbs name it, in the order written. A chain registering two (`.get(a).post(b)`) names both, because # a line is all a reference records and it cannot say which argument list the handler sat in @@ -92,7 +94,8 @@ def registrations(q, site_file=None): if k == 'route' and vs: m = re.match(r'registered as a \w+ route (".*?") here', w) if m: out[(f, l)] = (k, key, f"registered as a {'/'.join(sorted(vs, key=vs.get))} route {m.group(1)} here — the router calls it, no call site does") - return sorted((ref_at.get((f, l), ''), f, l, k, key, w) for (f, l), (k, key, w) in out.items()) + rows = sorted((ref_at.get((f, l), ''), f, l, k, key, w) for (f, l), (k, key, w) in out.items()) + return (rows, own) if with_own_lines else rows def route_site_lines(q, site_file=None): @@ -289,6 +292,7 @@ class _Handlers: def __init__(self, q, read, sf): self.q, self.read, self.sf, self.names, self.consts, self.held = q, read, sf, {}, {}, {} + self.tables = None # const_value_tables, read on the first const asked about if _has(q, 'symbols'): for n, k, f, l in q("SELECT name, kind, file, line FROM symbols WHERE name IS NOT NULL"): self.names.setdefault(n, {}).setdefault(k or '', set()).add(f) @@ -310,7 +314,8 @@ def holds_fn(self, n, files=None): # a module a const requires may export the handler itself: `const h = require('./h'); app.get('/x', h)` required = any(re.search(rf'(? 0"""): callee_at.setdefault((sf(fp), l, col), set()).add(m) + return fns, rets, callee_at + + +def const_values(q, code, decls, site_file=None, tables=None): + """For each (file, line, name) const declaration: ({(file, line)} that hold a function, {(file, line, wrapper)} the + client callable its initializer calls, {(file, line, name)} the function it is an alias of). `tables` is + const_value_tables(q, site_file), when the caller already has it.""" + callable_, wrap, alias = set(), set(), set() + if not decls or code is None: + return callable_, wrap, alias + fns, rets, callee_at = tables if tables is not None else const_value_tables(q, site_file) def call_value(text, i, f, a, starts, depth=0): """(callable?, wrappers) for the expression at text[i:]""" diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index 48079480f..be9e94efc 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -256,8 +256,17 @@ class Impact: triple = py or file.endswith(('.java', '.kt', '.scala', '.groovy')) # a Java text block is """ too js = file.endswith(('.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx')) prev = '' # the last code character seen + # only these characters can open a comment, a string or a regex literal: a run of any others is code, copied whole + special = self._CODE_SPECIAL_PY if py else self._CODE_SPECIAL while i < n: c = t[i] + if c not in '/"\'`#': + m = special.search(t, i); j = m.start() if m else n + out.append(t[i:j]) + k = j - 1 # `prev`: the run's last non-space, or its last newline + while k >= i and t[k].isspace() and t[k] != '\n': k -= 1 + if k >= i: prev = t[k] if not t[k].isspace() else '\n' + i = j; continue if js and c == '/' and i + 1 < n and t[i + 1] not in '/*' and prev in '(,=:[!&|?{};+-*%~^<>' + '\n': j = i + 1 # a regex literal: /…/flags, never a division while j < n and t[j] not in '/\n': @@ -284,8 +293,9 @@ class Impact: if not c.isspace(): prev = c elif c == '\n': prev = '\n' i += 1; continue - out.append(''.join('\n' if ch == '\n' else ' ' for ch in t[i:j])); prev = 'x'; i = j + out.append(self._BLANK.sub(' ', t[i:j])); prev = 'x'; i = j self.src[('code', file)] = ''.join(out).split('\n'); return self.src[('code', file)] + _CODE_SPECIAL = re.compile(r'[/"\'`]'); _CODE_SPECIAL_PY = re.compile(r'[/"\'`#]'); _BLANK = re.compile(r'[^\n]') # ── targets ────────────────────────────────────────────────────────────────────────────────────────────────── @@ -1835,13 +1845,18 @@ class Impact: # `Accessors = ...`): if the declaration's own source names a generator, it generates the same members gen_table = {d: set(ws) for d, ws in GENERATED.items()} used = {r[1].split('.')[-1] for r in g.q("SELECT owner_id, name FROM decorations")} if g.has('decorations') else set() - for d in used - set(gen_table): + # the first declaration of each such name with a file and a line, in the symbol table's order: found in one pass + # rather than one pass over every symbol per decorator name + wanted = used - set(gen_table); first_decl = {} + if wanted: for i, sy in g.sym.items(): - if sy['name'] != d or not sy.get('file') or not sy.get('line'): continue + if sy['name'] in wanted and sy['name'] not in first_decl and sy.get('file') and sy.get('line'): first_decl[sy['name']] = sy + for d in wanted: + sy = first_decl.get(d) + if sy is not None: body = '\n'.join(self.code(sy['file'])[sy['line'] - 1: (sy['end_line'] or sy['line']) + 1]) hit = {w for k, ws in GENERATED.items() if re.search(rf'\b{re.escape(k)}\b', body) for w in ws} if hit: gen_table[d] = hit - break # A BASE CLASS BUILT BY A FACTORY: `Base = declarative_base()`, then `class Customer(Base)`. The written # base name is `Base`, which discriminates nothing, and the alias rule cannot help because an assignment # is not a declaration with a body to read. What identifies it is the assignment itself, in the source the @@ -1855,6 +1870,7 @@ class Impact: pat = re.compile(rf'^\s*{re.escape(b)}\s*(?::[^=]+)?=\s*(\w+)\s*\(') for (fl,) in g.q("SELECT DISTINCT file FROM type_refs WHERE name = ? AND file IS NOT NULL", b): for ln in (self.code(fl) or []): + if b not in ln: continue # the pattern starts with the name itself m = pat.match(ln) if m and m.group(1) in GENERATED: gen_table[b] = set(GENERATED[m.group(1)]); break @@ -1869,9 +1885,10 @@ class Impact: W('cert_tier', sorted({(t, ax_edges.direct_cert(t), ax_edges.direct_why(t)) for t in {r[0] for r in g.q("SELECT DISTINCT tier FROM call_edges")} | {ax_edges.STUB_TIER, 'overload'}})) # a call written inside a mock's stub or verification keeps its row and loses its tier: "stub" (ax_edges.stub_sites) - stubs = ax_edges.stub_sites(lambda s, p: g.q(s, *p)) if g.has('call_sites') else set() + stubs = g.stub_sites() # a chained route link's edge sits on the line of its own verb and path, where `registration` labels it - link_line = ax_registration.route_site_lines(g.q, g.site_file) + # the registrations come out of the same walk over the route calls, so they are kept for `registration` below + regs, link_line = ax_registration.registrations(g.q, g.site_file, with_own_lines=True) # LEADS A LINK REJECTED (ax_links.py): a site's by-name match or set member someone read and found wrong is not walked; # the engine's row stays in graph.sqlite. `rejected` feeds the by-name rules, the set members leave `calls` here. import ax_links @@ -1970,7 +1987,7 @@ class Impact: L = self.code(r['file']); text = L[r['line'] - 1] if r['line'] <= len(L) else '' for qn in re.findall(rf'([A-Za-z_$][\w$]*)\s*\.\s*{re.escape(r["name"])}\b', text): quals.append((r['file'], r['line'], r['name'], qn)) W('ref', refs); W('qualifier', sorted(set(quals))) - regs = ax_registration.registrations(g.q, g.site_file) # computed once: reg_key_fact below reads it too + # `regs`, computed once above with link_line: reg_key_fact below reads it too W('registration', regs) trs = []; trf = set() # a reference on a module-level type alias's own lines belongs to the alias, not the module (#784) @@ -2331,7 +2348,19 @@ class Impact: # These are edges no call site carries — without them a change to a key or a bean has no visible consumer at all. cfg = []; keys = set(); beans = []; inj = []; factories = [] if g.has('ext_config_affects_method'): - for r in g.q("SELECT a.c0, a.c1, a.c2 FROM ext_config_affects_method a JOIN symbols s ON s.method_id = a.c1"): cfg.append((ckey(r[0]), r[1], r[2] or 'bound')) + # the rows whose method the index names, once per symbol row that names it: the join + # `ext_config_affects_method a JOIN symbols s ON s.method_id = a.c1`, in a's order, without probing the + # symbols index once per row (millions of rows over a few keys) + per_method = collections.Counter(m for (m,) in g.q("SELECT method_id FROM symbols WHERE method_id IS NOT NULL")) + ck = {} + for r in g.q("SELECT c0, c1, c2 FROM ext_config_affects_method"): + n = per_method.get(r[1], 0) + if not n: continue + k = ck.get(r[0]) + if k is None: k = ck[r[0]] = ckey(r[0]) + row = (k, r[1], r[2] or 'bound') + cfg.append(row) + for _ in range(n - 1): cfg.append(row) keys |= {ckey(r[0]) for r in g.q("SELECT DISTINCT c0 FROM ext_config_affects_method")} for t in ('ext_config_binding', 'ext_config_key_ref', 'ext_config_unresolved'): if g.has(t): keys |= {ckey(r[0]) for r in g.q(f"SELECT DISTINCT c0 FROM {t}")} diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path index 8a79bdd54..741164a7f 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path @@ -293,6 +293,13 @@ class G: self.con.executemany("INSERT INTO decorations VALUES (?,?,?,?,?)", out) self.decorations_synthesised = len(out) def q(self, s, *p): return self.con.execute(s, p).fetchall() + def stub_sites(self): + """ax_edges.stub_sites of this graph, read once per connection: both exports (path's and impact's) ask for it. + The set is shared — a caller must not change it.""" + if getattr(self, '_stubs_con', None) is not self.con: + self._stubs = ax_edges.stub_sites(lambda s, p: self.q(s, *p)) if self.has('call_sites') else set() + self._stubs_con = self.con + return self._stubs def has(self, t): return bool(self.q("SELECT 1 FROM sqlite_master WHERE name=?", t)) def loc(self, i): r = self.sym.get(i) or self.all_sym.get(i) @@ -1050,7 +1057,7 @@ class G: # …except a call written inside a mocking library's stub or verification (`when(repo.find(1))`, # `mock.Setup(r => r.Find(1))`): the receiver is a mock, so the real body never runs there, and walked as a call # it made every test that stubs a method a `sound` route to it. impact lists those sites as "stubs it". - stubs = ax_edges.stub_sites(lambda s, p: self.q(s, *p)) if self.has('call_sites') else set() + stubs = self.stub_sites() # leads an asserted-links rejection takes out (ax_links.suppressed): a set member every site of which was rejected import ax_links _rej, rej_edges = ax_links.suppressed(self.q, self.site_file) From 25af282851b85d63d67570249ce36b38a5e7a3e5 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:41:00 -0700 Subject: [PATCH 02/18] release 0.2.1: every package and plugin manifest at 0.2.1 (the parser keeps its own version) --- gemini-extension.json | 2 +- package.json | 12 ++++++------ plugins/axiomcode/.claude-plugin/plugin.json | 2 +- plugins/axiomcode/.codex-plugin/plugin.json | 2 +- plugins/axiomcode/.cursor-plugin/plugin.json | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gemini-extension.json b/gemini-extension.json index 4a44f6eae..dad925927 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.2.0", + "version": "0.2.1", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", "contextFileName": "plugins/axiomcode/AGENTS.md", "mcpServers": { diff --git a/package.json b/package.json index eac44e3dc..17f29b528 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@axiomcode/code-graph", - "version": "0.2.0", + "version": "0.2.1", "description": "AxiomCode Graph: a resolved call graph of your codebase grounded in formal methods, so you and your coding agents can see who calls what, what a change breaks, and which tests it reaches.", "repository": { "type": "git", @@ -31,11 +31,11 @@ ], "license": "FSL-1.1-Apache-2.0", "optionalDependencies": { - "@axiomcode/engine-darwin-arm64": "0.2.0", - "@axiomcode/engine-darwin-x64": "0.2.0", - "@axiomcode/engine-linux-x64": "0.2.0", - "@axiomcode/engine-linux-arm64": "0.2.0", - "@axiomcode/engine-win32-x64": "0.2.0" + "@axiomcode/engine-darwin-arm64": "0.2.1", + "@axiomcode/engine-darwin-x64": "0.2.1", + "@axiomcode/engine-linux-x64": "0.2.1", + "@axiomcode/engine-linux-arm64": "0.2.1", + "@axiomcode/engine-win32-x64": "0.2.1" }, "bin": { "axiomcode": "bin/axiomcode.js" diff --git a/plugins/axiomcode/.claude-plugin/plugin.json b/plugins/axiomcode/.claude-plugin/plugin.json index 917704f8c..7e4ba77b7 100644 --- a/plugins/axiomcode/.claude-plugin/plugin.json +++ b/plugins/axiomcode/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "axiomcode", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", - "version": "0.2.0", + "version": "0.2.1", "author": { "name": "AxiomCode" } diff --git a/plugins/axiomcode/.codex-plugin/plugin.json b/plugins/axiomcode/.codex-plugin/plugin.json index 82d1aa414..f7780f510 100644 --- a/plugins/axiomcode/.codex-plugin/plugin.json +++ b/plugins/axiomcode/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.2.0", + "version": "0.2.1", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", "author": { "name": "AxiomCode", "url": "https://github.com/AxiomCodeAI/axiomcodegraph" }, "homepage": "https://github.com/AxiomCodeAI/axiomcodegraph", diff --git a/plugins/axiomcode/.cursor-plugin/plugin.json b/plugins/axiomcode/.cursor-plugin/plugin.json index 06402050e..ce3908644 100644 --- a/plugins/axiomcode/.cursor-plugin/plugin.json +++ b/plugins/axiomcode/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.2.0", + "version": "0.2.1", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed. Java, TypeScript, Python, JavaScript.", "author": { "name": "AxiomCode" }, "homepage": "https://github.com/AxiomCodeAI/axiomcodegraph", From da61e3f130f6b186e7cb93c889d4a606a8cf7e63 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:11:09 -0700 Subject: [PATCH 03/18] plugin: two axiomcode versions over one graph rebuild once, say so, and an orphan writes nothing An agent session started before an engine or plugin upgrade keeps the old version's MCP server and hooks running beside the new ones. The old one rebuilt the graph with its engine and the new one rebuilt it back, a full rebuild on every turn (reproduced: 3 rebuilds in 3 turns against a 0.1.9 build). The answers only said the graph was stale. And the old process recreated .axiomcode (refresh.log, refresh.json) while the user deleted it. - ax_orphan.orphaned(): one check for a plugin directory marked .orphaned_at, or one whose scripts are gone. When it fires, the refresher, hooks, timer, query, relink and write_state write nothing. The MCP server answers with "restart the agent session", stops its timer and refuses to start. - ax_fresh.displace/contested: a build that replaces another axiomcode's graph records who built that graph. If the same builder rebuilds it again later, this version stops rebuilding it in the background and says two versions are running. `axiomcode index` still rebuilds it. - The newer-graph note and the `newer` command now name the two-versions case and say to restart the session. - README: restart the agent session after upgrading the engine or the plugin. Test: tests/two_versions.py (fails 14 of 19 on the base, passes 19 of 19 now). Refs #1835 --- README.md | 3 + plugins/axiomcode/hooks/_host.py | 16 +- plugins/axiomcode/hooks/refresh.py | 2 +- plugins/axiomcode/mcp/server.py | 22 ++ .../skills/axiomcode/scripts/ax_fresh.py | 104 +++++++- .../skills/axiomcode/scripts/ax_orphan.py | 46 ++++ tests/README.md | 5 + tests/two_versions.py | 244 ++++++++++++++++++ 8 files changed, 428 insertions(+), 14 deletions(-) create mode 100644 plugins/axiomcode/skills/axiomcode/scripts/ax_orphan.py create mode 100644 tests/two_versions.py diff --git a/README.md b/README.md index ba0d1d453..eab92e7f4 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,9 @@ Any other agent that speaks MCP takes one entry in its MCP config; see [Support for agents](#support-for-agents). Start a new agent session afterward: plugins are loaded at startup. Add `.axiomcode/` to your `.gitignore`; the graph is built there on first use. +After upgrading the engine or the plugin, **restart the agent session**: a session started before the upgrade keeps the +old version's MCP server and hooks running beside the new ones, and the two rebuild the graph in turn. + ### Uninstallation ```bash diff --git a/plugins/axiomcode/hooks/_host.py b/plugins/axiomcode/hooks/_host.py index e133112a2..5ae1b7335 100644 --- a/plugins/axiomcode/hooks/_host.py +++ b/plugins/axiomcode/hooks/_host.py @@ -34,10 +34,22 @@ GEMINI = None # the Gemini event name this hook was called for, once read() has seen one -def read(): - """the event from stdin, with Cursor's or Gemini's names mapped onto the ones the hooks use""" +def orphaned(): + """ax_orphan.orphaned for the install these hooks belong to: '' while it is current""" + try: + sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'skills', 'axiomcode', 'scripts')) + import ax_orphan + return ax_orphan.orphaned() + except Exception: + return '' + +def read(orphan_ok=False): + """the event from stdin, with Cursor's or Gemini's names mapped onto the ones the hooks use. A hook of an ORPHANED + older install (ax_orphan.orphaned: an agent session that outlived an upgrade) exits here, silent, so it writes nothing + into a repository's .axiomcode; only the refresh hook (orphan_ok) goes on, to say at a session start and on a prompt that the session wants a restart""" global GEMINI ev = json.load(sys.stdin) + if not orphan_ok and orphaned(): sys.exit(0) if isinstance(ev, dict) and ev.get('hook_event_name') in GEMINI_EVENTS: GEMINI = ev['hook_event_name'] ev['hook_event_name'] = GEMINI_EVENTS[GEMINI] diff --git a/plugins/axiomcode/hooks/refresh.py b/plugins/axiomcode/hooks/refresh.py index e5798d22d..87bf297e5 100644 --- a/plugins/axiomcode/hooks/refresh.py +++ b/plugins/axiomcode/hooks/refresh.py @@ -14,7 +14,7 @@ import _host, _where try: - ev = _host.read() + ev = _host.read(orphan_ok=True) # an orphaned install's hook only says so (ax_fresh.hook_kick) except Exception: sys.exit(0) cwd = ev.get('cwd') or os.getcwd() diff --git a/plugins/axiomcode/mcp/server.py b/plugins/axiomcode/mcp/server.py index 27c4c3574..b1ebc5e62 100755 --- a/plugins/axiomcode/mcp/server.py +++ b/plugins/axiomcode/mcp/server.py @@ -164,10 +164,26 @@ async def call_tool(self, name, arguments, *a, **k): # the SDK: refused as a # the flags whose next argument is their value, as the dispatcher skips them when it looks for the repository VALUED = {'--in', '--from', '--budget', '--seeds', '--depth', '--limit', '--tests-in', '--kind', '--range', '--old', '--new', '--file', '--page', '--page-budget', '--out'} +def orphaned(root=None): + """ax_orphan.orphaned for the install this server runs now (plugin_root): '' while it is current. An install whose + scripts are gone cannot be imported, so that is said here""" + root = root or plugin_root() + try: return scripts_module('ax_orphan').orphaned(root) + except Exception: + if os.path.isfile(os.path.join(root, 'skills', 'axiomcode', 'scripts', 'ax_fresh.py')): return '' + return (f"this is an orphaned older axiomcode (its install directory {root} is gone: a newer version was installed in " + "its place), still running in an agent session started before the upgrade; it writes nothing into the " + "repository's .axiomcode and rebuilds nothing. Restart the agent session so that only the installed axiomcode runs") + def _timer(interval): import time while True: time.sleep(max(1.0, min(60.0, interval / 3))) + # AN ORPHANED SERVER STOPS (ax_orphan.orphaned): its install was replaced and no newer one is recorded to run in its + # place, so all it could still do is rebuild graphs with the older version, the turn-taking an upgrade started + o = orphaned() + if o: + sys.stderr.write(f"axiomcode mcp: {o}. This server stops.\n"); sys.stderr.flush(); os._exit(0) for repo in list(SEEN): try: fresh = scripts_module('ax_fresh') @@ -183,6 +199,9 @@ def run(args, cwd=None, timeout=900): env = dict(os.environ, AXIOMCODE_BUILD_NOWAIT='1', AXIOMCODE_SURFACE='mcp') if args and args[0] != 'index' else None root = plugin_root() ax = os.path.join(root, 'skills', 'axiomcode', 'scripts', 'axiomcode') + # an orphaned older install answers with the restart and runs nothing: its verbs would write into .axiomcode + o = orphaned(root) + if o: return f"axiomcode: {o}." note = stale_note(root) head = (note + '\n') if note else '' try: @@ -272,6 +291,9 @@ def tests() -> str: if __name__ == '__main__': # catch up on whatever changed while no session was running (#1305): started, never waited on + o = orphaned() + if o: + sys.stderr.write(f"axiomcode mcp: {o}. Not starting.\n"); sys.exit(1) try: scripts_module('ax_fresh').hook_kick(os.getcwd(), 'the MCP server starting') if os.path.isdir(os.path.join(os.getcwd(), '.axiomcode')): SEEN.add(os.path.realpath(os.getcwd())) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py index e1fc01047..1958b7e3a 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py @@ -652,10 +652,14 @@ def newer_build(repo, t=None): IMPACT_VERSION is not re-exported either (rewarm): that would record the older version over the newer one's""" return built_by_state(repo, t)[1] +TWO_VERSIONS = ("two axiomcode versions are running over this repository (an agent session started before an upgrade keeps " + "the older version's hooks and MCP server); restart the agent session after upgrading the engine or the plugin") + def newer_note(n): - """what an answer from a graph a newer axiomcode built says""" - return (f"graph refresh: {n}; answering from it as is, and this older axiomcode does not rebuild it; rows in files " - "edited since are marked. Update this axiomcode, or `AXIOMCODE_REINDEX=1 axiomcode index` rebuilds it with this one") + """what an answer from a graph a newer axiomcode built says: most often the older of TWO VERSIONS RUNNING at once (an + agent session that outlived an upgrade), which a restart ends, not a stale graph""" + return (f"graph refresh: {n}: {TWO_VERSIONS}. This older axiomcode answers from that graph as it is and does not rebuild " + "it; rows in files edited since are marked. Update it, or `AXIOMCODE_REINDEX=1 axiomcode index` rebuilds the graph with this one") def engine_change(repo, t=None): """'' when the graph was built by the axiomcode that would build it now, else what differs, written @@ -676,6 +680,54 @@ def export_behind(t): if _vnum(have) and _vnum(now) and _vnum(have) > _vnum(now): return '' return have +# ── TWO VERSIONS TAKING TURNS: REBUILT ONCE, NOT ON EVERY TURN ───────────────────────────────────────────────────────── +# NEVER A DOWNGRADE (newer_build) holds only when the older axiomcode knows it: one from before it (or one whose version +# numbers tie with this one's) rebuilds the graph with its own engine, this one rebuilt it back, and every answer between +# came from whichever had won last (measured: IMPACT_VERSION 41 -> 76 and back, a full rebuild on each turn). So a build +# that replaces a graph another axiomcode built records that graph's maker (displace); when a graph by that same maker +# turns up again AFTER that, both are still running, and this one does not rebuild it again in the background: it says +# that two versions are running and that a restart of the agent session ends it (contested). `axiomcode index` still +# rebuilds it: asked for by name, it is not a turn. +DISPLACED_KEEP = 8 + +def build_key(b): + """a short identity of what built a graph (its file table's built_by): every axiomcode that writes it differently differs""" + if not isinstance(b, dict): return 'unrecorded' + keep = {k: b.get(k) for k in ('impact', 'rules', 'index', 'engine_version', 'engine_hash')} + return hashlib.sha1(json.dumps(keep, sort_keys=True).encode()).hexdigest()[:16] + +def displace(repo, t): + """a build of this axiomcode's is about to replace the graph table t describes, which another axiomcode built""" + if not t: return + d = dict(read_state(repo).get('displaced') or {}) + d[build_key(t.get('built_by'))] = time.time() + write_state(repo, displaced=dict(sorted(d.items(), key=lambda kv: kv[1])[-DISPLACED_KEEP:])) + +def _maker(b): + if not isinstance(b, dict): return 'an axiomcode that did not record its version' + return f"axiomcode {b.get('engine_version') or '?'} (IMPACT_VERSION {b.get('impact') or '?'})" + +def contested(repo, t=None, older=None): + """'' unless the graph (an older axiomcode's: `older` is engine_change's line) was built by an axiomcode this one had + already rebuilt it over, after that rebuild: then one line naming both, that two versions are running, and that this + one does not rebuild it again in the background""" + t = t if t is not None else load_table(repo) + older = engine_change(repo, t) if older is None else older + if not t or not older: return '' + when = (read_state(repo).get('displaced') or {}).get(build_key(t.get('built_by'))) + built = t.get('built') + if built is None: + try: built = os.path.getmtime(table_path(repo)) + except OSError: built = 0 + if not when or float(built) <= float(when): return '' + eng = current_engine(repo); mine = f"IMPACT_VERSION {plugin_id()[1]}" + if eng: + try: mine = f"{json.load(open(os.path.join(eng, 'package.json'))).get('version', '?')} ({mine})" + except (OSError, ValueError): pass + return (f"this graph was rebuilt by {_maker(t.get('built_by'))} after this axiomcode {mine} had rebuilt it: {TWO_VERSIONS}. " + "This one does not rebuild it again in the background, so the two stop rebuilding it in turn; once only one runs, " + "`axiomcode index` rebuilds it once") + def _seen_hash(repo, e): """the content hash of engine e: an engine whose files moved (a reinstall, a new checkout) but whose bytes did not is hashed once, and its hash kept beside the graph for its stat signature, so later queries only stat it""" @@ -695,12 +747,16 @@ def read_state(repo): except (OSError, ValueError): return {} def write_state(repo, **kv): + if orphaned(): return # an orphaned older install writes nothing (orphaned) st = read_state(repo); st.update(kv) tmp = state_path(repo) + f'.{os.getpid()}' try: json.dump(st, open(tmp, 'w')); os.replace(tmp, state_path(repo)) except OSError: pass +# ── AN ORPHANED INSTALL WRITES NOTHING: ax_orphan.orphaned, the one check every path that writes unasked makes ────────── +from ax_orphan import ORPHAN_MARK, install_dir, orphaned + def has_graph(repo): """a graph was built here: graph.sqlite exists, or is a pointer whose target is gone. A broken pointer is a graph to REPAIR, not a repository without one: taken for the latter, the refresher stopped for good and the next query @@ -717,7 +773,7 @@ def relink(repo, locked=False, say=True): interrupted". A pointer that leaves this .axiomcode/out is re-pointed at the same graph here (the copy's own), and says which case it was; one with no graph here to take is removed, so the query builds one. Returns what it did.""" out = out_dir(repo); p = os.path.join(out, 'graph.sqlite') - if not os.path.islink(p) or (not locked and building(repo)): return '' # a running build owns the pointer + if not os.path.islink(p) or (not locked and building(repo)) or orphaned(): return '' # a running build owns the pointer try: t = os.readlink(p) except OSError: return '' if not os.path.isabs(t): return '' # relative: this out/'s own; dangling, it is an interrupted build to repair @@ -963,6 +1019,8 @@ def status(repo): return dict(state='stale', changed=[unbuilt_row(repo)], added=[], removed=[], **nw) if unbuilt(repo) else dict(state='fresh', **nw) d = dict(state='building' if busy else 'stale', changed=changed, added=added, removed=removed, **nw) if eng: d['engine'] = eng + two = contested(repo, t, eng) if eng else '' + if two: d['contested'] = two if busy and not (changed or added or removed): d.update(pending(repo)) if not busy and failed_on(st, c, eng): d['failed'] = st.get('failed_log', ''); d['failed_reason'] = st.get('failed_reason', '') return d @@ -983,7 +1041,7 @@ def behind(s): return bool(edited(s) or s.get('engine')) def enabled(repo): - return not os.environ.get('AXIOMCODE_NO_REFRESH') and not os.environ.get('AXIOMCODE_GRAPH') and has_graph(repo) + return not os.environ.get('AXIOMCODE_NO_REFRESH') and not os.environ.get('AXIOMCODE_GRAPH') and has_graph(repo) and not orphaned() def refresh_off(repo): """the refresher is switched off (AXIOMCODE_NO_REFRESH) over a graph of this repository's own: nothing rebuilds it, @@ -1071,7 +1129,10 @@ def hook_kick(repo, trigger, session='', event='', tool=''): """what a hook (and the MCP server's timer) runs in place of kick: '' after starting the refresher as kick does, or, for a graph another axiomcode built (engine_change), nothing started and one line to say so, once per session and difference ('' when it was said already). A graph a newer axiomcode built is never rebuilt anyway (newer_build). - `event` and `tool` name the hook event and the tool it followed, for batching edits (batch_held)""" + `event` and `tool` name the hook event and the tool it followed, for batching edits (batch_held). An orphaned older + install (orphaned) starts and writes nothing, and says so at a session's start and on a prompt""" + o = orphaned() + if o: return f"graph refresh: {o}" if event in ('SessionStart', 'UserPromptSubmit') and has_graph(repo) else '' if not enabled(repo): return '' held = engine_change(repo) if not held: @@ -1080,6 +1141,8 @@ def hook_kick(repo, trigger, session='', event='', tool=''): key = f"{session}|{held}" if read_state(repo).get('hook_held') == key: return '' write_state(repo, hook_held=key) + two = contested(repo, None, held) + if two: return f"graph refresh: {two}" return (f"graph refresh: {held}; the hooks do not rebuild a graph another axiomcode built, so it is kept as it is. " f"`axiomcode index` rebuilds it with engine {engine_label(repo)}, and so does a query, which says so on its first " f"line; {NO_REFRESH_HOW}") @@ -1189,6 +1252,7 @@ def rewarm(repo, t): except OSError: pass def worker(repo): + if orphaned(): return 0 # an orphaned older install builds nothing (orphaned) lock = os.path.join(repo, '.axiomcode', 'refresh.lock') fd = os.open(lock, os.O_RDWR | os.O_CREAT, 0o644) if not _flock(fd, False): return 0 # single flight @@ -1219,6 +1283,12 @@ def worker(repo): # BUILT BY ANOTHER AXIOMCODE: rebuilt with no file changed, once per worker. A rebuild that leaves the same # difference behind (a build that could not record what built it) is not repeated eng = engine_change(repo, t) + two = contested(repo, t, eng) if eng else '' + if two: + # TWO VERSIONS TAKING TURNS: this one rebuilt it once already, and the other rebuilt it back + write_state(repo, state='contested', checked=time.time(), contested=two, checked_by=os.environ.get('AXIOMCODE_REFRESH_TRIGGER', '')) + print(f"{time.strftime('%H:%M:%S')} refresh: {two}", flush=True) + return 0 if eng and eng == engine_done: eng = '' if (c is None or not any(c)) and not eng and not base_moved(repo): if export_behind(t): rewarm(repo, t) @@ -1232,6 +1302,7 @@ def worker(repo): why = (f"{n} file(s) changed" if n else eng if eng else f"HEAD moved to {(head(repo) or '')[:10]}") + \ (f"; {eng}" if n and eng else '') + f", found by {os.environ.get('AXIOMCODE_REFRESH_TRIGGER') or 'an edit'}" engine_done = eng + if eng: displace(repo, t) env = rebuild_env(t, AXIOMCODE_BACKGROUND='1', AXIOMCODE_REFRESH_REASON=why) slot = take_slot(repo) # queued behind the machine's other background builds t0 = time.time(); write_state(repo, state='building', started=t0, files=sum(len(x) for x in c), edits=0) @@ -1281,7 +1352,7 @@ def wait(repo, seconds): if s['state'] == 'unknown': kick(repo, 'a query'); return s # a graph from before the file table: its first refresh records one # a FIRST build's other languages are not waited for: there is no graph of theirs to refresh, only a compile # that can take an hour (#1555). A rebuild's are, for the bounded while any refresh is. - if s['state'] in ('fresh', 'no graph') or s.get('failed') or s.get('first') or s.get('newer'): return s + if s['state'] in ('fresh', 'no graph') or s.get('failed') or s.get('first') or s.get('newer') or s.get('contested'): return s if s['state'] == 'stale': kick(repo, 'a query') # idempotent: a no-op while a worker holds its lock if time.time() >= end: return s time.sleep(0.5) @@ -1292,6 +1363,8 @@ def wait_baseline(repo, seconds, hook=False): hook: asked by a hook, which never rebuilds a graph another axiomcode built (hook_kick)""" if not enabled(repo) or not base_moved(repo): return '' if hook and engine_change(repo): return '' + two = contested(repo) + if two: return f"graph refresh: HEAD moved since the baseline was set ({head(repo)[:10]}), and {two}" if newer_build(repo): return (f"graph refresh: HEAD moved since the baseline was set ({head(repo)[:10]}), and the graph was built by a newer " "axiomcode, which this one does not rebuild: the baseline stays where it was, so this answer also counts what the " @@ -1350,6 +1423,9 @@ def note(s, marked=None, named=None, off=False): if s.get('state') not in ('stale', 'building'): return first if s.get('engine'): # BUILT BY ANOTHER AXIOMCODE: every row may differ from what this version answers, so none is marked; the line says so + if s.get('contested'): + return (first + '\n' if first else '') + f"graph refresh: {s['contested']}; this answer is from that graph, " \ + "and may lack what this version finds" + (f"; it predates edits to {', '.join(edited(s)[:5])}" if edited(s) else '') if off: says = "refresh is OFF (--no-refresh, or AXIOMCODE_NO_REFRESH is set), so nothing rebuilds it; `axiomcode index` does" elif s.get('failed'): says = "the rebuild FAILED" + (f" — {s['failed_reason']}" if s.get('failed_reason') else '') + f" (see {s['failed']})" else: says = "rebuilding in the background; --fresh waits for the rebuild" @@ -1536,7 +1612,7 @@ def wait_fresh(repo, seconds, say=False, files=()): while True: s = status(repo) if (s['state'] in ('fresh', 'no graph', 'unknown') or not behind(s)) and not _waits_on(s, files): return True - if s.get('failed') or s.get('newer'): return False # nothing here rebuilds a newer axiomcode's graph + if s.get('failed') or s.get('newer') or s.get('contested'): return False # nothing here rebuilds a newer axiomcode's graph if s['state'] == 'stale': kick(repo, 'a query') now = time.time() if now >= end: return False @@ -1573,6 +1649,11 @@ def with_note(n): # the answer as it is, then # REFRESH SWITCHED OFF IS NOT "UP TO DATE". With AXIOMCODE_NO_REFRESH nothing rebuilds the graph, which is exactly when # an answer from it most needs to say which edits it predates: a "nothing named X" for an X an edit just added read # as X not existing. It is checked, marked and noted as below; it only never kicks a refresh or waits for one. + o = orphaned() + if o: + # AN ORPHANED OLDER INSTALL answers nothing of its own: its verbs would write their caches, or a first build, into + # .axiomcode with the older version's rules (orphaned) + print(f"graph refresh: {o}.", flush=True); return 0 off = not enabled(repo) and refresh_off(repo) if not enabled(repo) and not off: # the refresh switched off still leaves a build that is solving the repository's other languages (#1555); a @@ -1592,7 +1673,7 @@ def with_note(n): # the answer as it is, then if not note(s): passthrough() return with_note(note(s)) # a graph a newer axiomcode built is never rebuilt here: no refresh is started or waited for, as with refresh off - hold = off or bool(s.get('newer')) + hold = off or bool(s.get('newer')) or bool(s.get('contested')) started = not s.get('failed') and not hold and kick(repo, 'a query') # a refresher already running (a hook's, the timer's, another query's) replaces this graph all the same: said too running = not started and not s.get('failed') and not hold and refresher_running(repo) @@ -1631,6 +1712,7 @@ def with_note(n): # the answer as it is, then obj['freshness'] = dict(state='off' if off else s['state'], edited=edited(s), rows_marked=n, **({'built_by': s['engine']} if s.get('engine') else {}), **({'newer': s['newer']} if s.get('newer') else {}), + **({'contested': s['contested']} if s.get('contested') else {}), **({'built_with': s['built_with']} if s.get('built_with') else {}), **({'failed': s['failed']} if s.get('failed') else {}), **({'named_in_edits': [dict(name=a, file=b) for a, b in missed]} if missed else {})) @@ -1698,7 +1780,7 @@ def main(argv): # the files are the graph's; the axiomcode that built it must be the one building now (axiomcode-build passes # the engine it found as AXIOMCODE_ENGINE), or the graph is rebuilt, saying why eng = engine_change(repo, t) - if eng: print(f"{eng}; rebuilding", flush=True); return 1 + if eng: displace(repo, t); print(f"{eng}; rebuilding", flush=True); return 1 return 0 if cmd == 'status': s = status(repo) @@ -1721,7 +1803,7 @@ def main(argv): # what axiomcode-build asks before it rebuilds: exit 0, saying why, when the graph here was built by a newer # axiomcode, which this one must not rebuild (newer_build); 1 otherwise n = newer_build(repo) - if n: print(f"{n}: kept, not rebuilt by this older axiomcode; update it, or AXIOMCODE_REINDEX=1 rebuilds the graph with this one") + if n: print(f"{n}: kept, not rebuilt by this older axiomcode; {TWO_VERSIONS}. Update it, or AXIOMCODE_REINDEX=1 rebuilds the graph with this one") return 0 if n else 1 if cmd == 'baseline': n = wait_baseline(repo, float(argv[3]) if len(argv) > 3 else 30) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_orphan.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_orphan.py new file mode 100644 index 000000000..b9148d809 --- /dev/null +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_orphan.py @@ -0,0 +1,46 @@ +"""ax_orphan.py: is the axiomcode running here an orphaned older install? One check, for every long-running path. + +An agent session started before an upgrade keeps running the OLD version's MCP server and hooks. A plugin manager that +keeps one directory per version marks the one it replaced with `.orphaned_at` (and deletes its files later); a process +still running out of it is an older axiomcode, and what it wrote into a repository's .axiomcode was the older version's: +it rebuilt the graph back to its own engine after the new one had rebuilt it, every rebuild undoing the last, and it +recreated .axiomcode (its refresh.log, refresh.json) while the user was deleting it, so `rm -rf .axiomcode` failed with +"Directory not empty". So every path that writes there without being asked to (the refresher, the hooks, the MCP server +and its timer, a query's background build) asks orphaned() first, and an orphaned install writes nothing and starts +nothing: the MCP server stops, a hook and a query say so instead. + +Its own module, with nothing but os and json, because every hook asks it before anything else: importing ax_fresh for it +cost a hook 20-100 ms on every tool call.""" +import json, os + +H = os.path.dirname(os.path.abspath(__file__)) +ORPHAN_MARK = '.orphaned_at' + +def install_dir(scripts=None): + """the plugin directory a scripts directory belongs to (/skills/axiomcode/scripts), these scripts' by default""" + return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(scripts or H)))) + +def _install_version(root): + try: return str(json.load(open(os.path.join(root, '.claude-plugin', 'plugin.json'), encoding='utf-8')).get('version') or '') + except (OSError, ValueError, AttributeError): pass + b = os.path.basename(os.path.normpath(root)) # a versioned plugin cache: /// + return b if all(x.isdigit() for x in b.split('.')) else '' + +def orphaned(root=None): + """'' while the install is current, else one line saying this is an orphaned older axiomcode and the agent session + is to be restarted. root: the plugin directory to judge (the MCP server passes the one it resolved); by default the + one these scripts belong to. Orphaned: the plugin directory carries ORPHAN_MARK, or its scripts are gone""" + if root: + root = os.path.abspath(root) + gone = not os.path.isfile(os.path.join(root, 'skills', 'axiomcode', 'scripts', 'ax_fresh.py')) + else: + # these scripts' own file is what is checked for "gone": an engine that lays them out differently is never orphaned + root = install_dir(); gone = not os.path.isfile(os.path.join(H, 'ax_fresh.py')) + if os.path.lexists(os.path.join(root, ORPHAN_MARK)): + why = f"{root} is marked {ORPHAN_MARK}: a newer version was installed in its place" + elif gone: why = f"its install directory {root} is gone: a newer version was installed in its place" + else: return '' + v = _install_version(root) + return (f"this is an orphaned older axiomcode{' ' + v if v else ''} ({why}), still running in an agent session started " + "before the upgrade; it writes nothing into the repository's .axiomcode and rebuilds nothing. Restart the agent " + "session so that only the installed axiomcode runs") diff --git a/tests/README.md b/tests/README.md index 06150d2fd..69540784a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -68,6 +68,11 @@ One check needs no graph and is its own script: and the refresh is expected within the budget, never on a rules compile; --fresh waits and MCP takes fresh=true; the file table prunes exactly what each parser skips, so an edit under out/ or build/ is seen (#1594, #1595; no engine) + python3 tests/two_versions.py two axiomcode versions over one repository: an older one's graph is rebuilt + once, not again each time the older one rebuilds it back, and every answer says + two versions are running and to restart the agent session; an install marked + .orphaned_at, or whose files are gone, writes nothing under .axiomcode from its + hooks, timer, refresher, query or MCP server (no engine) python3 tests/refresh_races.py what a refresh after an edit can leave behind, each with a control: a broken graph pointer is repaired and keeps the baseline; a query mid-build waits for it and starts no build; a failed build never leaves the pointer dangling; a failed diff --git a/tests/two_versions.py b/tests/two_versions.py new file mode 100644 index 000000000..f36c86f59 --- /dev/null +++ b/tests/two_versions.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""tests/two_versions.py: two axiomcode versions over one repository rebuild its graph once, not on every turn, say that +two versions are running, and an orphaned older install writes nothing into .axiomcode. + +An agent session started before an engine or plugin upgrade keeps the old version's MCP server and hooks running beside +the new ones. The old one rebuilt the graph with its engine, the new one rebuilt it back, and so on, a full rebuild on +every turn; the answers said "stale graph"; and the old one's refresher recreated .axiomcode while the user deleted it. + + turns this axiomcode rebuilds a graph an older one built, once; when that older one rebuilds it again afterwards, + this one does not rebuild it in the background again (refresher, hook, query), and the status, the query's + note and the hook all say two versions are running and to restart the agent session. Controls: an older + graph met for the first time is rebuilt; a rebuild that failed (the older graph not rebuilt since) is retried; + `index` (uptodate) still rebuilds a contested graph, asked for by name + newer an older axiomcode over a graph a newer one built keeps it, and says the two-versions case (note and the + `newer` command), not only "built by a newer axiomcode" + orphan an install whose plugin directory carries .orphaned_at, or whose files are gone, writes nothing under + .axiomcode from its refresh hook (session start, an edit), its other hooks, its timer (hook_kick), its + refresher, a query, or its MCP server, and says it is an orphaned older version to restart; the MCP server + refuses to start. Control: the same copy without the mark does write (the refresher starts on an edit) + +No engine: a stand-in engine directory, and a stand-in build that records what built the graph, as freshness.py does. + + python3 tests/two_versions.py [-v] +""" +import json, os, shutil, subprocess, sys, tempfile, time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import freshness # its stand-in engine and repository +from freshness import ax_fresh, fake_engine, fake_repo, check, RESULTS, SCRIPTS, ROOT, DRIVER + +PLUGIN = os.path.join(ROOT, 'plugins', 'axiomcode') + +# the stand-in build: records that it ran, and writes the file table as this axiomcode's build would (built_by, built) +BUILD = r""" +import json, os, sys, time +sys.path.insert(0, os.environ['TV_SCRIPTS']); import ax_fresh +repo = sys.argv[2] +open(os.path.join(repo, 'builds'), 'a').write('x\n') +tp = ax_fresh.table_path(repo); t = json.load(open(tp)) +t.update(built=time.time(), files=ax_fresh.snapshot(repo, 'python', repo), built_by=ax_fresh.built_by(ax_fresh.current_engine(repo), 'python')) +json.dump(t, open(tp, 'w')) +""" + + +def tree(d): + """every path under d with its size and mtime: what a process wrote there""" + out = {} + for root, dirs, files in os.walk(d): + for f in dirs + files: + p = os.path.join(root, f) + try: st = os.lstat(p); out[os.path.relpath(p, d)] = (st.st_size, st.st_mtime_ns) + except OSError: pass + return out + + +def diff(a, b): + return sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k)) + + +def turns_checks(work, env): + fb = os.path.join(work, 'build.py'); open(fb, 'w').write(BUILD) + runner = os.path.join(work, 'runner.sh'); open(runner, 'w').write(f'#!/bin/sh\nexec "{sys.executable}" "{fb}" "$@"\n'); os.chmod(runner, 0o755) + wenv = dict(env, AXIOMCODE_BASH=runner, TV_SCRIPTS=SCRIPTS) + + def builds(repo): + try: return len(open(os.path.join(repo, 'builds')).read().split()) + except OSError: return 0 + + def worker(repo): + subprocess.run([sys.executable, os.path.join(SCRIPTS, 'ax_fresh.py'), 'worker', repo], env=wenv, capture_output=True, text=True, timeout=60) + return builds(repo) + + other = dict(engine_version='0.9.0', engine_hash='0' * 40, engine_stat='0' * 40) + + def older_turn(repo): + """the older axiomcode rebuilds the graph with its own engine: the table records it, built now""" + tp = ax_fresh.table_path(repo); t = json.load(open(tp)) + t['built_by'].update(other); t['built'] = time.time(); json.dump(t, open(tp, 'w')) + + repo = fake_repo(work, 'turns'); os.remove(os.path.join(repo, '.axiomcode/refresh.json')) + older_turn(repo) + s = ax_fresh.status(repo) + check("turns: control: an older axiomcode's graph met for the first time is stale and not contested", + s.get('engine', '').startswith('graph built by an older axiomcode') and not s.get('contested'), s) + n1 = worker(repo) + check("turns: control: the refresher rebuilds an older axiomcode's graph once", n1 == 1, n1) + check("turns: after this one's rebuild the graph is current", ax_fresh.status(repo).get('state') == 'fresh', ax_fresh.status(repo)) + rounds = [] + for i in range(3): + time.sleep(0.02); older_turn(repo) # the older one, still running, rebuilds it back + rounds.append(worker(repo)) + check(f"turns: the older one rebuilding it back three times costs no further rebuild here (builds after each turn: {rounds})", + rounds == [1, 1, 1], rounds) + s = ax_fresh.status(repo) + two = s.get('contested', '') + check("turns: the status names the two-versions case: both versions, and restart the agent session", + 'two axiomcode versions are running' in two and 'axiomcode 0.9.0' in two and 'axiomcode 1.0.0' in two + and 'restart the agent session' in two and ax_fresh.read_state(repo).get('state') == 'contested', s) + # a query answers from it, starts nothing, waits for nothing, and its note is the two-versions line, not a stale-graph one + driver = os.path.join(work, 'driver.py'); open(driver, 'w').write(DRIVER.replace("ax_fresh.kick = lambda *a, **k: True", + "ax_fresh.kick = lambda *a, **k: open(os.path.join(sys.argv[2], 'kicked'), 'w').close() or True")) + t0 = time.time() + r = subprocess.run([sys.executable, driver, SCRIPTS, repo, 'impact', '--', sys.executable, '-c', "print('rows')"], + capture_output=True, text=True, env=dict(env, AXIOMCODE_FRESH_WAIT='30')) + took = time.time() - t0 + check(f"turns: a query answers at once ({took:.1f}s), starts no rebuild, and says two versions are running rather than 'rebuilding'", + r.stdout.strip() == 'rows' and 'two axiomcode versions are running' in r.stderr and 'rebuilding in the background' not in r.stderr + and 'queued' not in r.stderr and 'background rebuild' not in r.stdout and not os.path.exists(os.path.join(repo, 'kicked')) and took < 10, (r.stdout, r.stderr)) + h = ax_fresh.hook_kick(repo, 'the PostToolUse hook', session='s9', event='UserPromptSubmit') + check("turns: the hook says the two-versions case once", 'two axiomcode versions are running' in h + and ax_fresh.hook_kick(repo, 'the PostToolUse hook', session='s9', event='UserPromptSubmit') == '', h) + time.sleep(0.02); older_turn(repo) # the older one's graph, whichever way the turns above went + u = subprocess.run([sys.executable, os.path.join(SCRIPTS, 'ax_fresh.py'), 'uptodate', repo, 'python', '', ''], capture_output=True, text=True, env=env) + check("turns: control: `index` (uptodate) still rebuilds a contested graph, asked for by name", u.returncode == 1 and 'rebuilding' in u.stdout, u.stdout) + + # control: a rebuild that FAILED leaves the older graph as it was (not rebuilt since): that is no contest, it is retried + repo = fake_repo(work, 'failed'); os.remove(os.path.join(repo, '.axiomcode/refresh.json')) + older_turn(repo); ax_fresh.displace(repo, ax_fresh.load_table(repo)) + check("turns: control: an older graph not rebuilt since this one displaced it (a failed rebuild) is not contested, and is rebuilt", + not ax_fresh.status(repo).get('contested') and worker(repo) == 1, ax_fresh.status(repo)) + + +def newer_checks(work, env): + repo = fake_repo(work, 'newer'); os.remove(os.path.join(repo, '.axiomcode/refresh.json')) + tp = ax_fresh.table_path(repo); t = json.load(open(tp)); t['built_by'].update(engine_version='1.0.1'); json.dump(t, open(tp, 'w')) + s = ax_fresh.status(repo) + note = ax_fresh.note(s) + check("newer: an older axiomcode over a newer one's graph says two versions are running and to restart, and that it does not rebuild", + s.get('newer') and 'engine 1.0.1, this one is 1.0.0' in note and 'two axiomcode versions are running' in note + and 'restart the agent session' in note and 'does not rebuild' in note, note) + n = subprocess.run([sys.executable, os.path.join(SCRIPTS, 'ax_fresh.py'), 'newer', repo], capture_output=True, text=True, env=env) + check("newer: `index` is told the two-versions case too", n.returncode == 0 and 'two axiomcode versions are running' in n.stdout, n.stdout) + + +def orphan_checks(work, env): + fb = os.path.join(work, 'build-orphan.py'); open(fb, 'w').write("import os, sys; open(os.path.join(sys.argv[2], 'builds'), 'a').write('x\\n')\n") + runner = os.path.join(work, 'runner-orphan.sh'); open(runner, 'w').write(f'#!/bin/sh\nexec "{sys.executable}" "{fb}" "$@"\n'); os.chmod(runner, 0o755) + cfg = os.path.join(work, 'agent-config'); os.makedirs(cfg) + # TMPDIR: the hooks remember a session's repositories there, and a session-start hook refreshes them too + oenv = dict(env, AXIOMCODE_BASH=runner, AXIOMCODE_REFRESH_DEBOUNCE='0.05', CLAUDE_CONFIG_DIR=cfg, TMPDIR=os.path.join(work, 'tmp')) + os.makedirs(oenv['TMPDIR']) + for k in ('AXIOMCODE_PLUGIN_ROOT', 'AXIOMCODE_NO_REFRESH'): oenv.pop(k, None) + + # this checkout's plugin version, which the orphan message names (it moves with every release) + VERSION = json.load(open(os.path.join(PLUGIN, '.claude-plugin', 'plugin.json'), encoding='utf-8'))['version'] + + def install(name, mark): + """a copy of this plugin in a versioned plugin cache: ///""" + d = os.path.join(work, 'cache', name, 'axiomcode', VERSION) + shutil.copytree(PLUGIN, d, ignore=shutil.ignore_patterns('__pycache__')) + if mark: open(os.path.join(d, '.orphaned_at'), 'w').write('1791880805485\n') + return d + + def edited_repo(name): + repo = fake_repo(work, name); os.remove(os.path.join(repo, '.axiomcode/refresh.json')) + open(os.path.join(repo, 'shop/api.py'), 'a').write('\ndef audit(items):\n return total(items)\n') + return repo + + def hook(plugin, repo, event, **extra): + ev = dict(hook_event_name=event, cwd=repo, session_id='two-versions-' + os.path.basename(repo), **extra) + return subprocess.run([sys.executable, os.path.join(plugin, 'hooks', 'refresh.py')], input=json.dumps(ev), capture_output=True, text=True, env=oenv, timeout=30) + + def everything(plugin, repo): + """every path by which an install writes into .axiomcode without being asked to; returns what the hooks printed""" + said = hook(plugin, repo, 'SessionStart').stdout + hook(plugin, repo, 'PostToolUse', tool_name='Edit', tool_input=dict(file_path=os.path.join(repo, 'shop/api.py'))) + for h in ('changes.py', 'orient.py', 'assist.py'): + subprocess.run([sys.executable, os.path.join(plugin, 'hooks', h)], input=json.dumps(dict(hook_event_name='UserPromptSubmit', cwd=repo, session_id='two-versions-' + os.path.basename(repo), + prompt='how does run reach total')), capture_output=True, text=True, env=oenv, timeout=60) + scripts = os.path.join(plugin, 'skills', 'axiomcode', 'scripts') + code = ("import sys; sys.path.insert(0, sys.argv[1]); import ax_fresh; ax_fresh.hook_kick(sys.argv[2], 'the timer'); " + "ax_fresh.kick(sys.argv[2]); ax_fresh.worker(sys.argv[2])") + subprocess.run([sys.executable, '-c', code, scripts, repo], capture_output=True, text=True, env=oenv, timeout=60) + q = subprocess.run([sys.executable, os.path.join(scripts, 'ax_fresh.py'), 'query', repo, 'impact', '--', sys.executable, '-c', "print('rows')"], + capture_output=True, text=True, env=oenv, timeout=60) + time.sleep(1.0) # a detached refresher, had one been started, has written by now + return said, q.stdout + + # control first: the same copy, not orphaned, does write on an edit (else "wrote nothing" below would prove nothing) + live = install('live', mark=False); repo = edited_repo('orphan-control') + before = tree(os.path.join(repo, '.axiomcode')); everything(live, repo); time.sleep(1.5) + wrote = diff(before, tree(os.path.join(repo, '.axiomcode'))) + check("orphan: control: the same install, not orphaned, writes under .axiomcode after an edit (the refresher starts)", + 'refresh.log' in wrote and 'refresh.json' in wrote, wrote) + + plugin = install('marked', mark=True); repo = edited_repo('orphan-marked') + before = tree(os.path.join(repo, '.axiomcode')); said, q = everything(plugin, repo) + wrote = diff(before, tree(os.path.join(repo, '.axiomcode'))) + check("orphan: an install marked .orphaned_at writes nothing under .axiomcode (hooks, timer, refresher, query)", wrote == [] and not + os.path.exists(os.path.join(repo, 'builds')), wrote) + check("orphan: its session-start hook says it is an orphaned older version and to restart the agent session", + f'orphaned older axiomcode {VERSION}' in said and 'Restart the agent session' in said, said) + check("orphan: its query answers with the restart, not from a build of its own", 'orphaned older axiomcode' in q and 'rows' not in q, q) + # the MCP server: a tool call answers with the restart and writes nothing; started anew it refuses to start + code = ("import os, sys; sys.path.insert(0, os.path.join(sys.argv[1], 'mcp')); import server; " + "print(server.run(['impact', 'run', sys.argv[2]]))") + before = tree(os.path.join(repo, '.axiomcode')) + r = subprocess.run([sys.executable, '-c', code, plugin, repo], capture_output=True, text=True, cwd=repo, timeout=60, + env=dict(oenv, AXIOMCODE_PLUGIN_ROOT=plugin)) + check("orphan: its MCP server's tool call answers that it is an orphaned older version, running nothing", + 'orphaned older axiomcode' in r.stdout and 'Restart the agent session' in r.stdout and diff(before, tree(os.path.join(repo, '.axiomcode'))) == [], + (r.stdout, r.stderr[-400:])) + r = subprocess.run([sys.executable, os.path.join(plugin, 'mcp', 'server.py')], capture_output=True, text=True, cwd=repo, timeout=60, + stdin=subprocess.DEVNULL, env=dict(oenv, AXIOMCODE_PLUGIN_ROOT=plugin)) + check("orphan: its MCP server, started, says so and does not start, writing nothing", + r.returncode != 0 and 'orphaned older axiomcode' in r.stderr and 'Not starting' in r.stderr + and diff(before, tree(os.path.join(repo, '.axiomcode'))) == [], (r.returncode, r.stderr[-400:])) + + # the install directory deleted under a running process (the plugin manager's later step): nothing is written either + plugin = install('gone', mark=False); repo = edited_repo('orphan-gone') + scripts = os.path.join(plugin, 'skills', 'axiomcode', 'scripts') + before = tree(os.path.join(repo, '.axiomcode')) + code = ("import shutil, sys; sys.path.insert(0, sys.argv[1]); import ax_fresh; shutil.rmtree(sys.argv[3]); " + "print(ax_fresh.orphaned()); ax_fresh.hook_kick(sys.argv[2], 'the timer'); ax_fresh.kick(sys.argv[2]); ax_fresh.worker(sys.argv[2])") + r = subprocess.run([sys.executable, '-c', code, scripts, repo, plugin], capture_output=True, text=True, env=oenv, timeout=60) + time.sleep(1.0) + wrote = diff(before, tree(os.path.join(repo, '.axiomcode'))) + check("orphan: a running process whose install directory was deleted says so and writes nothing under .axiomcode", + 'is gone' in r.stdout and 'Restart the agent session' in r.stdout and wrote == [], (r.stdout, r.stderr[-400:], wrote)) + check("orphan: control: this checkout (no mark, files present) is not orphaned", ax_fresh.orphaned() == '', ax_fresh.orphaned()) + + +def main(): + work = tempfile.mkdtemp(prefix='axiomcode-two-versions-'); saved = os.environ.get('AXIOMCODE_ENGINE') + try: + e1 = fake_engine(os.path.join(work, 'e1'), version='1.0.0'); os.environ['AXIOMCODE_ENGINE'] = e1 + env = {k: v for k, v in os.environ.items() if k not in ('AXIOMCODE_NO_REFRESH', 'AXIOMCODE_FRESH', 'AXIOMCODE_GRAPH')} + env.update(AXIOMCODE_REFRESH_DEBOUNCE='0.05', AXIOMCODE_REFRESH_SLOTS=os.path.join(work, 'slots')) + for part in (turns_checks, newer_checks, orphan_checks): + try: part(work, env) + except Exception as e: # a missing helper is a failure of that part, not of the run + check(f"{part.__name__}: ran to the end", False, f"{type(e).__name__}: {e}") + finally: + if saved is None: os.environ.pop('AXIOMCODE_ENGINE', None) + else: os.environ['AXIOMCODE_ENGINE'] = saved + shutil.rmtree(work, ignore_errors=True) + bad = [n for n, ok in RESULTS if not ok] + print(f"\n{len(RESULTS) - len(bad)} passed, {len(bad)} failed") + return 1 if bad or not RESULTS else 0 + + +if __name__ == '__main__': + sys.exit(main()) From b0ad90ff243fa57223a5901f79fbcba342279766 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:17:30 -0700 Subject: [PATCH 04/18] Stage IR rows with perl when it is there, awk otherwise Staging copies every IR CSV into a Souffle fact file and drops rows whose tab-separated field count differs from the header's. BSD awk spends most of that time splitting fields it never reads: on a ~1,300-file Python project the step took 16-19 s of wall time per build, 13.6 CPU-s of it on the largest IR file (expressions). perl counts tabs instead and stages the same bytes (identical md5 over every staged file of three projects, two Python and one Java) at about a tenth of the CPU. The filter moves to graph/pipeline/stage-rows.sh. It picks perl when it runs (macOS, mainstream Linux, and Git for Windows' bash, which is the shell the pipeline runs under on win32) and falls back to the same awk program where perl is missing, so no machine loses the step. AXIOM_STAGE_WITH=awk forces the fallback. graph/test/tools/stage-rows-test.sh holds both to the same stdout and stderr on the edge cases (trailing tabs, empty lines, CRLF, no final newline, torn rows) and runs as a Java suite preflight. Refs #1835 --- graph/pipeline/run-souffle.sh | 6 ++- graph/pipeline/stage-rows.sh | 50 ++++++++++++++++++++++ graph/test/java/run-tests.sh | 7 ++++ graph/test/tools/stage-rows-test.sh | 65 +++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 graph/pipeline/stage-rows.sh create mode 100644 graph/test/tools/stage-rows-test.sh diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index f903e654b..b0ce60a1f 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -88,6 +88,8 @@ ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; } # shellcheck source=souffle-include.sh . "$SRC/pipeline/souffle-include.sh" +# shellcheck source=stage-rows.sh +. "$SRC/pipeline/stage-rows.sh" # shellcheck source=engine.conf . "$SRC/pipeline/engine.conf" # Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). @@ -817,7 +819,7 @@ while IFS=$'\t' read -r rel csv; do # deterministically on some inputs — takes down a run of tens of thousands of sites. The # header's field count is the contract; a row that does not meet it is dropped and # COUNTED, so the loss is visible rather than fatal and never silent. - if [ -f "$CLIENT/$csv.csv" ]; then awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" + if [ -f "$CLIENT/$csv.csv" ]; then stage_rows "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" else : > "$FACTS/$rel.facts"; fi CLIENT_INPUTS="$CLIENT_INPUTS$rel"$'\n' done < <(read_map "$TPL/client-ir.map"; if [ -f "$TPL/client-extra.map" ]; then read_map "$TPL/client-extra.map"; fi) @@ -879,7 +881,7 @@ if [ ! -d "$LIBDIR" ]; then : > "$TMPDIR_L/$rel.facts" for root in "${LIB_ROOTS[@]}"; do while IFS= read -r mod; do - [ -f "$mod/$csv.csv" ] && awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts" + [ -f "$mod/$csv.csv" ] && stage_rows "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts" done < <(lib_modules "$root") done done < <(printf '%s' "$LIB_SIG_RELS") diff --git a/graph/pipeline/stage-rows.sh b/graph/pipeline/stage-rows.sh new file mode 100644 index 000000000..200ff7670 --- /dev/null +++ b/graph/pipeline/stage-rows.sh @@ -0,0 +1,50 @@ +# stage-rows.sh: copy one IR CSV into a Soufflé fact file, dropping torn rows. Sourced by run-souffle.sh. +# +# stage_rows FILE print FILE's rows without its header, keeping only the rows whose tab-separated +# field count equals the header's; report how many were dropped on stderr. +# +# The filter is line based: a row is one newline-terminated line, a field count is the number of tabs +# plus one (an empty line has none), and a last line without a newline is printed with one. Those are +# exactly awk -F'\t' semantics, which is what this step used to run. +# +# PERL FIRST, AWK AS THE FALLBACK. The two print byte-identical output, but BSD awk (macOS) spends most +# of a build's staging time splitting fields it never reads: 13.6 CPU-s on a 352 MB expressions file, +# against 0.65 s for perl, which only counts tabs. perl ships with macOS, every mainstream Linux +# distribution and Git for Windows' bash (the shell this runs under on win32); a minimal image without +# it (Alpine, distroless) gets the same rows from awk, just slower. graph/test/tools/stage-rows-test.sh +# holds both to the same bytes on the edge cases (trailing tabs, empty lines, CRLF, no final newline). +# +# AXIOM_STAGE_WITH=awk forces the fallback (the test uses it; so can anyone bisecting a staging change). + +if [ "${AXIOM_STAGE_WITH:-}" != awk ] && command -v perl >/dev/null 2>&1 && perl -e 1 >/dev/null 2>&1; then + STAGE_ROWS_TOOL=perl +else + STAGE_ROWS_TOOL=awk +fi + +stage_rows(){ + if [ "$STAGE_ROWS_TOOL" = perl ]; then + # Bytes in, bytes out (no PerlIO layers, whatever PERL_UNICODE says), and $/ pinned to "\n". + perl -e ' + my $f = $ARGV[0]; + open(my $h, "<:raw", $f) or die "cannot read $f: $!\n"; + binmode STDOUT, ":raw"; + local $/ = "\n"; + my $hdr = <$h>; + exit 0 unless defined $hdr; + my $hl = length($hdr) - (substr($hdr, -1) eq "\n" ? 1 : 0); + my $n = $hl ? ($hdr =~ tr/\t//) + 1 : 0; + my $bad = 0; + while (my $l = <$h>) { + my $nl = substr($l, -1) eq "\n"; + my $nf = (length($l) - ($nl ? 1 : 0)) ? ($l =~ tr/\t//) + 1 : 0; + if ($nf == $n) { print($nl ? $l : "$l\n") } else { $bad++ } + } + close($h); + close(STDOUT) or die "cannot write the staged rows of $f: $!\n"; + printf STDERR " ! dropped %d malformed row(s) from %s\n", $bad, $f if $bad > 0; + ' "$1" + else + awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$1" + fi +} diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 18db81e1d..e2a48d386 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -99,6 +99,13 @@ fi if ! bash "$ROOT/graph/test/tools/lib-cache-key-test.sh"; then echo "FAIL: the library-facts cache key is not a function of the staged modules' content (#588)"; exit 1 fi +# ── Staging must not depend on which row filter the machine has ───────────── +# graph/pipeline/stage-rows.sh runs perl where it exists and awk where it does not; both must stage +# the same bytes, or the facts every language solves from would differ by machine. +if ! bash "$ROOT/graph/test/tools/stage-rows-test.sh"; then + echo "aborting: perl and awk stage different rows from the same IR" + exit 1 +fi # ── The -I for soufflé's headers must be the one that actually compiles ────── # Also a preflight, and for the same reason: the old resolution returned a path that only built on # the machine it was written on, and every run here compiles the engine through it. diff --git a/graph/test/tools/stage-rows-test.sh b/graph/test/tools/stage-rows-test.sh new file mode 100644 index 000000000..67abcca69 --- /dev/null +++ b/graph/test/tools/stage-rows-test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# Staging copies every IR CSV into a Soufflé fact file and drops the rows whose field count differs +# from the header's (graph/pipeline/stage-rows.sh). It runs perl where perl exists and awk where it +# does not, so the two must print the same bytes: a difference would make the facts, and so the +# graph, a function of which tools the machine happens to have. +# +# Each fixture below is staged by both, and the stdout bytes and the stderr report must match. The +# fixtures are the shapes where a line-based tab counter and awk's field splitting could part: a +# trailing tab (an empty last field), an empty line, CRLF, a last line with no newline, a header with +# no newline, an empty file, a quoted field holding a tab (both see two fields: neither parses quotes), +# bytes that are not UTF-8, and torn rows of both kinds (too few fields, too many). +# A NUL byte is left out on purpose: BSD awk ends the line at it (and drops the row as torn) while gawk +# and perl keep the whole line, so awk does not agree with itself there and is no reference. +# ───────────────────────────────────────────────────────────────────────────── +set -uo pipefail +unset AXIOM_STAGE_WITH +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" + +command -v perl >/dev/null 2>&1 || { echo "stage-rows-test: FAILED (no perl on PATH; this test compares perl against awk)"; exit 1; } + +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +fail=0; checks=0 +bad(){ checks=$((checks+1)); printf ' FAIL %s\n' "$1"; fail=1; } +ok(){ checks=$((checks+1)); } + +# The tool the sourced file picks: perl here, awk when forced. +( . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = perl ] ) && ok || bad "stage-rows.sh did not pick perl although perl is on PATH" +( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = awk ] ) && ok || bad "AXIOM_STAGE_WITH=awk did not force awk" + +printf 'a\tb\tc\n1\t2\t3\n4\t5\t6\n' > "$W/clean.csv" +printf 'a\tb\tc\n1\t2\t3\n1\t2\n1\t2\t3\t4\n7\t8\t9\n' > "$W/torn.csv" +printf 'a\tb\tc\n1\t2\t\n\t\t\n1\t2\t3\t\n' > "$W/trailing-tab.csv" +printf 'a\tb\tc\n1\t2\t3\n\n4\t5\t6\n\n' > "$W/empty-line.csv" +printf 'a\tb\tc\r\n1\t2\t3\r\n4\t5\r\n' > "$W/crlf.csv" +printf 'a\tb\tc\n1\t2\t3\n4\t5\t6' > "$W/no-final-newline.csv" +printf 'a\tb\tc\n1\t2' > "$W/torn-last-line.csv" +printf 'a\tb\tc' > "$W/header-only-no-newline.csv" +printf 'a\tb\tc\n' > "$W/header-only.csv" +: > "$W/empty.csv" +printf 'a\tb\n"x\ty"\tz\n"p\tq"\n' > "$W/quoted-tab.csv" +printf 'a\tb\n\xff\xfe\t\xc3\x28\n' > "$W/bytes.csv" +printf 'a\n1\n\n2\n' > "$W/one-column.csv" +printf '\n\n1\n\n' > "$W/empty-header.csv" + +n=0 +for f in "$W"/*.csv; do + n=$((n+1)); name="$(basename "$f")" + ( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/p.out" 2> "$W/p.err"; prc=$? + ( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/a.out" 2> "$W/a.err"; arc=$? + [ "$prc" = 0 ] && [ "$arc" = 0 ] && ok || bad "$name: exit status perl=$prc awk=$arc" + cmp -s "$W/p.out" "$W/a.out" && ok || { bad "$name: staged rows differ"; od -c "$W/a.out" | sed 's/^/ awk /'; od -c "$W/p.out" | sed 's/^/ perl /'; } + cmp -s "$W/p.err" "$W/a.err" && ok || bad "$name: report differs: perl '$(cat "$W/p.err")' awk '$(cat "$W/a.err")'" +done + +# Not vacuous: the fixtures must hold rows that are kept and rows that are dropped. +( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/torn.csv" ) > "$W/t.out" 2> "$W/t.err" +[ "$(printf '1\t2\t3\n7\t8\t9\n')" = "$(cat "$W/t.out")" ] && ok || bad "torn.csv: expected the two whole rows, got '$(cat "$W/t.out")'" +grep -q "dropped 2 malformed row(s) from $W/torn.csv" "$W/t.err" && ok || bad "torn.csv: expected a report of 2 dropped rows, got '$(cat "$W/t.err")'" +# The control: a clean file comes through as its rows, header removed, nothing reported. +( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/clean.csv" ) > "$W/c.out" 2> "$W/c.err" +tail -n +2 "$W/clean.csv" | cmp -s - "$W/c.out" && [ ! -s "$W/c.err" ] && ok || bad "clean.csv: not passed through unchanged" + +[ "$n" -ge 14 ] || bad "only $n fixtures were staged" +[ $fail = 0 ] && echo "stage-rows-test: ok ($checks checks over $n fixtures)" || { echo "stage-rows-test: FAILED"; exit 1; } From 957a55e1eec54b63006a3ec65db5a8f044242063 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:38:05 -0700 Subject: [PATCH 05/18] axiomcode-index: read the IR with a line splitter, skip a VACUUM with nothing to reclaim, do not rebuild bundle indexes On a ~1,300-file Python project axiomcode-index took 12-14 s. Three parts of that bought nothing: - csv.DictReader over the IR. The reader is QUOTE_NONE, so a row is one line and a field is the text between tabs; ir_rows() does exactly that with str.split and dict(zip()) at about half the cost. It keeps the reader's semantics where they are not obvious: \r, \n and \r\n each end a row, blank lines are skipped, a short row gets None for the missing columns and a long one keeps the rest under the key None. tests/index_reader.py compares the two on fixtures (quoted tabs and newlines, CRLF, lone CR, repeated header names, invalid UTF-8) and on any IR directory given to it. - VACUUM of a database the bundle wrote seconds earlier. It now runs only when this run dropped a table it recreates (a re-index) or the file has free pages; a freshly bundled graph has neither. - ce_caller, ce_callee and fa_caller duplicate the bundle's own idx_call_edges_caller_id, idx_call_edges_callee_method_id and idx_field_access_caller_id. They are created only when the graph has no index on that column, so an older graph keeps getting them. graph.sqlite holds the same rows in every table (rowids included) on two Python projects and one Java project; only index_meta.built_at and the three duplicate indexes differ. Refs #1835 --- .../skills/axiomcode/scripts/axiomcode-index | 58 ++++++++-- tests/index_reader.py | 106 ++++++++++++++++++ 2 files changed, 154 insertions(+), 10 deletions(-) create mode 100644 tests/index_reader.py diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index index 2751c4430..b5ddd518d 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index @@ -20,8 +20,7 @@ to --src). The language's IR is read through the ADAPTER table below — header positions — mirroring graph/bundle/languages.ts. When the IR is absent the index degrades: symbols come from methods/types alone, refs/literals/comments are empty, and index_meta says so. """ -import csv, os, re, sqlite3, sys, time, glob, collections, functools -csv.field_size_limit(10**9) +import os, re, sqlite3, sys, time, glob, collections, functools INDEX_VERSION = '9' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; # v5: JavaScript fields owned by their class, computed-key members named by their key, anonymous class expressions named by their binding; # v6: TypeScript class-property arrows named after their field; @@ -273,13 +272,40 @@ def is_test_file(fp): def rows(name): p = os.path.join(IR, name) if not HAVE_IR or not os.path.exists(p): return iter(()) - f = open(p, newline='', encoding='utf-8', errors='replace') - return csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE) + return ir_rows(p) + +def ir_rows(p): + """The rows of one IR file as dicts keyed by its header, EXACTLY as csv.DictReader(f, delimiter='\\t', + quoting=csv.QUOTE_NONE) over open(p, newline='') reads them, at about half the cost (the expressions file is + most of this script's time on a large repository). That reader treats a quote as an ordinary character, so a row + is one line and a field is the text between tabs: a line ends at \\n, \\r or \\r\\n (newline='' splits on all + three and keeps them, and each ends a record), a blank line is skipped, a short row has None for its missing + columns and a long one keeps the rest as a list under the key None. The writer's RFC 4180 quoting is NOT undone + here, as it was not before (see unquoted() below). tests/index_reader.py holds the two to the same dicts.""" + with open(p, newline='', encoding='utf-8', errors='replace') as f: + head = f.readline() + if not head: return + if head[-1:] == '\n': head = head[:-1] + if head[-1:] == '\r': head = head[:-1] + keys = head.split('\t') if head else [] + n = len(keys) + for line in f: + if line[-1:] == '\n': line = line[:-1] + if line[-1:] == '\r': line = line[:-1] + if not line: continue + v = line.split('\t') + d = dict(zip(keys, v)) + if len(v) != n: + if len(v) > n: d[None] = v[n:] + else: + for k in keys[len(v):]: d[k] = None + yield d # an older augment step left `symbols` as a VIEW; a name is dropped as whichever it is +dropped = False # whether this run frees pages: it decides the VACUUM at the end for name in ('symbols', 'refs', 'literals', 'comments', 'nesting', 'type_refs', 'decorations', 'skipped', 'index_meta', 'paths', 'callers', 'callees', 'source', 'sites'): kind = c.execute("SELECT type FROM sqlite_master WHERE name=?", (name,)).fetchone() - if kind: c.execute(f"DROP {kind[0].upper()} IF EXISTS {name}") + if kind: c.execute(f"DROP {kind[0].upper()} IF EXISTS {name}"); dropped = True c.executescript(''' DROP TABLE IF EXISTS symbols; DROP TABLE IF EXISTS refs; DROP TABLE IF EXISTS literals; DROP TABLE IF EXISTS comments; DROP TABLE IF EXISTS nesting; DROP TABLE IF EXISTS type_refs; DROP TABLE IF EXISTS decorations; DROP TABLE IF EXISTS skipped; DROP TABLE IF EXISTS index_meta; DROP TABLE IF EXISTS paths; @@ -765,11 +791,21 @@ if dec: c.executemany("INSERT INTO skipped VALUES (?,?)", ((rel(r.get('filePath') or r.get('file') or ''), r.get('reason') or r.get('skipReason') or '') for r in rows(A['skipped']))) # ── views: the joins written once ──────────────────────────────────────────────────────────────── +# Who calls a method and what it calls are looked up by call_edges(callee_method_id) and call_edges(caller_id); the +# fields a caller reads by field_access(caller_id): via_base_rows asks per single-target call site, and without that +# index each ask scanned the whole table (65,797 rows × 5,162 sites on an 8,619-file repository = 24 s of a 171 s warm +# → 1.4 s). The bundle indexes all three columns itself (idx__, schema.ts `indexed`), so they are made +# here only for a graph that lacks one: building the same index twice cost a second pass over the largest tables. +def has_index_on(table, cols): + for row in c.execute(f"PRAGMA index_list({table})").fetchall(): + if row[4] == 0 and [r[2] for r in c.execute(f"PRAGMA index_info({row[1]})")] == cols: return True # not partial + return False +for _name, _table, _col in (('ce_callee', 'call_edges', 'callee_method_id'), ('ce_caller', 'call_edges', 'caller_id'), ('fa_caller', 'field_access', 'caller_id')): + if not has_index_on(_table, [_col]): c.execute(f"CREATE INDEX IF NOT EXISTS {_name} ON {_table}({_col})") c.executescript(''' CREATE INDEX IF NOT EXISTS symbols_name ON symbols(name); CREATE INDEX IF NOT EXISTS symbols_display ON symbols(display); CREATE INDEX IF NOT EXISTS symbols_file ON symbols(file, line); CREATE INDEX IF NOT EXISTS symbols_mid ON symbols(method_id); CREATE INDEX IF NOT EXISTS symbols_tid ON symbols(type_id); CREATE INDEX IF NOT EXISTS refs_name ON refs(name); CREATE INDEX IF NOT EXISTS type_refs_name ON type_refs(name); CREATE INDEX IF NOT EXISTS decorations_owner ON decorations(owner_id); CREATE INDEX IF NOT EXISTS literals_value ON literals(value); CREATE INDEX IF NOT EXISTS comments_file ON comments(file); -CREATE INDEX IF NOT EXISTS ce_callee ON call_edges(callee_method_id); CREATE INDEX IF NOT EXISTS ce_caller ON call_edges(caller_id); -- lookups by id and by file + line: without them each per-site or per-test query scans the whole table CREATE INDEX IF NOT EXISTS symbols_id ON symbols(id); CREATE INDEX IF NOT EXISTS refs_file_line ON refs(file, line); CREATE INDEX IF NOT EXISTS literals_file_line ON literals(file, line); @@ -777,9 +813,6 @@ CREATE INDEX IF NOT EXISTS literals_file_line ON literals(file, line); -- the parts of a partial type (type_parts): without these each ask scanned type_refs / symbols, 1.2 s of a 1.8 s impact -- on a 1,300-file Java repository → 0.5-0.75 s, answers unchanged CREATE INDEX IF NOT EXISTS type_refs_file_line ON type_refs(file, line); CREATE INDEX IF NOT EXISTS symbols_qn ON symbols(qualified_name); --- the fields a caller reads: via_base_rows asks per single-target call site, and without the index each ask --- scanned the whole table (65,797 rows × 5,162 sites on an 8,619-file repository = 24 s of a 171 s warm → 1.4 s) -CREATE INDEX IF NOT EXISTS fa_caller ON field_access(caller_id); -- who calls a method: one row per (site, target), with the engine's tier. caller_id is symbols.id: a -- Java call written in a field initializer has the TYPE as its caller, and that row is kept, not dropped CREATE VIEW callers AS @@ -804,7 +837,12 @@ meta = dict(language=LANG, repo=REPO, source_dir=SRC, ir=IR if HAVE_IR else '', symbols=n('symbols'), refs=n('refs'), type_refs=n('type_refs'), decorations=n('decorations'), literals=n('literals'), comments=n('comments'), nested_types=n('nesting'), skipped_files=n('skipped')) c.executemany("INSERT INTO index_meta VALUES (?,?)", [(k, str(v)) for k, v in meta.items()]) con.commit() -con.execute("VACUUM") # the tables above are dropped and recreated on every run; without this the file only grows +# The tables above are dropped and recreated on every run, so a re-index leaves their old pages free and the file only +# grows without a VACUUM. A graph the bundle has just written has nothing to reclaim: no table of this script existed +# in it, and nothing freed a page (freelist_count). Rewriting it anyway cost 4.2 s of a 12 s index on a ~1,300-file +# Python repository, for the same rows. +if dropped or c.execute("PRAGMA freelist_count").fetchone()[0] > 0: + con.execute("VACUUM") con.close() # THE ASSERTED LINKS (ax_links.py, axiomcode-links.tsv) are applied to every graph as it is indexed, after the engine's # export and before the graph is published, so a rebuild keeps them and no query ever sees a graph without them: each is diff --git a/tests/index_reader.py b/tests/index_reader.py new file mode 100644 index 000000000..bc43878aa --- /dev/null +++ b/tests/index_reader.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""tests/index_reader.py: axiomcode-index reads the IR exactly as csv.DictReader did, row for row and key for key. + +axiomcode-index reads every IR row through ir_rows(), a line splitter that replaced +csv.DictReader(f, delimiter='\\t', quoting=csv.QUOTE_NONE) because the DictReader was most of the index's time on a +large repository. The index's tables are made of these dicts, so any difference is a different graph. Each fixture is +read by both and the lists of dicts must be equal: the shapes where a line splitter and the csv module could part +are a quoted field holding a tab or a newline (QUOTE_NONE does not unquote, so both split it), CRLF, a lone CR, a +blank line, a short row (None for the missing columns), a long row (the rest under the key None), a repeated header +name, a blank header, an empty file, a header with no rows, no final newline, bytes that are not UTF-8, and the +Unicode line separators that str.splitlines() would split on and the csv module does not. + + python3 tests/index_reader.py [IR_DIR ...] each IR_DIR: every *.csv in it is compared as well +""" +import ast, csv, glob, itertools, os, sys, tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCRIPT = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode-index') +csv.field_size_limit(10**9) +MISSING = object() + + +def load_ir_rows(): + """ir_rows() out of the script without running it (the script indexes a graph at import).""" + tree = ast.parse(open(SCRIPT, encoding='utf-8').read()) + fn = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'ir_rows'] + if not fn: + sys.exit('FAIL: axiomcode-index defines no ir_rows()') + ns = {} + exec(compile(ast.Module(body=fn, type_ignores=[]), SCRIPT, 'exec'), ns) + return ns['ir_rows'] + + +def dictreader(p): + with open(p, newline='', encoding='utf-8', errors='replace') as f: + return list(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)) + + +FIXTURES = { + 'plain': b'a\tb\tc\n1\t2\t3\n4\t5\t6\n', + 'quoted-tab': b'a\tb\tc\n"x\ty"\t2\t3\n1\t"p\tq"\t3\n', + 'quoted-newline': b'a\tb\tc\n"line one\nline two"\t2\t3\n4\t5\t6\n', + 'quoted-crlf-inside': b'a\tb\n"x\r\ny"\tz\n', + 'doubled-quotes': b'a\tb\n"@r.post(""/orders"")"\t1\n"""Fee"""\t2\n', + 'crlf': b'a\tb\tc\r\n1\t2\t3\r\n4\t5\t6\r\n', + 'lone-cr': b'a\tb\r1\t2\r3\t4\r', + 'mixed-endings': b'a\tb\n1\t2\r\n3\t4\r5\t6\n', + 'blank-lines': b'a\tb\n\n1\t2\n\r\n\n3\t4\n\n', + 'short-row': b'a\tb\tc\n1\t2\n1\n', + 'long-row': b'a\tb\n1\t2\t3\t4\n', + 'trailing-tab': b'a\tb\n1\t\n\t\n', + 'repeated-header': b'id\tname\tid\n1\tx\t2\n3\n', + 'blank-header': b'\n1\t2\n3\n', + 'empty': b'', + 'header-only': b'a\tb\n', + 'header-only-no-newline': b'a\tb', + 'no-final-newline': b'a\tb\n1\t2\n3\t4', + 'not-utf8': b'a\tb\n\xff\xfe\t\xc3\x28\n', + 'unicode-separators': 'a\tb\nx
y\tz\u0085w\n1\u000b2\t3\u000c4\n'.encode('utf-8'), + 'spaces': b'a\tb\n 1 \t 2 \n', +} + + +def main(argv): + ir_rows = load_ir_rows() + fails = checks = 0 + rows_compared = 0 + with tempfile.TemporaryDirectory() as w: + for name, data in FIXTURES.items(): + p = os.path.join(w, name + '.csv') + open(p, 'wb').write(data) + want, got = dictreader(p), list(ir_rows(p)) + checks += 1; rows_compared += len(want) + if want != got: + fails += 1 + print(f'FAIL {name}: csv.DictReader {want!r}\n ir_rows {got!r}') + # not vacuous: the fixtures hold rows, and the shapes the reader special-cases are among them + p = os.path.join(w, 'short-row.csv') + checks += 1 + if list(ir_rows(p)) != [{'a': '1', 'b': '2', 'c': None}, {'a': '1', 'b': None, 'c': None}]: + fails += 1; print(f'FAIL short-row: {list(ir_rows(p))!r}') + p = os.path.join(w, 'long-row.csv') + checks += 1 + if list(ir_rows(p)) != [{'a': '1', 'b': '2', None: ['3', '4']}]: + fails += 1; print(f'FAIL long-row: {list(ir_rows(p))!r}') + # a real IR: every file of every directory named on the command line + files = 0 + for d in argv: + for p in sorted(glob.glob(os.path.join(d, '*.csv'))): + files += 1; checks += 1 + n = 0 + with open(p, newline='', encoding='utf-8', errors='replace') as f: # streamed: an IR file can be 10^6 rows + for a, b in itertools.zip_longest(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE), ir_rows(p), fillvalue=MISSING): + n += 1 + if a != b: + fails += 1; print(f'FAIL {p} row {n}: csv.DictReader {a!r}\n ir_rows {b!r}'); break + rows_compared += n + if rows_compared < 20: + fails += 1; print(f'FAIL only {rows_compared} rows compared') + print(f"index_reader: {'ok' if not fails else 'FAILED'} ({checks - fails}/{checks} checks, {rows_compared} rows," + f" {len(FIXTURES)} fixtures{f', {files} IR files' if files else ''})") + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) From 4bf09a96294518c6d238ab2f5ab5cc0f4635c796 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:38:05 -0700 Subject: [PATCH 06/18] AXIOMCODE_REINDEX=1 rebuilds a graph whose files are unchanged The build's messages, SKILL.md and ax_fresh all describe AXIOMCODE_REINDEX=1 as a rebuild ("rebuilds it with this one", "to force"). It skipped the newer-build shortcut and the remembered languages, but both up-to-date checks still answered "graph up to date" whenever the files matched the file table, so it rebuilt nothing. Both checks now step aside when it is set. tests/refresh.py checks that `index` with AXIOMCODE_REINDEX=1 rebuilds over an unchanged tree; the existing check beside it, the same `index` without it, is the control. Refs #1835 --- .../axiomcode/skills/axiomcode/scripts/axiomcode-build | 6 ++++-- tests/refresh.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build index 7a52d595c..28ec84d31 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build @@ -316,14 +316,16 @@ if [ -f "$OUT/corrupt" ]; then echo "rebuilding: $(head -1 "$OUT/corrupt")"; fi if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && NEWER="$(AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" newer "$REPO")"; then echo "$NEWER"; exit 0 fi -if [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then +# AXIOMCODE_REINDEX=1 IS A REBUILD, as every message that names it promises ("rebuilds it with this one", "to force"): +# it skips both up-to-date answers below, which otherwise said "graph up to date" whenever the files matched the table. +if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then if [ "$(cat "$OUT/stamp")" != "$STAMP" ]; then echo "$STAMP" > "$OUT/stamp"; INDEXED_TREE="$OLD_INDEXED"; set_base "$OLD_INDEXED"; PREV=""; keep_base_graph; commit_base; fi set_base_files has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO" [ -f "$REPO/.axiomcode/engine" ] || printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine" || true echo "graph up to date (${LANGS//,/, }) at $OUT/graph.sqlite"; exit 0 fi -if [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then +if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO" python3 "$H/axiomcode-impact" --warm "$REPO" || true # export the graph's facts now (~9 s on 1,373 files): the first # query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once. diff --git a/tests/refresh.py b/tests/refresh.py index 2a02b7c6a..405029879 100644 --- a/tests/refresh.py +++ b/tests/refresh.py @@ -3,7 +3,7 @@ Each language gets a throwaway git repository with a real graph. Then, per language: - up to date `index` with nothing changed does not rebuild + up to date `index` with nothing changed does not rebuild; with AXIOMCODE_REINDEX=1 it does query an edit adds a function; a query verb, which starts the refresher and waits for it, finds it baseline `changed` answers the same before and after the refresh absorbed the edit: the refresh moves the graph, not the baseline edits are measured against. The answer must name the edit (a pass on two @@ -110,6 +110,13 @@ def check(ok, why, detail=''): check(m.get('refresh_reason') == 'axiomcode index' and m.get('refreshed_at', '').endswith('Z'), f'{lang}: the graph records when and why it was built', json.dumps(m)) t0 = time.time(); again = sh(repo, AX, 'index', '.', '--lang', lang, env=env); took = time.time() - t0 check('graph up to date' in again.stdout, f'{lang}: `index` with nothing changed does not rebuild ({took:.1f}s)', again.stdout + again.stderr) + # AXIOMCODE_REINDEX=1 rebuilds with nothing changed, as SKILL.md and the build's own messages promise; it used to + # answer "graph up to date" whenever the files matched the table. The control is the check above: the same + # `index` without it does not rebuild + built = meta().get('refreshed_at') + rx = sh(repo, AX, 'index', '.', '--lang', lang, env=dict(env, AXIOMCODE_REINDEX='1')) + check(rx.returncode == 0 and 'graph up to date' not in rx.stdout and 'building' in rx.stdout and meta().get('refreshed_at') != built, + f'{lang}: AXIOMCODE_REINDEX=1 `index` rebuilds although nothing changed', rx.stdout + rx.stderr) # ── built by an older axiomcode ──────────────────────────────────────────────────────────────────── # no file changed, but the table says another engine built the graph (what an update leaves behind): not up From cd0bf00c5277727cfb78d82ec4c68a9ebfbc678b Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:15:30 -0700 Subject: [PATCH 07/18] tests/README: list index_reader.py Refs #1835 --- tests/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/README.md b/tests/README.md index 06150d2fd..bd5377d2d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -26,6 +26,9 @@ One check needs no graph and is its own script: signature edit is reported before it lands with nothing upstream did; `--range` from a branch left behind its remote reads from the remote's fork (indexes a small project) + python3 tests/index_reader.py axiomcode-index's IR reader returns the same rows csv.DictReader did, on quoted tabs + and newlines, CRLF, a lone CR, a short or long row and invalid UTF-8; pass IR + directories to compare them too (no graph needed) python3 tests/refresh.py the graph refreshes itself after an edit in every language: a query sees the edit, `changed` answers the same before and after, a burst costs one rebuild and queries during it answer (#1305; builds real graphs, needs the engine) From c5d95a284ed8a96496c19371a7e833108f5a9f65 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:27:32 -0700 Subject: [PATCH 08/18] plugin: axiomcode-index decodes quoted IR fields, so a literal with quotes is found The parser writes the IR as RFC 4180 over a tab: EntityUtils.escapeTsv turns newlines, CRs and tabs into two-character escapes first, then quotes a field that holds a double quote and doubles the quote. axiomcode-index read it with QUOTE_NONE semantics, so the bundle decoded these fields and the index did not: a literal `Order "A-1" Is Late` was stored with doubled quotes, `impact` on that string said it appears in no literal, `impact KEY` printed 'a \""b\"" c', a Java annotation was stored as @Route("""/x"""), and a C# member read on a string literal was named `Length"`. ir_rows() now decodes a field that opens with a quote exactly as the bundle's readRfc4180 does (a doubled quote is one, the closing quote ends the field, a quote elsewhere is text). A line with no quote is still a plain tab split, so the reader costs the same. A quoted field left open at a line end takes the next line, as the bundle's reader does; escapeTsv never writes one. The hand-written decoders that ran on the quoted text go: decorations' unquoted() (Python decorator text starts with @, so it can no longer fire), and the C# literal un-doubling, replaced by cs_string_text(), which reads the C# token: regular, verbatim (a doubled quote is one) and raw (as many quotes close it as open it). On decoded text the old un-doubling read the raw string """"a "" b"""" as "a " b" and left a verbatim string's doubled quotes. C# verbatim and raw strings are now literals. ax_registration's '"""' replace stays: a Python triple-quoted decorator path still needs it. INDEX_VERSION 10. On four measured projects every table but literals, comments and decorations is identical, and those now match an index read with csv QUOTE_MINIMAL row for row. Tests: tests/index_reader.py holds ir_rows to csv.DictReader and to the built bundle reader, with the decoded values written out; tests/cases quoted-literal-text for python, java and csharp, with controls. Refs #1835 --- .../skills/axiomcode/scripts/axiomcode-index | 97 ++++++++---- tests/README.md | 8 +- .../csharp/quoted-literal-text/case.json | 28 ++++ .../csharp/quoted-literal-text/src/Orders.cs | 20 +++ .../quoted-literal-text/src/Orders.csproj | 1 + .../quoted-literal-text/src/RouteAttribute.cs | 8 + .../cases/java/quoted-literal-text/case.json | 24 +++ .../quoted-literal-text/src/app/Orders.java | 20 +++ .../quoted-literal-text/src/app/Route.java | 10 ++ .../python/quoted-literal-text/case.json | 23 +++ .../quoted-literal-text/src/app/__init__.py | 0 .../quoted-literal-text/src/app/orders.py | 22 +++ tests/index_reader.py | 138 +++++++++++++++--- 13 files changed, 348 insertions(+), 51 deletions(-) create mode 100644 tests/cases/csharp/quoted-literal-text/case.json create mode 100644 tests/cases/csharp/quoted-literal-text/src/Orders.cs create mode 100644 tests/cases/csharp/quoted-literal-text/src/Orders.csproj create mode 100644 tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs create mode 100644 tests/cases/java/quoted-literal-text/case.json create mode 100644 tests/cases/java/quoted-literal-text/src/app/Orders.java create mode 100644 tests/cases/java/quoted-literal-text/src/app/Route.java create mode 100644 tests/cases/python/quoted-literal-text/case.json create mode 100644 tests/cases/python/quoted-literal-text/src/app/__init__.py create mode 100644 tests/cases/python/quoted-literal-text/src/app/orders.py diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index index b5ddd518d..28ac9874b 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index @@ -21,14 +21,16 @@ positions — mirroring graph/bundle/languages.ts. When the IR is absent the ind come from methods/types alone, refs/literals/comments are empty, and index_meta says so. """ import os, re, sqlite3, sys, time, glob, collections, functools -INDEX_VERSION = '9' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; +INDEX_VERSION = '10' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; # v5: JavaScript fields owned by their class, computed-key members named by their key, anonymous class expressions named by their binding; # v6: TypeScript class-property arrows named after their field; # v7: data keys of module-level const objects declared, Object.freeze seen through; # v8: literals carry numbers and booleans with a kind column, and the name of the constant # a scalar initializes — a spec's numbers were the one thing the graph could not answer; # v9: the web graph (HTML + CSS) is indexed: pages, stylesheets, .classes, #ids, custom properties (var(--x)), - # @keyframes/@font-face/@layer/@container, selectors and inline scripts are symbols); the query + # @keyframes/@font-face/@layer/@container, selectors and inline scripts are symbols; + # v10: the IR's RFC 4180 quoting is decoded, so a literal, comment, decoration or name holding a `"` + # is stored as written (`a "b" c`, not `a ""b"" c`), and C# verbatim and raw strings are literals); the query # frontend re-indexes an older graph when the IR is still there REPO = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].startswith('-') else os.environ.get('AXIOMCODE_REPO') or '.') @@ -192,7 +194,7 @@ A = { # name elsewhere is a by-name reader whatever its qualifier says, so `_r.Limit` was listed under AuditOptions.Limit memberName=dict(role='edgeRole', value='MEMBER_NAME', parent='parentExpressionHash', id='csExpressionUniqueHash'), # the literal TYPE column is literalKind here, not literalType as in java/typescript/python - litKinds={'LITERAL'}, litType=('literalKind', {'STRING': 'string', 'INTEGER': 'number', 'REAL': 'number', 'BOOLEAN': 'bool'}), litValue='literalValue', litLinkRoles={'ROOT'}), + litKinds={'LITERAL'}, litType=('literalKind', {'STRING': 'string', 'VERBATIM_STRING': 'string', 'RAW_STRING': 'string', 'INTEGER': 'number', 'REAL': 'number', 'BOOLEAN': 'bool'}), litValue='literalValue', litLinkRoles={'ROOT'}), comments=dict(file='all-csharp-comments.csv', text='commentText', kind='commentKind', line='startLine', fileVia=('modules', 'csModuleLinkHash')), # type references carry no module link of their own. The owner is a type only for a base-list reference; for # `M()`, `new T()`, a parameter, a return or a field it is the expression, method, parameter or field, so @@ -275,25 +277,61 @@ def rows(name): return ir_rows(p) def ir_rows(p): - """The rows of one IR file as dicts keyed by its header, EXACTLY as csv.DictReader(f, delimiter='\\t', - quoting=csv.QUOTE_NONE) over open(p, newline='') reads them, at about half the cost (the expressions file is - most of this script's time on a large repository). That reader treats a quote as an ordinary character, so a row - is one line and a field is the text between tabs: a line ends at \\n, \\r or \\r\\n (newline='' splits on all - three and keeps them, and each ends a record), a blank line is skipped, a short row has None for its missing - columns and a long one keeps the rest as a list under the key None. The writer's RFC 4180 quoting is NOT undone - here, as it was not before (see unquoted() below). tests/index_reader.py holds the two to the same dicts.""" + """The rows of one IR file as dicts keyed by its header, its fields DECODED the way the bundle's reader + (graph/bundle/csv.ts readRfc4180) decodes them: the parser writes RFC 4180 over a tab, so a field that STARTS with + a double quote is quoted, `""` inside it is one quote, and the closing quote ends it (what follows the close, up + to the tab, is kept as written); a quote anywhere else in a field is an ordinary character. The writer + (EntityUtils.escapeTsv) turns a newline, a CR and a tab into the two-character escapes BEFORE it quotes, and quotes + only for a `"`, so a quoted field it wrote never holds a tab or a line break and a row is one line. Reading + QUOTE_NONE stored `Order "A-1"` as `Order ""A-1""`, so `impact` said the string was in no literal (#1835). + The rest is csv.DictReader(f, delimiter='\\t') over open(p, newline=''), at about half its cost (the expressions + file is most of this script's time on a large repository): a line ends at \\n, \\r or \\r\\n, a blank line is + skipped, a short row has None for its missing columns and a long one keeps the rest as a list under the key None. + A line with no quote in it is split on tabs and nothing else: that is nearly every line, and a test for a quote + that OPENS a field cost more than the decoding it saved. A quoted field left open at the end of a line, which this + writer does not produce, takes the line break and the next line, as the bundle's reader does. + tests/index_reader.py holds this to both readers.""" with open(p, newline='', encoding='utf-8', errors='replace') as f: - head = f.readline() + def fields(line): + # a line holding a quote: split on tabs, then decode each field that opens with one, rejoined across the tabs + # (and, if it is still open at the end of the line, the line breaks) it holds + out = []; parts = line.split('\t'); i = 0 + while True: + if i == len(parts): return out + s = parts[i]; i += 1 + if s[:1] != '"': out.append(s); continue + buf = []; k = 1 + while True: + j = s.find('"', k) + if j >= 0: + buf.append(s[k:j]) + if s[j + 1:j + 2] == '"': buf.append('"'); k = j + 2; continue + buf.append(s[j + 1:]); break # the closing quote; the rest of the field as written + buf.append(s[k:]); k = 0 + if i < len(parts): buf.append('\t'); s = parts[i]; i += 1; continue + buf.append(line_end[0]); nxt = f.readline() # still open at the end of the line: the break is text + if not nxt: break + body = nxt.rstrip('\r\n'); line_end[0] = nxt[len(body):] + parts = body.split('\t'); s = parts[0]; i = 1 + out.append(''.join(buf)) + def decoded(line, raw): + line_end[0] = raw[len(line):] + v = fields(line) + if v[-1][-1:] == '\r': v[-1] = v[-1][:-1] # as the bundle's reader drops a CR before the newline + return v + line_end = [''] + head = raw = f.readline() if not head: return if head[-1:] == '\n': head = head[:-1] if head[-1:] == '\r': head = head[:-1] - keys = head.split('\t') if head else [] + keys = (head.split('\t') if '"' not in head else decoded(head, raw)) if head else [] n = len(keys) - for line in f: + for raw in f: + line = raw if line[-1:] == '\n': line = line[:-1] if line[-1:] == '\r': line = line[:-1] if not line: continue - v = line.split('\t') + v = line.split('\t') if '"' not in line else decoded(line, raw) d = dict(zip(keys, v)) if len(v) != n: if len(v) > n: d[None] = v[n:] @@ -699,6 +737,20 @@ for name, od, v, k in data_keys: c.executemany("INSERT INTO symbols VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", sym) # ── references, literals, comments ─────────────────────────────────────────────────────────────── +def cs_string_text(v): + """The text of a C# string token, as ir_rows hands it over (the IR's RFC 4180 quoting already undone): `"x"` is x + with its escapes as written, as Java's and Python's are; a verbatim `@"x"` is x with a doubled quote made one, the + language's own escape; a raw string is x, closed by as many quotes as opened it (three or more: a regular string + cannot open with three, `""` is the whole empty string); a `$` is dropped. Before ir_rows decoded the IR, a C# + literal holding a quote was un-doubled here by hand; on decoded text that is a second decoding, which reads the + raw `\"\"\"\"a \"\" b\"\"\"\"` as `"a " b"`, and a verbatim string's doubled quote was never made one.""" + p = 0 + while p < len(v) and v[p] in '@$': p += 1 + t = v[p:] + q = len(t) - len(t.lstrip('"')) + if q >= 3 and len(t) >= 2 * q and t.endswith('"' * q): return t[q:-q] + if len(t) >= 2 and t[0] == '"' == t[-1]: return t[1:-1].replace('""', '"') if '@' in v[:p] else t[1:-1] + return v e = A['expr']; refs = []; lits = [] namecol = e['name'] if isinstance(e['name'], str) else dict(x.split(':') for x in e['name']) mname = e.get('memberName'); child_ek = {} @@ -727,11 +779,8 @@ for r in rows(e['file']): kind = e['litType'][1][r.get(e['litType'][0])] # 'string' | 'number' | 'bool' across every language v = (r.get(e['litValue']) or '') if kind == 'string': - # C# keeps the source token, and the IR writer RFC4180-quotes a field holding a double quote, so `"Fee"` arrived - # as `"""Fee"""` and was stored `""Fee""`: no key, route or member name written in a C# string ever matched - if LANG == 'csharp' and len(v) > 1 and v[0] == '"' == v[-1] and '""' in v: v = v[1:-1].replace('""', '"') - if LANG == 'csharp' and len(v) > 2 and v[0] in '@$' and v[-1] == '"': v = v.lstrip('@$') # @"verbatim", $"interpolated" - if len(v) >= 2 and v[0] in '\'"`' and v[-1] == v[0]: v = v[1:-1] # JavaScript keeps the quotes + if LANG == 'csharp': v = cs_string_text(v) # C# keeps the source token: "x", @"x", """x""" + elif len(v) >= 2 and v[0] in '\'"`' and v[-1] == v[0]: v = v[1:-1] # JavaScript keeps the quotes if v: lits.append((v[:200], file_of(r, e), int(r.get(e['line']) or 0), kind, r.get('edgeRole', ''))) # a scalar's NAME: `MAX_ITEMS = 5` — the one literal on a const/variable/field declaration's line IS its value, # so the row gets the symbol's display and "what is MAX_ITEMS" has an answer with a source location. Strictly: @@ -767,13 +816,6 @@ if tr: c.executemany("INSERT INTO type_refs VALUES (?,?,?,?,?)", ((r.get(tr['name'], '').split('.')[-1].split('<')[0], file_of(r, tr), int(r.get(tr['line']) or 0), r.get(tr['context'], ''), r.get(tr['ownerKind'], '')) for r in rows(tr['file']) if r.get(tr['name']))) except (KeyError, ValueError): pass # decorations: @Annotation(args) / @decorator(args) on a method or type — a route, a bean name, a validator, a marker -def unquoted(v): - """The IR is tab-separated, but the writer still RFC4180-quotes a field that contains a double quote and the - reader is QUOTE_NONE, so `@router.post("/orders")` arrives as `"@router.post(""/orders"")"`. Every decoration - whose argument is a double-quoted string is in that shape — which is exactly the decorations that carry a KEY - (a route path, a command name, a topic, a validated field), so everything downstream that reads a key out of - the text read the quoting instead: `@exporter("csv")` yielded `@exporter(` and `)` and never `csv`.""" - return v[1:-1].replace('""', '"') if len(v) > 1 and v[0] == '"' == v[-1] and '""' in v else v dec = A.get('decorations') if dec: args = collections.defaultdict(list) @@ -784,7 +826,8 @@ if dec: for r in rows(dec['file']): nm = r.get(dec['name'], '') if not nm: continue - text = unquoted(r.get(dec['text'], '')) if dec.get('text') else ('@' + nm + ('(' + ', '.join(args.get(r.get(dec['id'], ''), [])) + ')' if args.get(r.get(dec['id'], '')) else '')) + # the text as written (ir_rows undoes the writer's RFC 4180 quoting, so `@exporter("csv")` arrives as that) + text = r.get(dec['text'], '') if dec.get('text') else ('@' + nm + ('(' + ', '.join(args.get(r.get(dec['id'], ''), [])) + ')' if args.get(r.get(dec['id'], '')) else '')) try: drows.append((r.get(dec['owner'], ''), nm, text[:200], file_of(r, dec), int(r.get(dec['line']) or 0))) except ValueError: pass c.executemany("INSERT INTO decorations VALUES (?,?,?,?,?)", drows) diff --git a/tests/README.md b/tests/README.md index bd5377d2d..a9cac2520 100644 --- a/tests/README.md +++ b/tests/README.md @@ -26,9 +26,11 @@ One check needs no graph and is its own script: signature edit is reported before it lands with nothing upstream did; `--range` from a branch left behind its remote reads from the remote's fork (indexes a small project) - python3 tests/index_reader.py axiomcode-index's IR reader returns the same rows csv.DictReader did, on quoted tabs - and newlines, CRLF, a lone CR, a short or long row and invalid UTF-8; pass IR - directories to compare them too (no graph needed) + python3 tests/index_reader.py axiomcode-index's IR reader decodes the IR's RFC 4180 quoting as csv.DictReader and + the bundle's reader do (a literal, decoration or name holding a `"`, a C# raw + string), and agrees with them on quoted tabs and newlines, CRLF, a lone CR, a + short or long row and invalid UTF-8; pass IR directories to compare them too + (no graph needed; the bundle half runs node on dist/bundle/csv.js) python3 tests/refresh.py the graph refreshes itself after an edit in every language: a query sees the edit, `changed` answers the same before and after, a burst costs one rebuild and queries during it answer (#1305; builds real graphs, needs the engine) diff --git a/tests/cases/csharp/quoted-literal-text/case.json b/tests/cases/csharp/quoted-literal-text/case.json new file mode 100644 index 000000000..c2ae33141 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/case.json @@ -0,0 +1,28 @@ +{"lang": "csharp", "src": "src", + "checks": [ + {"why": "a member read on a string literal is named by the member: the IR quotes the expression (RFC 4180) and the index kept a stray quote, `Length\"` (#1835)", + "run": ["impact", "Length"], "expect_error": true, + "want": ["[by name] Orders.LabelWidth"]}, + {"why": "a raw string is a literal, and its value is the text between the quotes that open and close it", + "run": ["impact", "Orders.RawKey"], + "want": ["const Orders.RawKey = 'order.created'"]}, + {"why": "a raw string holding a quote keeps it, once", + "run": ["impact", "Orders.Raw"], + "want": ["const Orders.Raw = 'raw \"x\" here'"]}, + {"why": "a raw string opened by four quotes may hold two, and both are kept: un-doubling quotes by hand on the decoded IR read it as `\"a \" b\"`", + "run": ["impact", "Orders.Raw4"], + "want": ["const Orders.Raw4 = 'a \"\" b'"]}, + {"why": "a verbatim string's doubled quote is one quote", + "run": ["impact", "Orders.Verbatim"], + "want": ["const Orders.Verbatim = 'say \"hi\" now'"]}, + {"why": "control: a regular string holding an escaped quote is printed as written, in single quotes", + "run": ["impact", "Orders.Key"], + "want": ["const Orders.Key = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "control: a string literal holding an escaped quote is found", + "run": ["impact", "\"Order \\\"Foo\\\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "Orders.Plain"], + "want": ["const Orders.Plain = 'a plain value'"]} + ]} diff --git a/tests/cases/csharp/quoted-literal-text/src/Orders.cs b/tests/cases/csharp/quoted-literal-text/src/Orders.cs new file mode 100644 index 000000000..474829f26 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/Orders.cs @@ -0,0 +1,20 @@ +namespace App; + +public class Orders +{ + public const string Key = "a \"b\" c"; + public const string Plain = "a plain value"; + public const string Verbatim = @"say ""hi"" now"; + public const string RawKey = """order.created"""; + public const string Raw = """raw "x" here"""; + public const string Raw4 = """"a "" b""""; + + string Send(string subject) => subject; + + [Route("/orders/{code}")] + public string NotifyLate() => Send("Order \"Foo\" Is Late"); + + public string Resolve() => Send("The order has shipped"); + + public int LabelWidth() => "header".Length; +} diff --git a/tests/cases/csharp/quoted-literal-text/src/Orders.csproj b/tests/cases/csharp/quoted-literal-text/src/Orders.csproj new file mode 100644 index 000000000..d3b2f0306 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/Orders.csproj @@ -0,0 +1 @@ +net8.0 diff --git a/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs b/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs new file mode 100644 index 000000000..5c155c401 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs @@ -0,0 +1,8 @@ +namespace App; + +[System.AttributeUsage(System.AttributeTargets.Method)] +public class RouteAttribute : System.Attribute +{ + public RouteAttribute(string path) { Path = path; } + public string Path { get; } +} diff --git a/tests/cases/java/quoted-literal-text/case.json b/tests/cases/java/quoted-literal-text/case.json new file mode 100644 index 000000000..b771009ca --- /dev/null +++ b/tests/cases/java/quoted-literal-text/case.json @@ -0,0 +1,24 @@ +{"lang": "java", "src": "src", + "checks": [ + {"why": "a string literal holding an escaped double quote is found: the IR quotes such a field (RFC 4180) and the index stored it doubled, so `impact` said no literal held it (#1835)", + "run": ["impact", "\"Order \\\"Foo\\\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "a constant's value holding a quote is printed as written, in single quotes, not with each quote doubled", + "run": ["impact", "Orders.KEY"], + "want": ["const Orders.KEY = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "an annotation argument holding a quote is found as written: the decoration was stored with every quote doubled", + "run": ["impact", "\"/say \\\"hi\\\"\""], + "want": ["(0 literal site(s), 1 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "Orders.PLAIN"], + "want": ["const Orders.PLAIN = 'a plain value'"]}, + {"why": "control: a literal with no quote in it is found as before", + "run": ["impact", "\"The order has shipped\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "control: an annotation's quoted path still binds the method and registers it under the path", + "run": ["impact", "Orders.notifyLate"], + "want": ["registered by @Route \"/orders/{code}\""]} + ]} diff --git a/tests/cases/java/quoted-literal-text/src/app/Orders.java b/tests/cases/java/quoted-literal-text/src/app/Orders.java new file mode 100644 index 000000000..f8c987bfd --- /dev/null +++ b/tests/cases/java/quoted-literal-text/src/app/Orders.java @@ -0,0 +1,20 @@ +package app; + +public class Orders { + static final String KEY = "a \"b\" c"; + static final String PLAIN = "a plain value"; + + String send(String subject) { + return subject; + } + + @Route("/orders/{code}") + public String notifyLate() { + return send("Order \"Foo\" Is Late"); + } + + @Route("/say \"hi\"") + public String greet() { + return send("The order has shipped"); + } +} diff --git a/tests/cases/java/quoted-literal-text/src/app/Route.java b/tests/cases/java/quoted-literal-text/src/app/Route.java new file mode 100644 index 000000000..4bc1364fd --- /dev/null +++ b/tests/cases/java/quoted-literal-text/src/app/Route.java @@ -0,0 +1,10 @@ +package app; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +public @interface Route { + String value() default ""; + String name() default ""; +} diff --git a/tests/cases/python/quoted-literal-text/case.json b/tests/cases/python/quoted-literal-text/case.json new file mode 100644 index 000000000..04c622f2d --- /dev/null +++ b/tests/cases/python/quoted-literal-text/case.json @@ -0,0 +1,23 @@ +{"lang": "python", "src": "src", + "checks": [ + {"why": "a string literal holding a double quote is found: the IR quotes such a field (RFC 4180) and the index stored it doubled, so `impact` said no literal held it (#1835)", + "run": ["impact", "\"Order \"Foo\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "a constant's value holding a quote is printed as written, in single quotes, not with each quote doubled", + "run": ["impact", "KEY"], + "want": ["const KEY = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "PLAIN"], + "want": ["const PLAIN = 'a plain value'"]}, + {"why": "control: a literal with no quote in it is found as before", + "run": ["impact", "\"The order has shipped\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "a decorator's quoted path reads cleanly: it binds the function and registers it under the path", + "run": ["impact", "\"/orders/{code}\""], + "want": ["1 decoration site(s)", "binds to it through @route"]}, + {"why": "a decorator path written as a triple-quoted string is read as the path", + "run": ["impact", "notify_triple"], + "want": ["registered by @route \"/triple/{code}\""]} + ]} diff --git a/tests/cases/python/quoted-literal-text/src/app/__init__.py b/tests/cases/python/quoted-literal-text/src/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cases/python/quoted-literal-text/src/app/orders.py b/tests/cases/python/quoted-literal-text/src/app/orders.py new file mode 100644 index 000000000..85a5aadbf --- /dev/null +++ b/tests/cases/python/quoted-literal-text/src/app/orders.py @@ -0,0 +1,22 @@ +KEY = "a \"b\" c" +PLAIN = "a plain value" + + +def route(path): + def deco(f): + return f + return deco + + +def send(subject): + return subject + + +@route("/orders/{code}") +def notify(): + return send('Order "Foo" Is Late') + + +@route("""/triple/{code}""") +def notify_triple(): + return send("The order has shipped") diff --git a/tests/index_reader.py b/tests/index_reader.py index bc43878aa..3f6508eac 100644 --- a/tests/index_reader.py +++ b/tests/index_reader.py @@ -1,21 +1,33 @@ #!/usr/bin/env python3 -"""tests/index_reader.py: axiomcode-index reads the IR exactly as csv.DictReader did, row for row and key for key. - -axiomcode-index reads every IR row through ir_rows(), a line splitter that replaced -csv.DictReader(f, delimiter='\\t', quoting=csv.QUOTE_NONE) because the DictReader was most of the index's time on a -large repository. The index's tables are made of these dicts, so any difference is a different graph. Each fixture is -read by both and the lists of dicts must be equal: the shapes where a line splitter and the csv module could part -are a quoted field holding a tab or a newline (QUOTE_NONE does not unquote, so both split it), CRLF, a lone CR, a -blank line, a short row (None for the missing columns), a long row (the rest under the key None), a repeated header -name, a blank header, an empty file, a header with no rows, no final newline, bytes that are not UTF-8, and the -Unicode line separators that str.splitlines() would split on and the csv module does not. - - python3 tests/index_reader.py [IR_DIR ...] each IR_DIR: every *.csv in it is compared as well +"""tests/index_reader.py: axiomcode-index reads the IR as the bundle does, row for row and key for key. + +axiomcode-index reads every IR row through ir_rows(), a line splitter that replaced csv.DictReader because the +DictReader was most of the index's time on a large repository. The index's tables are made of these dicts, so any +difference is a different graph. The parser writes RFC 4180 over a tab (EntityUtils.escapeTsv quotes a field that +holds a `"` and doubles the quote), and ir_rows decodes it: read QUOTE_NONE, as it once was, the literal +`Order "Foo"` was stored `Order ""Foo""` and `impact` said no literal held it (#1835). + +Each fixture is read three ways, and they must agree: + - csv.DictReader(f, delimiter='\\t') over open(p, newline=''), RFC 4180 quoting on: the same list of dicts; + - the bundle's own reader, graph/bundle/csv.ts readRfc4180 (the built dist/bundle/csv.js, run with node): its rows, + zipped with the header, must give the same dicts. A lone CR is left out of this half on purpose: ir_rows keeps + the csv module's rule that it ends a line, and the bundle splits on \\n only. A blank line is no row in either + (the bundle yields one empty field for it, which is dropped here); + - for the shapes the bug was about, the decoded value itself, written out, so two readers agreeing on a wrong value + is not a pass: a literal with quotes, a decoration with a quoted path, a C# raw string `\"\"\"x\"\"\"`, a C# + member access on a string (`"x".Length`), a quote inside a field, and the control, a field with no quote. +The other shapes are where a line splitter and the csv module could part: a quoted field holding a tab or a newline, +CRLF, a lone CR, a blank line, a short row (None for the missing columns), a long row (the rest under the key None), +a repeated header name, a blank header, an empty file, a header with no rows, no final newline, bytes that are not +UTF-8, and the Unicode line separators that str.splitlines() would split on and the csv module does not. + + python3 tests/index_reader.py [IR_DIR ...] each IR_DIR: every *.csv in it is compared with csv.DictReader too """ -import ast, csv, glob, itertools, os, sys, tempfile +import ast, csv, glob, itertools, json, os, shutil, subprocess, sys, tempfile ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SCRIPT = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode-index') +BUNDLE_CSV = os.path.join(ROOT, 'dist', 'bundle', 'csv.js') # graph/bundle/csv.ts, built csv.field_size_limit(10**9) MISSING = object() @@ -33,7 +45,46 @@ def load_ir_rows(): def dictreader(p): with open(p, newline='', encoding='utf-8', errors='replace') as f: - return list(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)) + return list(csv.DictReader(f, delimiter='\t')) + + +BUNDLE_JS = r''' +const { readRfc4180 } = require(process.argv[1]); +(async () => { + const out = {}; + for (const p of process.argv.slice(2)) { out[p] = []; for await (const r of readRfc4180(p)) out[p].push(r); } + process.stdout.write(JSON.stringify(out)); +})(); +''' + + +def bundle_rows(paths): + """{path: the bundle reader's rows}, or None when node or the built reader is not there.""" + node = shutil.which('node') + if not node or not os.path.exists(BUNDLE_CSV): + return None + r = subprocess.run([node, '-e', BUNDLE_JS, BUNDLE_CSV] + paths, capture_output=True, text=True, encoding='utf-8') + if r.returncode: + sys.exit(f'FAIL: the bundle reader did not run: {r.stderr[-400:]}') + return json.loads(r.stdout) + + +def as_dicts(rows): + """The bundle's rows as ir_rows' dicts: the first row is the header, a blank line (one empty field) is no row.""" + if not rows: + return [] + keys = rows[0] if rows[0] != [''] else [] + n, out = len(keys), [] + for v in rows[1:]: + if v == ['']: + continue + d = dict(zip(keys, v)) + if len(v) > n: + d[None] = v[n:] + for k in keys[len(v):]: + d[k] = None + out.append(d) + return out FIXTURES = { @@ -56,30 +107,71 @@ def dictreader(p): 'header-only-no-newline': b'a\tb', 'no-final-newline': b'a\tb\n1\t2\n3\t4', 'not-utf8': b'a\tb\n\xff\xfe\t\xc3\x28\n', - 'unicode-separators': 'a\tb\nx
y\tz\u0085w\n1\u000b2\t3\u000c4\n'.encode('utf-8'), + 'unicode-separators': 'a\tb\nx y\tz\u0085w\n1\u000b2\t3\u000c4\n'.encode('utf-8'), 'spaces': b'a\tb\n 1 \t 2 \n', + # the quoting EntityUtils.escapeTsv writes, as the parser wrote it for each shape + 'quoted-literal': b'value\tkind\n"Order ""Foo"" Is Late"\tstring\n', + 'quoted-decoration': b'name\ttext\nRequestMapping\t"@RequestMapping(""/a/{b}"")"\nRequestMapping\t"""/a/{b}"""\n', + 'csharp-raw-string': b'value\tkind\n"""""""x"""""""\tstring\n', + 'csharp-string-member': b'name\tkind\n"""x"".Length"\tMEMBER_ACCESS\n', + 'plain-control': b'value\tkind\nThe order has shipped\tstring\n', + # what RFC 4180 says about a quote that does not open a field, and about text after the closing quote + 'quote-inside-field': b'a\tb\tc\nsay "hi"\tx"\ty"z"\n', + 'text-after-close': b'a\tb\n"ab"cd\t"x""y"z\n', + 'empty-quoted': b'a\tb\tc\n""\t""""\t\n', + 'quoted-crlf-row': b'a\tb\r\n"x ""y"""\t2\r\n', + 'quoted-header': b'"a"\t"b ""c"""\n1\t2\n', + 'quoted-open-at-eof': b'a\tb\n"x\ty\n', +} + +# the decoded values, written out +DECODED = { + 'quoted-literal': [{'value': 'Order "Foo" Is Late', 'kind': 'string'}], + 'quoted-decoration': [{'name': 'RequestMapping', 'text': '@RequestMapping("/a/{b}")'}, + {'name': 'RequestMapping', 'text': '"/a/{b}"'}], + 'csharp-raw-string': [{'value': '"""x"""', 'kind': 'string'}], + 'csharp-string-member': [{'name': '"x".Length', 'kind': 'MEMBER_ACCESS'}], + 'plain-control': [{'value': 'The order has shipped', 'kind': 'string'}], + 'quote-inside-field': [{'a': 'say "hi"', 'b': 'x"', 'c': 'y"z"'}], + 'quoted-open-at-eof': [{'a': 'x\ty\n', 'b': None}], + 'text-after-close': [{'a': 'abcd', 'b': 'x"yz'}], + 'quoted-header': [{'a': '1', 'b "c"': '2'}], } +# a lone CR ends an ir_rows line and is text to the bundle's reader (the parser never writes one: escapeTsv escapes it) +NOT_BUNDLE = {'lone-cr', 'mixed-endings'} def main(argv): ir_rows = load_ir_rows() fails = checks = 0 - rows_compared = 0 + rows_compared = bundle_checked = 0 with tempfile.TemporaryDirectory() as w: + paths = {} for name, data in FIXTURES.items(): - p = os.path.join(w, name + '.csv') + p = paths[name] = os.path.join(w, name + '.csv') open(p, 'wb').write(data) + bundle = bundle_rows(list(paths.values())) + for name, p in paths.items(): want, got = dictreader(p), list(ir_rows(p)) checks += 1; rows_compared += len(want) if want != got: fails += 1 print(f'FAIL {name}: csv.DictReader {want!r}\n ir_rows {got!r}') + if name in DECODED: + checks += 1 + if got != DECODED[name]: + fails += 1; print(f'FAIL {name}: decoded {got!r}\n want {DECODED[name]!r}') + if bundle is not None and name not in NOT_BUNDLE: + checks += 1; bundle_checked += 1 + b = as_dicts(bundle[p]) + if b != got: + fails += 1; print(f'FAIL {name}: bundle readRfc4180 {b!r}\n ir_rows {got!r}') # not vacuous: the fixtures hold rows, and the shapes the reader special-cases are among them - p = os.path.join(w, 'short-row.csv') + p = paths['short-row'] checks += 1 if list(ir_rows(p)) != [{'a': '1', 'b': '2', 'c': None}, {'a': '1', 'b': None, 'c': None}]: fails += 1; print(f'FAIL short-row: {list(ir_rows(p))!r}') - p = os.path.join(w, 'long-row.csv') + p = paths['long-row'] checks += 1 if list(ir_rows(p)) != [{'a': '1', 'b': '2', None: ['3', '4']}]: fails += 1; print(f'FAIL long-row: {list(ir_rows(p))!r}') @@ -90,15 +182,19 @@ def main(argv): files += 1; checks += 1 n = 0 with open(p, newline='', encoding='utf-8', errors='replace') as f: # streamed: an IR file can be 10^6 rows - for a, b in itertools.zip_longest(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE), ir_rows(p), fillvalue=MISSING): + for a, b in itertools.zip_longest(csv.DictReader(f, delimiter='\t'), ir_rows(p), fillvalue=MISSING): n += 1 if a != b: fails += 1; print(f'FAIL {p} row {n}: csv.DictReader {a!r}\n ir_rows {b!r}'); break rows_compared += n if rows_compared < 20: fails += 1; print(f'FAIL only {rows_compared} rows compared') + if bundle is None: + print('index_reader: the bundle half was NOT run (no node on PATH, or dist/bundle/csv.js is not built)') + elif bundle_checked < len(FIXTURES) - len(NOT_BUNDLE): + fails += 1; print(f'FAIL the bundle reader was compared on only {bundle_checked} fixtures') print(f"index_reader: {'ok' if not fails else 'FAILED'} ({checks - fails}/{checks} checks, {rows_compared} rows," - f" {len(FIXTURES)} fixtures{f', {files} IR files' if files else ''})") + f" {len(FIXTURES)} fixtures, {bundle_checked} against the bundle reader{f', {files} IR files' if files else ''})") return 1 if fails else 0 From 80430553fb9cd01270cf03021758baecac66563d Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:26:17 -0700 Subject: [PATCH 09/18] python: a method handed over as a value reaches its target A bound or unbound method or a function named as a VALUE and not called (executor.submit(self._work, x), Thread(target=svc.run), sorted(xs, key=self.rank), [h.on_event], partial(svc.cancel, oid), view_func= views.index, return self._handle) had no caller in the Python graph, so impact OrderService.cancel did not name the function that queued it and path said the two were independent. The engine already resolved what each of these expressions denotes; it dropped the answer because no call wrapped it. call-edge-generation/method-reference.dl emits it as a call_chain_edge with kind METHOD_REFERENCE and tier callback_registered, the tier the other front ends use for a function handed over, so answers say "handed over as a value" and never "calls it". Only what is resolved: a typed receiver through the MRO, a class's unbound method, a scoped name, a module member. Refused: an untyped receiver (stays as unresolved as thing.cancel() is), a flowed value (a parameter, a name or attribute assigned from an expression), a property read, a decorator application, the argument of a one-dunder builtin (str(f)), and a comparison operand. Measured on three open-source Python projects used for tuning: 3-4% more resolved caller->callee pairs, and 20 of 20 sampled new edges right after the guards (17 of 20 before the flowed-value guard). On three held-out projects, graded blind by a reviewer who did not tune it: 39 of 40 sampled new edges right (Wilson 95% interval 0.87-1.00). The one miss is a function handed to an unregistration call; the only repeated wrong shape left is a function passed to inspect.signature, about 1% of the new edges. The engine goldens gain only reference rows; the torture oracle scorecard is unchanged. Refs #1832 --- graph/bundle/SCHEMA.md | 6 +- graph/bundle/schema.ts | 6 +- .../call-edge-generation/method-reference.dl | 123 ++++++++++ .../test/python/expected/01-plain-calls.edges | 1 + .../test/python/expected/01-plain-calls.tiers | 8 +- .../test/python/expected/07-decorators.edges | 5 + .../test/python/expected/07-decorators.tiers | 8 +- graph/test/python/expected/08-callables.edges | 2 + graph/test/python/expected/08-callables.tiers | 8 +- .../python/expected/09-args-forwarding.edges | 1 + .../python/expected/09-args-forwarding.tiers | 8 +- .../test/python/expected/12-blind-spots.edges | 1 + .../test/python/expected/12-blind-spots.tiers | 8 +- .../expected/17-rebound-metaclass-slot.edges | 1 + .../expected/17-rebound-metaclass-slot.tiers | 8 +- .../expected/21-url-and-signal-dispatch.edges | 7 + .../expected/21-url-and-signal-dispatch.tiers | 8 +- .../python/expected/22-di-providers.edges | 6 + .../python/expected/22-di-providers.tiers | 8 +- .../25-imported-decorated-names.edges | 2 + .../25-imported-decorated-names.tiers | 8 +- .../expected/27-returned-expression.edges | 3 + .../expected/27-returned-expression.tiers | 8 +- .../python/expected/28-assigned-members.edges | 2 + .../python/expected/28-assigned-members.tiers | 8 +- .../python/expected/33-url-view-forms.edges | 10 + .../python/expected/33-url-view-forms.tiers | 8 +- .../36-call-through-model-decorators.edges | 1 + .../36-call-through-model-decorators.tiers | 8 +- .../python/expected/37-model-signals.edges | 1 + .../python/expected/37-model-signals.tiers | 8 +- .../test/python/expected/38-app-wiring.edges | 6 + .../test/python/expected/38-app-wiring.tiers | 8 +- .../39-model-construction-hooks.edges | 2 + .../39-model-construction-hooks.tiers | 8 +- .../python/expected/41-di-marker-forms.edges | 8 + .../python/expected/41-di-marker-forms.tiers | 8 +- .../python/expected/42-signal-forms.edges | 1 + .../python/expected/42-signal-forms.tiers | 8 +- .../expected/project-name-collision.tiers | 8 +- .../project-two-service-fastapi.tiers | 8 +- graph/test/python/tools/literal_gate.py | 6 +- .../python/torture/expected/torture.edges | 5 + .../skills/axiomcode/scripts/ax_edges.py | 2 +- .../case.json | 230 ++++++++++++++++++ .../orders.py | 163 +++++++++++++ .../tests/test_ledger.py | 5 + .../tests/test_queue.py | 9 + 48 files changed, 708 insertions(+), 67 deletions(-) create mode 100644 graph/python/engine/call-edge-generation/method-reference.dl create mode 100644 tests/cases/python/method-reference-reaches-its-target/case.json create mode 100644 tests/cases/python/method-reference-reaches-its-target/orders.py create mode 100644 tests/cases/python/method-reference-reaches-its-target/tests/test_ledger.py create mode 100644 tests/cases/python/method-reference-reaches-its-target/tests/test_queue.py diff --git a/graph/bundle/SCHEMA.md b/graph/bundle/SCHEMA.md index 6b33b221c..6cd2c4bd2 100644 --- a/graph/bundle/SCHEMA.md +++ b/graph/bundle/SCHEMA.md @@ -442,7 +442,7 @@ One row per place a call is written (or, for a synthesised edge, the construct t - **typescript** — end_line / end_column come from the expression row; the call-site row itself records only the start. - **javascript** — caller_id is the parser's enclosing method, or the module initializer for top-level code. end_line / end_column come from the expression row. `require()` is a module edge, not a call site. - **typescript** — PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor's name is on the callee's methods row. Filter them out with kind NOT IN (…) when counting calls. -- **python** — PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. +- **python** — PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METHOD_REFERENCE, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. - **python** — The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class's TYPE hash for METACLASS_CREATION, positioned at the class declaration. ### `call_edges` @@ -541,6 +541,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | `SUBSCRIPT_PROTOCOL` | python | `x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver's class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call. | | `BUILTIN_PROTOCOL` | python | `repr(x)`, `str(x)`, `len(x)`, `hash(x)`, `bool(x)`, `iter(x)`, `next(x)`, `abs(x)`, `format(x)` and `reversed(x)` run the matching dunder of the argument's class (str falls back to `__repr__`, bool to `__len__`). The written call is to the builtin; this edge is the dunder it runs. The site is the argument expression. Its own kind so it is never counted as a written call. | | `OPERATOR_PROTOCOL` | python | An operator runs a dunder of its operands: `a + b` the left operand's `__add__` and the right one's `__radd__`, a comparison the left slot and the right one's mirror (`a < b` -> `b.__gt__`, `!=` falls back to `__eq__`), `x in c` the right operand's `__contains__` (or `__iter__`), `-a`/`+a`/`~a` its `__neg__`/`__pos__`/`__invert__`. No written call; the site is the operator expression. | +| `METHOD_REFERENCE` | python | A method or function READ AS A VALUE, not called: an argument (`pool.submit(self.work, x)`, `map(Cls.convert, rows)`, `partial(svc.cancel, oid)`), a keyword (`Thread(target=svc.run)`), an element, a stored or returned value. The target is what the expression resolves to (a typed receiver through the MRO, a class, a scoped name, a module member); an untyped receiver gives no row. Always tier callback_registered; the site is the reference expression (call-edge-generation/method-reference.dl). | | `TRUTH_PROTOCOL` | python | A truth test runs the tested value's `__bool__`, or `__len__` when its class has none: `if x:`, `while x:`, `assert x`, `not x`, a conditional or comprehension condition, and an operand of `and`/`or` (the right one only when the whole expression is tested). No written call; the site is the tested expression. | **`call_edges.tier` values** @@ -564,6 +565,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | `dynamic_terminal` | javascript | `obj[expr]()`, `eval`, `import()`: no static target by construction; callee is NULL. | | `fan_capped` | javascript, java, csharp | More targets than --dispatch-cap: the set was refused rather than emitted. JavaScript: callee is NULL. Java and C#: callee is the declared base method the fan would have started from; dispatch-capped-sites.csv carries the refused count. | | `callback_registered` | javascript, typescript | The site HANDS the callee this function (`xs.forEach(f)`, `p.then(f)`, `emitter.on('x', h)`, `setTimeout(f)`), which may invoke it. Not the site's own callee; a reachability edge, labelled so it is never read as a resolved call. | +| `callback_registered` | python | A method or function named as a VALUE and handed over, stored or returned (`pool.submit(self.work)`, `Thread(target=svc.run)`, `sorted(xs, key=self.rank)`, `[h.on_event]`), always kind METHOD_REFERENCE. Its target is resolved the way a call on the same receiver would be; whoever receives the value calls it. A reachability edge, labelled so it is never read as a resolved call. | | `event_dispatch` | javascript | `x.emit('name')` reaching a handler registered by `x.on('name', h)` on a value x may hold — name-sensitive for literal names, every handler on that value for a computed one. | | `event_dispatch` | typescript | A publish on a project class's keyed registry reaching each function filed under the same key: `bus.publish('x', p)` where `publish` (or a method it passes the key to) calls what `this.handlers.get(name)` holds, to the function a `bus.subscribe('x', f)` stored there, or an entry `{ ['x']: f }` of a table a subscribe-all method files by key. A literal, a const or a const-object member key; a key only known at run time matches every key, and a constant the dispatch also looks up (a wildcard) matches every publish. The site is the publish call (call-edge-generation/keyed_registry.dl). | | `event_dispatch` | java | A Spring application event: `publishEvent(e)` reaching each listener (`@EventListener`, `@TransactionalEventListener`, `ApplicationListener.onApplicationEvent`) whose declared event type e's static type is, or is a subtype of. Added beside the publishEvent boundary row, never in place of it (call-edge-generation/event_dispatch.dl). Also a JPA entity write (`save`, `persist`, `merge`, `delete` on a repository or EntityManager) reaching the `@PrePersist`/`@PreUpdate`/`@PreRemove`-style callbacks of the written entity's type and of the listeners `@EntityListeners` names on it or a superclass (kind entity_callback; call-edge-generation/entity_lifecycle.dl). | @@ -586,7 +588,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h - **python** — A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row. - **java** — A `boundary_lib` edge with callee_provenance external names a method of an ancestor type no staged IR declares (callee_label `external:.`, no methods row). A site whose receiver is declared as such a type is multi_inferred even with one client override: the platform method itself, and the platform's own subclasses, are the other possible targets. Stage the library to replace the label with the real method. - **python** — The reason a site is ambiguous_unknown is exported per site in ext_call_site_unresolved (site, caller, reason, detail). -- **all** — THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge and multi_inferred in every language, and boundary_lib where the library is staged (--library) — without it boundary_lib names the target in callee_label and leaves callee_method_id NULL; ALSO ambient_terminal in TypeScript, fan_capped in Java and C# (the declared base, the fan refused), and runtime_observed in C#. HANDED OVER (callee set, but the site passes the function rather than calling it): callback_registered and event_dispatch in JavaScript and TypeScript; event_dispatch in C# too, where a mediator Send or Publish runs the handler for the request type, beside the row for the site itself. CORRECT END (callee NULL, and nothing is missing): intrinsic_terminal in TypeScript; ambient_terminal, implicit_constructor and dynamic_terminal in JavaScript; known_implicit_ctor, known_builtin_operator and boundary_generated (callee_label set) in C#. BLIND SPOT (callee NULL; exactly the tiers named `ambiguous_*`, which are what unresolved_sites holds): ambiguous_unknown everywhere, ALSO ambiguous_anon in Java and ambiguous_dynamic in C#. CAPPED (callee NULL, not in unresolved_sites): fan_capped in JavaScript. Python emits only the four shared tiers. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in every language but Python — derive the set from this note or from unresolved_sites, never from a hardcoded list. +- **all** — THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge and multi_inferred in every language, and boundary_lib where the library is staged (--library) — without it boundary_lib names the target in callee_label and leaves callee_method_id NULL; ALSO ambient_terminal in TypeScript, fan_capped in Java and C# (the declared base, the fan refused), and runtime_observed in C#. HANDED OVER (callee set, but the site passes the function rather than calling it): callback_registered and event_dispatch in JavaScript and TypeScript, callback_registered (kind METHOD_REFERENCE) in Python; event_dispatch in C# too, where a mediator Send or Publish runs the handler for the request type, beside the row for the site itself. CORRECT END (callee NULL, and nothing is missing): intrinsic_terminal in TypeScript; ambient_terminal, implicit_constructor and dynamic_terminal in JavaScript; known_implicit_ctor, known_builtin_operator and boundary_generated (callee_label set) in C#. BLIND SPOT (callee NULL; exactly the tiers named `ambiguous_*`, which are what unresolved_sites holds): ambiguous_unknown everywhere, ALSO ambiguous_anon in Java and ambiguous_dynamic in C#. CAPPED (callee NULL, not in unresolved_sites): fan_capped in JavaScript. Python emits the four shared tiers and callback_registered. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in every language but Python — derive the set from this note or from unresolved_sites, never from a hardcoded list. - **java** — A multi_inferred fan is CHA-wide: it is every override the hierarchy admits, bounded only by the dispatch cap. type_instantiated is computed and exported but NOT read by any rule, so the fan is not narrowed to types the program constructs. Narrow it yourself by joining dispatch_candidates to type_instantiated — see the dispatch_envelope_of query. A receiver is ALSO typed by what flows into it (a local's initializer, the arguments callers pass to a parameter, the receivers callers invoke a method on for its `this`), and each flow-in type resolves its member directly, outside the fan: that is why a `fan_capped` site still carries edges, and why they are the types the program was seen to hand over, not the whole hierarchy. - **typescript** — A multi_inferred fan is CHA-wide, as in Java: type_instantiated is computed and exported but NOT read by any rule. The fan also has sources that are not virtual dispatch at all — an overload set or a union-typed receiver produces one too. - **python** — A multi_inferred fan IS narrowed by the instantiation set: type_instantiated_reachable (the constructed classes and their bases) bounds dispatch in resolution/dispatch.dl. Python is the only front end where that narrowing is applied, so a fan here is tighter than the same shape would be in Java or TypeScript. diff --git a/graph/bundle/schema.ts b/graph/bundle/schema.ts index 731fb61b6..9d71c3906 100644 --- a/graph/bundle/schema.ts +++ b/graph/bundle/schema.ts @@ -524,6 +524,7 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'call_edges', column: 'tier', value: 'dynamic_terminal', languages: S, meaning: '`obj[expr]()`, `eval`, `import()`: no static target by construction; callee is NULL.' }, { table: 'call_edges', column: 'tier', value: 'fan_capped', languages: ['javascript', 'java', 'csharp'], meaning: 'More targets than --dispatch-cap: the set was refused rather than emitted. JavaScript: callee is NULL. Java and C#: callee is the declared base method the fan would have started from; dispatch-capped-sites.csv carries the refused count.' }, { table: 'call_edges', column: 'tier', value: 'callback_registered', languages: ['javascript', 'typescript'], meaning: 'The site HANDS the callee this function (`xs.forEach(f)`, `p.then(f)`, `emitter.on(\'x\', h)`, `setTimeout(f)`), which may invoke it. Not the site\'s own callee; a reachability edge, labelled so it is never read as a resolved call.' }, + { table: 'call_edges', column: 'tier', value: 'callback_registered', languages: P, meaning: 'A method or function named as a VALUE and handed over, stored or returned (`pool.submit(self.work)`, `Thread(target=svc.run)`, `sorted(xs, key=self.rank)`, `[h.on_event]`), always kind METHOD_REFERENCE. Its target is resolved the way a call on the same receiver would be; whoever receives the value calls it. A reachability edge, labelled so it is never read as a resolved call.' }, { table: 'call_edges', column: 'tier', value: 'event_dispatch', languages: S, meaning: '`x.emit(\'name\')` reaching a handler registered by `x.on(\'name\', h)` on a value x may hold — name-sensitive for literal names, every handler on that value for a computed one.' }, { table: 'call_edges', column: 'tier', value: 'event_dispatch', languages: T, meaning: 'A publish on a project class\'s keyed registry reaching each function filed under the same key: `bus.publish(\'x\', p)` where `publish` (or a method it passes the key to) calls what `this.handlers.get(name)` holds, to the function a `bus.subscribe(\'x\', f)` stored there, or an entry `{ [\'x\']: f }` of a table a subscribe-all method files by key. A literal, a const or a const-object member key; a key only known at run time matches every key, and a constant the dispatch also looks up (a wildcard) matches every publish. The site is the publish call (call-edge-generation/keyed_registry.dl).' }, { table: 'call_edges', column: 'tier', value: 'event_dispatch', languages: J, meaning: 'A Spring application event: `publishEvent(e)` reaching each listener (`@EventListener`, `@TransactionalEventListener`, `ApplicationListener.onApplicationEvent`) whose declared event type e\'s static type is, or is a subtype of. Added beside the publishEvent boundary row, never in place of it (call-edge-generation/event_dispatch.dl). Also a JPA entity write (`save`, `persist`, `merge`, `delete` on a repository or EntityManager) reaching the `@PrePersist`/`@PreUpdate`/`@PreRemove`-style callbacks of the written entity\'s type and of the listeners `@EntityListeners` names on it or a superclass (kind entity_callback; call-edge-generation/entity_lifecycle.dl).' }, @@ -684,6 +685,7 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'call_edges', column: 'kind', value: 'SUBSCRIPT_PROTOCOL', languages: P, meaning: '`x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver\'s class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call.' }, { table: 'call_edges', column: 'kind', value: 'BUILTIN_PROTOCOL', languages: P, meaning: '`repr(x)`, `str(x)`, `len(x)`, `hash(x)`, `bool(x)`, `iter(x)`, `next(x)`, `abs(x)`, `format(x)` and `reversed(x)` run the matching dunder of the argument\'s class (str falls back to `__repr__`, bool to `__len__`). The written call is to the builtin; this edge is the dunder it runs. The site is the argument expression. Its own kind so it is never counted as a written call.' }, { table: 'call_edges', column: 'kind', value: 'OPERATOR_PROTOCOL', languages: P, meaning: 'An operator runs a dunder of its operands: `a + b` the left operand\'s `__add__` and the right one\'s `__radd__`, a comparison the left slot and the right one\'s mirror (`a < b` -> `b.__gt__`, `!=` falls back to `__eq__`), `x in c` the right operand\'s `__contains__` (or `__iter__`), `-a`/`+a`/`~a` its `__neg__`/`__pos__`/`__invert__`. No written call; the site is the operator expression.' }, + { table: 'call_edges', column: 'kind', value: 'METHOD_REFERENCE', languages: P, meaning: 'A method or function READ AS A VALUE, not called: an argument (`pool.submit(self.work, x)`, `map(Cls.convert, rows)`, `partial(svc.cancel, oid)`), a keyword (`Thread(target=svc.run)`), an element, a stored or returned value. The target is what the expression resolves to (a typed receiver through the MRO, a class, a scoped name, a module member); an untyped receiver gives no row. Always tier callback_registered; the site is the reference expression (call-edge-generation/method-reference.dl).' }, { table: 'call_edges', column: 'kind', value: 'TRUTH_PROTOCOL', languages: P, meaning: 'A truth test runs the tested value\'s `__bool__`, or `__len__` when its class has none: `if x:`, `while x:`, `assert x`, `not x`, a conditional or comprehension condition, and an operand of `and`/`or` (the right one only when the whole expression is tested). No written call; the site is the tested expression.' }, // entry_points.reason @@ -763,7 +765,7 @@ export const NOTES: readonly NoteSpec[] = [ { language: 'typescript', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The TypeScript dispatch envelope is in dispatch_candidates, with basis `nominal` or `structural`.' }, { language: 'typescript', table: 'type_instantiated', note: 'Every row has how = `new`. Not restricted to client provenance: a type the library constructs is still a type that exists at run time, and dropping it would narrow the envelope unsoundly.' }, { language: 'typescript', table: 'call_sites', note: 'PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor\'s name is on the callee\'s methods row. Filter them out with kind NOT IN (…) when counting calls.' }, - { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, + { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METHOD_REFERENCE, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, { language: 'python', table: 'call_sites', note: 'The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class\'s TYPE hash for METACLASS_CREATION, positioned at the class declaration.' }, { language: 'python', table: 'call_edges', note: 'A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row.' }, { language: 'java', table: 'call_edges', note: 'A `boundary_lib` edge with callee_provenance external names a method of an ancestor type no staged IR declares (callee_label `external:.`, no methods row). A site whose receiver is declared as such a type is multi_inferred even with one client override: the platform method itself, and the platform\'s own subclasses, are the other possible targets. Stage the library to replace the label with the real method.' }, @@ -771,7 +773,7 @@ export const NOTES: readonly NoteSpec[] = [ { language: 'python', table: 'entry_points', note: 'Framework entry points only: url, http, orm_hook, task, signal_receiver, fixture, di_provider and grpc_service. There is no test and no main reason: a pytest test is recognised by the query layer from its file and name, not here.' }, { language: 'python', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The Python dispatch envelope is in dispatch_candidates with basis `mro`, and `value` for a function assigned onto an instance\'s member; the raw linearisation is in ext_mro_position.' }, { language: 'python', table: 'type_instantiated', note: 'Every row has how = `new`: the rule set records that some client call constructs the class, not which form.' }, - { language: 'all', table: 'call_edges', note: 'THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge and multi_inferred in every language, and boundary_lib where the library is staged (--library) — without it boundary_lib names the target in callee_label and leaves callee_method_id NULL; ALSO ambient_terminal in TypeScript, fan_capped in Java and C# (the declared base, the fan refused), and runtime_observed in C#. HANDED OVER (callee set, but the site passes the function rather than calling it): callback_registered and event_dispatch in JavaScript and TypeScript; event_dispatch in C# too, where a mediator Send or Publish runs the handler for the request type, beside the row for the site itself. CORRECT END (callee NULL, and nothing is missing): intrinsic_terminal in TypeScript; ambient_terminal, implicit_constructor and dynamic_terminal in JavaScript; known_implicit_ctor, known_builtin_operator and boundary_generated (callee_label set) in C#. BLIND SPOT (callee NULL; exactly the tiers named `ambiguous_*`, which are what unresolved_sites holds): ambiguous_unknown everywhere, ALSO ambiguous_anon in Java and ambiguous_dynamic in C#. CAPPED (callee NULL, not in unresolved_sites): fan_capped in JavaScript. Python emits only the four shared tiers. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in every language but Python — derive the set from this note or from unresolved_sites, never from a hardcoded list.' }, + { language: 'all', table: 'call_edges', note: 'THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge and multi_inferred in every language, and boundary_lib where the library is staged (--library) — without it boundary_lib names the target in callee_label and leaves callee_method_id NULL; ALSO ambient_terminal in TypeScript, fan_capped in Java and C# (the declared base, the fan refused), and runtime_observed in C#. HANDED OVER (callee set, but the site passes the function rather than calling it): callback_registered and event_dispatch in JavaScript and TypeScript, callback_registered (kind METHOD_REFERENCE) in Python; event_dispatch in C# too, where a mediator Send or Publish runs the handler for the request type, beside the row for the site itself. CORRECT END (callee NULL, and nothing is missing): intrinsic_terminal in TypeScript; ambient_terminal, implicit_constructor and dynamic_terminal in JavaScript; known_implicit_ctor, known_builtin_operator and boundary_generated (callee_label set) in C#. BLIND SPOT (callee NULL; exactly the tiers named `ambiguous_*`, which are what unresolved_sites holds): ambiguous_unknown everywhere, ALSO ambiguous_anon in Java and ambiguous_dynamic in C#. CAPPED (callee NULL, not in unresolved_sites): fan_capped in JavaScript. Python emits the four shared tiers and callback_registered. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in every language but Python — derive the set from this note or from unresolved_sites, never from a hardcoded list.' }, { language: 'java', table: 'call_edges', note: 'A multi_inferred fan is CHA-wide: it is every override the hierarchy admits, bounded only by the dispatch cap. type_instantiated is computed and exported but NOT read by any rule, so the fan is not narrowed to types the program constructs. Narrow it yourself by joining dispatch_candidates to type_instantiated — see the dispatch_envelope_of query. A receiver is ALSO typed by what flows into it (a local\'s initializer, the arguments callers pass to a parameter, the receivers callers invoke a method on for its `this`), and each flow-in type resolves its member directly, outside the fan: that is why a `fan_capped` site still carries edges, and why they are the types the program was seen to hand over, not the whole hierarchy.' }, { language: 'typescript', table: 'call_edges', note: 'A multi_inferred fan is CHA-wide, as in Java: type_instantiated is computed and exported but NOT read by any rule. The fan also has sources that are not virtual dispatch at all — an overload set or a union-typed receiver produces one too.' }, { language: 'python', table: 'call_edges', note: 'A multi_inferred fan IS narrowed by the instantiation set: type_instantiated_reachable (the constructed classes and their bases) bounds dispatch in resolution/dispatch.dl. Python is the only front end where that narrowing is applied, so a fan here is tighter than the same shape would be in Java or TypeScript.' }, diff --git a/graph/python/engine/call-edge-generation/method-reference.dl b/graph/python/engine/call-edge-generation/method-reference.dl new file mode 100644 index 000000000..c3bcf5da2 --- /dev/null +++ b/graph/python/engine/call-edge-generation/method-reference.dl @@ -0,0 +1,123 @@ +// ============================================================================ +// CALL-EDGE-GENERATION · METHOD REFERENCES (a callable named as a VALUE, not called) +// +// ── THE GAP ───────────────────────────────────────────────────────────────── +// executor.submit(self._do_work, item) threading.Thread(target=svc.run_job) +// loop.run_in_executor(None, svc.cancel, oid) sorted(xs, key=self._rank) +// callbacks = [handler.on_event] functools.partial(svc.cancel, oid) +// app.add_url_rule("/x", view_func=views.index) return self._handle +// Each hands a method or function over as a value; whoever receives it calls it, later, from +// a frame the graph cannot see (a worker thread, a builtin, a library). There is no call site +// for it, so before this file the method had NO caller here at all, and `impact +// OrderService.cancel` did not name the function that queued it. The engine already KNEW what +// each of these expressions denotes -- expr_denotes_method resolves `self._do_work` through the +// receiver's MRO exactly as it resolves `self._do_work()` -- and dropped the answer because no +// CALL wrapped it. +// +// ── WHAT IT EMITS ─────────────────────────────────────────────────────────── +// call_chain_edge(Expr, Caller, "-", Target, "client", "callback_registered", "METHOD_REFERENCE") +// Like PROPERTY_READ and the protocol rows in call_chain.dl, an edge with no call site: Site is +// the reference expression itself, and the kind is its own so a consumer can always tell it +// from a written call. The tier is the one every other front end uses for a function handed +// over (`xs.forEach(f)`, a JSX handler attribute): HANDED OVER, never `resolved`, so an answer +// says "handed over as a value" instead of "calls it". +// +// ── WHAT IT REFUSES ───────────────────────────────────────────────────────── +// - ONLY WHAT IS RESOLVED. The target comes from the same resolution an ordinary call's +// callee gets: a receiver typed through expr_type and the MRO, a class named by its +// declaration, a name bound by its scope, a module member. `thing.cancel` on an untyped +// parameter denotes nothing, so it emits nothing -- the same blind spot `thing.cancel()` +// has. No rule here matches by name. +// - NOT A FLOWED VALUE. `def run(cb): pool.submit(cb)` denotes, through +// param_arg_method, every function any caller passed in. The reference that matters is +// the one the caller WROTE (`run(self._work)`), which already has its own row; crediting +// `run` as well would make it reach the union of every callable handed to it. The same +// holds for an attribute that holds a function (see (b) below). +// - NOT A CALLEE, NOT A WRITE, NOT AN OBJECT OF ANOTHER ATTRIBUTE. `self.m()` is a call +// (call_chain.dl owns it); `self.m = x` is a store; `self.m.__name__` and +// `self.m.cache_clear()` read the function object and never run it. +// - NOT A PROPERTY, NOT A DECORATOR. `order.total` on a @property (or @cached_property) +// reads the VALUE the getter computes: call_chain.dl already emits that as PROPERTY_READ, +// and the function object is never handed anywhere. `@audited` is a decorator +// application, which call_chain.dl emits as DECORATOR_*. Before these two guards they +// were the two largest wrong classes in a sampled review. +// - NOT THE ARGUMENT OF A ONE-DUNDER BUILTIN. `str(fn)`, `repr(fn)`, `hash(fn)` +// (builtins.dl's builtin_protocol_slot) run a dunder of the FUNCTION OBJECT and never call +// it; a test asserting on `str(svc.cancel)` is not a caller of cancel. Other inspecting +// builtins (`id`, `callable`, `isinstance`) are not catalogued and still count. +// - NOT A COMPARISON OPERAND. `assert job.on_done is svc.cancel` tests identity. +// - ONLY NAMES AND ATTRIBUTE READS. A lambda, a subscript (`table[k]`) and a call that +// returns a function (`partial(...)`, a factory) also denote methods; each already has its +// own edge where it is invoked, or IS a call with its own row. +// The receiver-typing, decorator-chain and import rules this reads all live in resolution/; +// nothing here widens them. +// ============================================================================ + +.decl method_ref_expr(e:symbol, m:symbol) +// (a) a NAME: what resolution/ says it denotes -- a def bound in scope, an import, a +// decorated name's wrapper -- minus a name that holds a flowed VALUE: a parameter, or a +// local or module name assigned from an expression (`cb = job.on_done`). That +// assignment's right-hand side is credited where it is written when it is a reference +// itself; when it is a field read, what the name holds is the union of every write to that +// field, which was the largest wrong shape in a sampled review (`assert cb is not None` +// credited with every callback any job was ever given). +method_ref_expr(e, m) :- + expr_node("client", "NAME_REFERENCE", _, _, e), + expr_denotes_method("client", e, m), + !expr_names_param("client", e, _), + !method_ref_name_holds_value(e). +.decl method_ref_name_holds_value(e:symbol) +method_ref_name_holds_value(e) :- + expr_node("client", "NAME_REFERENCE", _, _, e), + expr_binding("client", b, ctx, e), ctx != "STORE", + binding_lookup("client", b, b2), binding_value_method("client", b2, _). +// (b) an ATTRIBUTE: a method found through the typed receiver's MRO (`svc.cancel`, +// `self._work`), or a module member (`views.index`, decorated or not). NOT an attribute +// that merely HOLDS a function some write put there (`job.on_done`): that is a flowed +// value like a parameter's, and a sampled review found exactly that shape wrong: +// `job.on_done is NO_OP`, credited to whatever any write stored. +method_ref_expr(e, m) :- expr_denotes_bound_method("client", e, m). +method_ref_expr(e, m) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, _, e), + expr_attr_names_method("client", e, _), + expr_denotes_method("client", e, m). +// (c) an UNBOUND method read off its class: `map(OrderService.convert, rows)`. The class +// object types the receiver, and the MRO names the method -- the rule +// callee-resolution.dl uses for `OrderService.convert(svc, row)`, without the call. +method_ref_expr(e, m) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), + expr_parent("client", e, "ATTRIBUTE_OBJECT", _, obj), + expr_type_class_object("client", obj, t), + mro_lookup("client", t, n, m). + +// the shape guards: read as a value, not called, not a write, not the object of another read +.decl method_ref_site(e:symbol, m:symbol) +method_ref_site(e, m) :- + method_ref_expr(e, m), + !call_callee_expr(_, e), + !expr_name_context("client", "STORE", e), !expr_name_context("client", "DEL", e), + !expr_parent("client", _, "ATTRIBUTE_OBJECT", _, e), + !decorator_expr("client", e, _), + !builtin_protocol_arg(_, _, e), + !method_ref_compared(e), + method_kind("client", kind, _, m), + kind != "MODULE_INITIALIZER", kind != "CLASS_INITIALIZER", kind != "OVERLOAD_STUB", kind != "LAMBDA", + kind != "PROPERTY_GETTER", kind != "PROPERTY_SETTER", kind != "PROPERTY_DELETER". + +// an operand of a comparison: `job.on_done is svc.cancel`, `h in self.handlers`, `f == g` test +// the function object's identity or equality and never call it +.decl method_ref_compared(e:symbol) +method_ref_compared(e) :- method_ref_expr(e, _), expr_parent("client", c, _, _, e), expr_node("client", "COMPARISON", _, _, c). + +// the caller: the function or lambda whose body evaluates the reference. A module-level +// reference and a parameter default are owned by the synthetic , a class-body one by +// -- the parser records that owner on the expression, as it does for a call. +.decl method_ref_caller(e:symbol, c:symbol) +method_ref_caller(e, c) :- method_ref_site(e, _), expr_owner("client", "METHOD", c, _, e). +method_ref_caller(e, c) :- method_ref_site(e, _), expr_owner("client", "LAMBDA", c, _, e). +method_ref_caller(e, c) :- method_ref_site(e, _), + !expr_owner("client", "METHOD", _, _, e), !expr_owner("client", "LAMBDA", _, _, e), + expr_ultimate_method("client", e, c). + +call_chain_edge(e, c, "-", m, "client", "callback_registered", "METHOD_REFERENCE") :- + method_ref_site(e, m), method_ref_caller(e, c). diff --git a/graph/test/python/expected/01-plain-calls.edges b/graph/test/python/expected/01-plain-calls.edges index c37ab9bcd..9df3ca724 100644 --- a/graph/test/python/expected/01-plain-calls.edges +++ b/graph/test/python/expected/01-plain-calls.edges @@ -1,4 +1,5 @@ boundary_lib SIMPLE_CALL main.main -> builtin:print +callback_registered METHOD_REFERENCE main.make_adder -> main.make_adder..add known_edge SIMPLE_CALL main. -> main.main known_edge SIMPLE_CALL main.main -> main.make_adder known_edge SIMPLE_CALL main.main -> main.make_adder..add diff --git a/graph/test/python/expected/01-plain-calls.tiers b/graph/test/python/expected/01-plain-calls.tiers index 5b235883c..0c7ce9f47 100644 --- a/graph/test/python/expected/01-plain-calls.tiers +++ b/graph/test/python/expected/01-plain-calls.tiers @@ -1,10 +1,12 @@ -distinct call sites emitted: 12 +distinct call sites emitted: 13 --- by tier: edge ROWS, and the distinct SITES they cover --- 3 rows 3 sites boundary_lib + 1 rows 1 sites callback_registered 9 rows 9 sites known_edge --- edge rows by call kind --- + 1 METHOD_REFERENCE 12 SIMPLE_CALL --- unresolved reasons --- @@ -16,7 +18,7 @@ distinct call sites emitted: 12 9 known_edge --- reconciling rows against the conserved site count --- - edge rows 12 + edge rows 13 minus extra rows from multi-target sites 0 - = tier/site pairs 12 + = tier/site pairs 13 engine's conserved site total 12 diff --git a/graph/test/python/expected/07-decorators.edges b/graph/test/python/expected/07-decorators.edges index 41e8ab4f3..fa1975f0d 100644 --- a/graph/test/python/expected/07-decorators.edges +++ b/graph/test/python/expected/07-decorators.edges @@ -8,6 +8,11 @@ boundary_lib SIMPLE_CALL main.inner_deco..inner_w -> builtin:str boundary_lib SIMPLE_CALL main.main -> builtin:print boundary_lib SIMPLE_CALL main.outer_deco..outer_w -> builtin:str boundary_lib SIMPLE_CALL main.wrapping..wrapper -> builtin:str +callback_registered METHOD_REFERENCE main.inner_deco -> main.inner_deco..inner_w +callback_registered METHOD_REFERENCE main.outer_deco -> main.outer_deco..outer_w +callback_registered METHOD_REFERENCE main.replacing -> main.replacing..substitute +callback_registered METHOD_REFERENCE main.route -> main.route..register +callback_registered METHOD_REFERENCE main.wrapping -> main.wrapping..wrapper known_edge DECORATOR_APPLICATION main. -> main.route..register known_edge DECORATOR_BARE main. -> main.inner_deco known_edge DECORATOR_BARE main. -> main.outer_deco diff --git a/graph/test/python/expected/07-decorators.tiers b/graph/test/python/expected/07-decorators.tiers index e9686fea6..33c95f323 100644 --- a/graph/test/python/expected/07-decorators.tiers +++ b/graph/test/python/expected/07-decorators.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 30 +distinct call sites emitted: 35 --- by tier: edge ROWS, and the distinct SITES they cover --- 3 rows 3 sites ambiguous_unknown 11 rows 11 sites boundary_lib + 5 rows 5 sites callback_registered 16 rows 16 sites known_edge --- edge rows by call kind --- 4 DECORATOR_APPLICATION 5 DECORATOR_BARE 4 DECORATOR_CALL + 5 METHOD_REFERENCE 16 SIMPLE_CALL 1 SUBSCRIPT_CALL @@ -22,7 +24,7 @@ distinct call sites emitted: 30 16 known_edge --- reconciling rows against the conserved site count --- - edge rows 30 + edge rows 35 minus extra rows from multi-target sites 0 - = tier/site pairs 30 + = tier/site pairs 35 engine's conserved site total 30 diff --git a/graph/test/python/expected/08-callables.edges b/graph/test/python/expected/08-callables.edges index 5b0c619d2..24c61eb13 100644 --- a/graph/test/python/expected/08-callables.edges +++ b/graph/test/python/expected/08-callables.edges @@ -2,6 +2,8 @@ ambiguous_unknown SELF_CALL main.Holder.use_partial -> - boundary_lib METHOD_CALL main.Holder.__init__ -> external:functools.partial boundary_lib METHOD_CALL main.main -> external:functools.partial boundary_lib SIMPLE_CALL main.main -> builtin:print +callback_registered METHOD_REFERENCE main.Holder.__init__ -> main.add +callback_registered METHOD_REFERENCE main.main -> main.add known_edge CHAINED_CALL main.main -> main.Holder.use known_edge CHAINED_CALL main.main -> main.Holder.use_partial known_edge SELF_CALL main.Holder.use -> main.add diff --git a/graph/test/python/expected/08-callables.tiers b/graph/test/python/expected/08-callables.tiers index 9d83aad23..7a2631ca0 100644 --- a/graph/test/python/expected/08-callables.tiers +++ b/graph/test/python/expected/08-callables.tiers @@ -1,13 +1,15 @@ -distinct call sites emitted: 20 +distinct call sites emitted: 23 --- by tier: edge ROWS, and the distinct SITES they cover --- 1 rows 1 sites ambiguous_unknown 8 rows 8 sites boundary_lib + 3 rows 3 sites callback_registered 11 rows 11 sites known_edge --- edge rows by call kind --- 2 CHAINED_CALL 3 METHOD_CALL + 3 METHOD_REFERENCE 3 SELF_CALL 11 SIMPLE_CALL 1 SUBSCRIPT_CALL @@ -22,7 +24,7 @@ distinct call sites emitted: 20 11 known_edge --- reconciling rows against the conserved site count --- - edge rows 20 + edge rows 23 minus extra rows from multi-target sites 0 - = tier/site pairs 20 + = tier/site pairs 23 engine's conserved site total 20 diff --git a/graph/test/python/expected/09-args-forwarding.edges b/graph/test/python/expected/09-args-forwarding.edges index d05a79897..1dfc354c7 100644 --- a/graph/test/python/expected/09-args-forwarding.edges +++ b/graph/test/python/expected/09-args-forwarding.edges @@ -1,5 +1,6 @@ boundary_lib SIMPLE_CALL main.main -> builtin:print boundary_lib SIMPLE_CALL main.target -> builtin:len +callback_registered METHOD_REFERENCE main.main -> main.target known_edge DYNAMIC_CALL main.main -> main.Delegator.__call__ known_edge SELF_CALL main.Delegator.__call__ -> main.target known_edge SIMPLE_CALL main. -> main.main diff --git a/graph/test/python/expected/09-args-forwarding.tiers b/graph/test/python/expected/09-args-forwarding.tiers index 61fade854..13ce90bc8 100644 --- a/graph/test/python/expected/09-args-forwarding.tiers +++ b/graph/test/python/expected/09-args-forwarding.tiers @@ -1,11 +1,13 @@ -distinct call sites emitted: 15 +distinct call sites emitted: 16 --- by tier: edge ROWS, and the distinct SITES they cover --- 5 rows 5 sites boundary_lib + 1 rows 1 sites callback_registered 10 rows 10 sites known_edge --- edge rows by call kind --- 1 DYNAMIC_CALL + 1 METHOD_REFERENCE 1 SELF_CALL 13 SIMPLE_CALL @@ -18,7 +20,7 @@ distinct call sites emitted: 15 10 known_edge --- reconciling rows against the conserved site count --- - edge rows 15 + edge rows 16 minus extra rows from multi-target sites 0 - = tier/site pairs 15 + = tier/site pairs 16 engine's conserved site total 15 diff --git a/graph/test/python/expected/12-blind-spots.edges b/graph/test/python/expected/12-blind-spots.edges index 014a110bf..910099c29 100644 --- a/graph/test/python/expected/12-blind-spots.edges +++ b/graph/test/python/expected/12-blind-spots.edges @@ -7,6 +7,7 @@ boundary_lib SIMPLE_CALL main.Installer.__new__ -> builtin:super boundary_lib SIMPLE_CALL main.call_by_computed_name -> builtin:getattr boundary_lib SIMPLE_CALL main.main -> builtin:print boundary_lib SUPER_CALL main.Installer.__new__ -> builtin:type.__new__ +callback_registered METHOD_REFERENCE main. -> main.patched known_edge METACLASS_CREATION main. -> main.Installer.__new__ known_edge METHOD_CALL main.main -> main.patched known_edge SIMPLE_CALL main. -> main.main diff --git a/graph/test/python/expected/12-blind-spots.tiers b/graph/test/python/expected/12-blind-spots.tiers index c2de53b23..77b9517f7 100644 --- a/graph/test/python/expected/12-blind-spots.tiers +++ b/graph/test/python/expected/12-blind-spots.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 19 +distinct call sites emitted: 20 --- by tier: edge ROWS, and the distinct SITES they cover --- 5 rows 5 sites ambiguous_unknown 10 rows 10 sites boundary_lib + 1 rows 1 sites callback_registered 4 rows 4 sites known_edge --- edge rows by call kind --- @@ -10,6 +11,7 @@ distinct call sites emitted: 19 1 DYNAMIC_CALL 1 METACLASS_CREATION 2 METHOD_CALL + 1 METHOD_REFERENCE 13 SIMPLE_CALL 1 SUPER_CALL @@ -25,8 +27,8 @@ distinct call sites emitted: 19 3 known_edge --- reconciling rows against the conserved site count --- - edge rows 19 + edge rows 20 minus extra rows from multi-target sites 0 - = tier/site pairs 19 + = tier/site pairs 20 engine's conserved site total 18 of which 1 are METACLASS_CREATION, an edge with no call site (README decision 5) diff --git a/graph/test/python/expected/17-rebound-metaclass-slot.edges b/graph/test/python/expected/17-rebound-metaclass-slot.edges index 8555ff828..01bd9313b 100644 --- a/graph/test/python/expected/17-rebound-metaclass-slot.edges +++ b/graph/test/python/expected/17-rebound-metaclass-slot.edges @@ -10,6 +10,7 @@ boundary_lib SUPER_CALL main.EarlyMeta.__init__ -> builtin:type.__init__ boundary_lib SUPER_CALL main.EarlyMeta.__new__ -> builtin:type.__new__ boundary_lib SUPER_CALL main.LateMeta.__init__ -> builtin:type.__init__ boundary_lib SUPER_CALL main.LateMeta.__new__ -> builtin:type.__new__ +callback_registered METHOD_REFERENCE main. -> main._late_new known_edge CHAINED_CALL main.main -> main.After.run known_edge CHAINED_CALL main.main -> main.AlsoBefore.run known_edge CHAINED_CALL main.main -> main.Before.run diff --git a/graph/test/python/expected/17-rebound-metaclass-slot.tiers b/graph/test/python/expected/17-rebound-metaclass-slot.tiers index 36fff263c..7b9785d53 100644 --- a/graph/test/python/expected/17-rebound-metaclass-slot.tiers +++ b/graph/test/python/expected/17-rebound-metaclass-slot.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 20 +distinct call sites emitted: 22 --- by tier: edge ROWS, and the distinct SITES they cover --- 3 rows 3 sites ambiguous_unknown 11 rows 11 sites boundary_lib + 2 rows 2 sites callback_registered 8 rows 6 sites known_edge --- edge rows by call kind --- 3 CHAINED_CALL 5 METACLASS_CREATION 1 METHOD_CALL + 2 METHOD_REFERENCE 9 SIMPLE_CALL 4 SUPER_CALL @@ -23,8 +25,8 @@ distinct call sites emitted: 20 3 known_edge --- reconciling rows against the conserved site count --- - edge rows 22 + edge rows 24 minus extra rows from multi-target sites 2 - = tier/site pairs 20 + = tier/site pairs 22 engine's conserved site total 17 of which 5 are METACLASS_CREATION, an edge with no call site (README decision 5) diff --git a/graph/test/python/expected/21-url-and-signal-dispatch.edges b/graph/test/python/expected/21-url-and-signal-dispatch.edges index 9d07c714d..0d85ec445 100644 --- a/graph/test/python/expected/21-url-and-signal-dispatch.edges +++ b/graph/test/python/expected/21-url-and-signal-dispatch.edges @@ -3,6 +3,13 @@ ambiguous_unknown DECORATOR_APPLICATION main. -> - ambiguous_unknown METHOD_CALL main.not_a_signal_send -> - boundary_lib SIMPLE_CALL main. -> builtin:object.__init__ boundary_lib SIMPLE_CALL signals. -> builtin:object.__init__ +callback_registered METHOD_REFERENCE main. -> main.detail +callback_registered METHOD_REFERENCE main. -> main.index +callback_registered METHOD_REFERENCE main. -> main.legacy +callback_registered METHOD_REFERENCE main. -> main.on_order_connected +callback_registered METHOD_REFERENCE main.not_a_route_table -> main.detail +callback_registered METHOD_REFERENCE urls. -> views.profile +callback_registered METHOD_REFERENCE urls. -> views.settings_page known_edge DECORATOR_CALL handlers. -> handlers.receiver known_edge DECORATOR_CALL main. -> main.receiver known_edge METHOD_CALL main. -> main.Signal.connect diff --git a/graph/test/python/expected/21-url-and-signal-dispatch.tiers b/graph/test/python/expected/21-url-and-signal-dispatch.tiers index 56e700521..03db6da82 100644 --- a/graph/test/python/expected/21-url-and-signal-dispatch.tiers +++ b/graph/test/python/expected/21-url-and-signal-dispatch.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 32 +distinct call sites emitted: 39 --- by tier: edge ROWS, and the distinct SITES they cover --- 5 rows 5 sites ambiguous_unknown 4 rows 4 sites boundary_lib + 7 rows 7 sites callback_registered 23 rows 23 sites known_edge --- edge rows by call kind --- 4 DECORATOR_APPLICATION 4 DECORATOR_CALL 4 METHOD_CALL + 7 METHOD_REFERENCE 20 SIMPLE_CALL --- unresolved reasons --- @@ -22,7 +24,7 @@ distinct call sites emitted: 32 23 known_edge --- reconciling rows against the conserved site count --- - edge rows 32 + edge rows 39 minus extra rows from multi-target sites 0 - = tier/site pairs 32 + = tier/site pairs 39 engine's conserved site total 32 diff --git a/graph/test/python/expected/22-di-providers.edges b/graph/test/python/expected/22-di-providers.edges index f73dff093..f02424a43 100644 --- a/graph/test/python/expected/22-di-providers.edges +++ b/graph/test/python/expected/22-di-providers.edges @@ -3,6 +3,12 @@ ambiguous_unknown DECORATOR_APPLICATION routes. -> - ambiguous_unknown METHOD_CALL main.read_item -> - boundary_lib SIMPLE_CALL main. -> builtin:object.__init__ boundary_lib SIMPLE_CALL routes. -> builtin:object.__init__ +callback_registered METHOD_REFERENCE deps. -> deps.get_repo +callback_registered METHOD_REFERENCE main. -> main.get_service +callback_registered METHOD_REFERENCE main. -> main.get_settings +callback_registered METHOD_REFERENCE main. -> main.get_user +callback_registered METHOD_REFERENCE routes. -> deps.get_page_size +callback_registered METHOD_REFERENCE routes. -> deps.get_repo known_edge DECORATOR_CALL main. -> main._App.get known_edge DECORATOR_CALL main. -> main._App.post known_edge DECORATOR_CALL routes. -> routes._Router.get diff --git a/graph/test/python/expected/22-di-providers.tiers b/graph/test/python/expected/22-di-providers.tiers index 193e49ac0..c1e8f4c11 100644 --- a/graph/test/python/expected/22-di-providers.tiers +++ b/graph/test/python/expected/22-di-providers.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 22 +distinct call sites emitted: 29 --- by tier: edge ROWS, and the distinct SITES they cover --- 4 rows 4 sites ambiguous_unknown 2 rows 2 sites boundary_lib + 7 rows 7 sites callback_registered 16 rows 16 sites known_edge --- edge rows by call kind --- 3 DECORATOR_APPLICATION 3 DECORATOR_CALL 1 METHOD_CALL + 7 METHOD_REFERENCE 15 SIMPLE_CALL --- unresolved reasons --- @@ -22,7 +24,7 @@ distinct call sites emitted: 22 16 known_edge --- reconciling rows against the conserved site count --- - edge rows 22 + edge rows 29 minus extra rows from multi-target sites 0 - = tier/site pairs 22 + = tier/site pairs 29 engine's conserved site total 22 diff --git a/graph/test/python/expected/25-imported-decorated-names.edges b/graph/test/python/expected/25-imported-decorated-names.edges index 7f267280d..1c4482423 100644 --- a/graph/test/python/expected/25-imported-decorated-names.edges +++ b/graph/test/python/expected/25-imported-decorated-names.edges @@ -3,6 +3,8 @@ ambiguous_unknown DECORATOR_ATTRIBUTE pkg.lib. -> - ambiguous_unknown METHOD_CALL main.run -> - ambiguous_unknown SIMPLE_CALL main.run -> - boundary_lib DECORATOR_CALL pkg.lib.wraps_deco -> external:functools.wraps +callback_registered METHOD_REFERENCE pkg.lib.plain_deco -> pkg.lib.plain_deco..wrapper +callback_registered METHOD_REFERENCE pkg.lib.wraps_deco -> pkg.lib.wraps_deco..wrapper known_edge DECORATOR_BARE pkg.lib. -> pkg.lib.passthrough known_edge DECORATOR_BARE pkg.lib. -> pkg.lib.plain_deco known_edge DECORATOR_BARE pkg.lib. -> pkg.lib.wraps_deco diff --git a/graph/test/python/expected/25-imported-decorated-names.tiers b/graph/test/python/expected/25-imported-decorated-names.tiers index b4fa51e62..2cd48680e 100644 --- a/graph/test/python/expected/25-imported-decorated-names.tiers +++ b/graph/test/python/expected/25-imported-decorated-names.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 24 +distinct call sites emitted: 26 --- by tier: edge ROWS, and the distinct SITES they cover --- 4 rows 4 sites ambiguous_unknown 1 rows 1 sites boundary_lib + 2 rows 2 sites callback_registered 19 rows 19 sites known_edge --- edge rows by call kind --- @@ -11,6 +12,7 @@ distinct call sites emitted: 24 3 DECORATOR_BARE 1 DECORATOR_CALL 5 METHOD_CALL + 2 METHOD_REFERENCE 13 SIMPLE_CALL --- unresolved reasons --- @@ -26,7 +28,7 @@ distinct call sites emitted: 24 19 known_edge --- reconciling rows against the conserved site count --- - edge rows 24 + edge rows 26 minus extra rows from multi-target sites 0 - = tier/site pairs 24 + = tier/site pairs 26 engine's conserved site total 24 diff --git a/graph/test/python/expected/27-returned-expression.edges b/graph/test/python/expected/27-returned-expression.edges index fed3df32d..c8699cfe3 100644 --- a/graph/test/python/expected/27-returned-expression.edges +++ b/graph/test/python/expected/27-returned-expression.edges @@ -6,6 +6,9 @@ boundary_lib METHOD_CALL main.Expr.copy -> external:copy.copy boundary_lib SIMPLE_CALL main. -> builtin:object.__init__ boundary_lib SIMPLE_CALL main.Expr.__init__ -> builtin:object.__init__ boundary_lib SIMPLE_CALL main.Expr.label -> builtin:str +callback_registered METHOD_REFERENCE main.Library.filter -> main.Library.filter..dec +callback_registered METHOD_REFERENCE main.defer_result -> main.defer_result..decorator +callback_registered METHOD_REFERENCE main.defer_result..decorator -> main.defer_result..decorator..wrapper known_edge DECORATOR_APPLICATION main. -> main.Library.filter..dec known_edge DECORATOR_BARE main. -> main.defer_text known_edge DECORATOR_CALL main. -> main.Library.filter diff --git a/graph/test/python/expected/27-returned-expression.tiers b/graph/test/python/expected/27-returned-expression.tiers index f44e09e11..61c44026e 100644 --- a/graph/test/python/expected/27-returned-expression.tiers +++ b/graph/test/python/expected/27-returned-expression.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 23 +distinct call sites emitted: 26 --- by tier: edge ROWS, and the distinct SITES they cover --- 2 rows 2 sites ambiguous_unknown 6 rows 6 sites boundary_lib + 3 rows 3 sites callback_registered 15 rows 15 sites known_edge 2 rows 1 sites multi_inferred @@ -12,6 +13,7 @@ distinct call sites emitted: 23 3 DECORATOR_CALL 1 DYNAMIC_CALL 5 METHOD_CALL + 3 METHOD_REFERENCE 3 SELF_CALL 10 SIMPLE_CALL @@ -27,7 +29,7 @@ distinct call sites emitted: 23 1 multi_inferred --- reconciling rows against the conserved site count --- - edge rows 25 + edge rows 28 minus extra rows from multi-target sites 1 - = tier/site pairs 24 + = tier/site pairs 27 engine's conserved site total 23 diff --git a/graph/test/python/expected/28-assigned-members.edges b/graph/test/python/expected/28-assigned-members.edges index c1af28b71..355e252d5 100644 --- a/graph/test/python/expected/28-assigned-members.edges +++ b/graph/test/python/expected/28-assigned-members.edges @@ -1,6 +1,8 @@ boundary_lib SIMPLE_CALL main. -> external:TypeVar boundary_lib SIMPLE_CALL main.Schema.__init__ -> builtin:object.__init__ boundary_lib SIMPLE_CALL main.main -> builtin:print +callback_registered METHOD_REFERENCE main.init -> main.Internals.parse +callback_registered METHOD_REFERENCE main.init -> main.init.._run known_edge SIMPLE_CALL main. -> main.main known_edge SIMPLE_CALL main.main -> main.Schema.__init__ known_edge SIMPLE_CALL main.main -> main.init diff --git a/graph/test/python/expected/28-assigned-members.tiers b/graph/test/python/expected/28-assigned-members.tiers index 4f1ba55ef..c4536e576 100644 --- a/graph/test/python/expected/28-assigned-members.tiers +++ b/graph/test/python/expected/28-assigned-members.tiers @@ -1,12 +1,14 @@ -distinct call sites emitted: 19 +distinct call sites emitted: 21 --- by tier: edge ROWS, and the distinct SITES they cover --- 4 rows 4 sites boundary_lib + 2 rows 2 sites callback_registered 7 rows 7 sites known_edge 30 rows 8 sites multi_inferred --- edge rows by call kind --- 30 METHOD_CALL + 2 METHOD_REFERENCE 11 SIMPLE_CALL --- unresolved reasons --- @@ -19,7 +21,7 @@ distinct call sites emitted: 19 8 multi_inferred --- reconciling rows against the conserved site count --- - edge rows 41 + edge rows 43 minus extra rows from multi-target sites 22 - = tier/site pairs 19 + = tier/site pairs 21 engine's conserved site total 19 diff --git a/graph/test/python/expected/33-url-view-forms.edges b/graph/test/python/expected/33-url-view-forms.edges index c05006003..0de9e0620 100644 --- a/graph/test/python/expected/33-url-view-forms.edges +++ b/graph/test/python/expected/33-url-view-forms.edges @@ -6,6 +6,16 @@ boundary_lib SIMPLE_CALL catalog.urls. -> external:path boundary_lib SIMPLE_CALL shop.urls. -> external:include boundary_lib SIMPLE_CALL shop.urls. -> external:path boundary_lib SIMPLE_CALL shop.urls. -> external:re_path +callback_registered METHOD_REFERENCE catalog.urls. -> catalog.views.catalog_index +callback_registered METHOD_REFERENCE shop.decorators.audited -> shop.decorators.audited..wrapper +callback_registered METHOD_REFERENCE shop.urls. -> shop.decorators.audited..wrapper +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.api_status +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_credit_note +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_draft +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_export +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_legacy +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_list +callback_registered METHOD_REFERENCE shop.urls. -> shop.views.order_report known_edge DECORATOR_BARE shop.views. -> shop.decorators.audited known_edge SIMPLE_CALL shop.decorators.audited..wrapper -> shop.views.order_history known_edge SIMPLE_CALL shop.views.BaseWidgetView.get -> shop.views._render diff --git a/graph/test/python/expected/33-url-view-forms.tiers b/graph/test/python/expected/33-url-view-forms.tiers index f05de8d3f..fd0afb6f1 100644 --- a/graph/test/python/expected/33-url-view-forms.tiers +++ b/graph/test/python/expected/33-url-view-forms.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 51 +distinct call sites emitted: 61 --- by tier: edge ROWS, and the distinct SITES they cover --- 6 rows 6 sites ambiguous_unknown 27 rows 27 sites boundary_lib + 10 rows 10 sites callback_registered 18 rows 18 sites known_edge --- edge rows by call kind --- @@ -10,6 +11,7 @@ distinct call sites emitted: 51 5 DECORATOR_BARE 1 DECORATOR_CALL 1 METHOD_CALL + 10 METHOD_REFERENCE 43 SIMPLE_CALL --- unresolved reasons --- @@ -24,7 +26,7 @@ distinct call sites emitted: 51 18 known_edge --- reconciling rows against the conserved site count --- - edge rows 51 + edge rows 61 minus extra rows from multi-target sites 0 - = tier/site pairs 51 + = tier/site pairs 61 engine's conserved site total 51 diff --git a/graph/test/python/expected/36-call-through-model-decorators.edges b/graph/test/python/expected/36-call-through-model-decorators.edges index 266e2812c..69c96d257 100644 --- a/graph/test/python/expected/36-call-through-model-decorators.edges +++ b/graph/test/python/expected/36-call-through-model-decorators.edges @@ -6,6 +6,7 @@ boundary_lib DECORATOR_CALL app.orders. -> external:pydantic.validate_ca boundary_lib METHOD_CALL app.checkout.show -> builtin:float.hex boundary_lib SIMPLE_CALL app.orders.apply_discount -> builtin:round boundary_lib SIMPLE_CALL app.orders.apply_tax -> builtin:round +callback_registered METHOD_REFERENCE app.orders.audit -> app.orders.audit..wrapper known_edge DECORATOR_BARE app.orders. -> app.orders.audit known_edge PROPERTY_READ app.checkout.show -> app.orders.Order.grand_total known_edge PROPERTY_READ app.checkout.show -> app.orders.Order.net_total diff --git a/graph/test/python/expected/36-call-through-model-decorators.tiers b/graph/test/python/expected/36-call-through-model-decorators.tiers index 2e14b72cd..e9d9a11ce 100644 --- a/graph/test/python/expected/36-call-through-model-decorators.tiers +++ b/graph/test/python/expected/36-call-through-model-decorators.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 19 +distinct call sites emitted: 20 --- by tier: edge ROWS, and the distinct SITES they cover --- 4 rows 4 sites ambiguous_unknown 6 rows 6 sites boundary_lib + 1 rows 1 sites callback_registered 9 rows 9 sites known_edge --- edge rows by call kind --- @@ -10,6 +11,7 @@ distinct call sites emitted: 19 6 DECORATOR_BARE 1 DECORATOR_CALL 1 METHOD_CALL + 1 METHOD_REFERENCE 3 PROPERTY_READ 7 SIMPLE_CALL @@ -24,8 +26,8 @@ distinct call sites emitted: 19 6 known_edge --- reconciling rows against the conserved site count --- - edge rows 19 + edge rows 20 minus extra rows from multi-target sites 0 - = tier/site pairs 19 + = tier/site pairs 20 engine's conserved site total 16 of which 3 are PROPERTY_READ, an edge with no call site (README decision 5) diff --git a/graph/test/python/expected/37-model-signals.edges b/graph/test/python/expected/37-model-signals.edges index 675318292..aed5cf7af 100644 --- a/graph/test/python/expected/37-model-signals.edges +++ b/graph/test/python/expected/37-model-signals.edges @@ -9,4 +9,5 @@ boundary_lib DECORATOR_CALL models. -> external:receiver boundary_lib METHOD_CALL models. -> external:post_save.connect boundary_lib METHOD_CALL models.Invoice. -> external:models.IntegerField boundary_lib METHOD_CALL models.Order. -> external:models.CharField +callback_registered METHOD_REFERENCE models. -> models.audit_order known_edge METHOD_CALL views.save_cache -> views.Cache.save diff --git a/graph/test/python/expected/37-model-signals.tiers b/graph/test/python/expected/37-model-signals.tiers index e87fd3fad..10a3c4b9a 100644 --- a/graph/test/python/expected/37-model-signals.tiers +++ b/graph/test/python/expected/37-model-signals.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 14 +distinct call sites emitted: 15 --- by tier: edge ROWS, and the distinct SITES they cover --- 8 rows 8 sites ambiguous_unknown 5 rows 5 sites boundary_lib + 1 rows 1 sites callback_registered 1 rows 1 sites known_edge --- edge rows by call kind --- 2 DECORATOR_APPLICATION 2 DECORATOR_CALL 10 METHOD_CALL + 1 METHOD_REFERENCE --- unresolved reasons --- 2 decorator_factory_result_untyped @@ -23,7 +25,7 @@ distinct call sites emitted: 14 1 known_edge --- reconciling rows against the conserved site count --- - edge rows 14 + edge rows 15 minus extra rows from multi-target sites 0 - = tier/site pairs 14 + = tier/site pairs 15 engine's conserved site total 14 diff --git a/graph/test/python/expected/38-app-wiring.edges b/graph/test/python/expected/38-app-wiring.edges index 39d93d28d..8813d217c 100644 --- a/graph/test/python/expected/38-app-wiring.edges +++ b/graph/test/python/expected/38-app-wiring.edges @@ -17,6 +17,12 @@ boundary_lib SIMPLE_CALL main. -> external:Starlette boundary_lib SIMPLE_CALL main. -> external:WebSocketRoute boundary_lib SIMPLE_CALL shop.jobs.nightly -> external:call_command boundary_lib SIMPLE_CALL shop.jobs.unknown -> external:call_command +callback_registered METHOD_REFERENCE main. -> main.cancel +callback_registered METHOD_REFERENCE main. -> main.feed +callback_registered METHOD_REFERENCE main. -> main.health +callback_registered METHOD_REFERENCE main. -> main.lifespan +callback_registered METHOD_REFERENCE main. -> main.plugin_hook +callback_registered METHOD_REFERENCE main. -> main.tick known_edge METHOD_CALL main. -> main.Registry.add_route known_edge SIMPLE_CALL main. -> main.configure known_edge SIMPLE_CALL shop.management.commands.close_orders.Command.handle -> shop.orders.close_all diff --git a/graph/test/python/expected/38-app-wiring.tiers b/graph/test/python/expected/38-app-wiring.tiers index 8496a4b97..ebe6e2258 100644 --- a/graph/test/python/expected/38-app-wiring.tiers +++ b/graph/test/python/expected/38-app-wiring.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 26 +distinct call sites emitted: 32 --- by tier: edge ROWS, and the distinct SITES they cover --- 8 rows 8 sites ambiguous_unknown 15 rows 15 sites boundary_lib + 6 rows 6 sites callback_registered 3 rows 3 sites known_edge --- edge rows by call kind --- @@ -11,6 +12,7 @@ distinct call sites emitted: 26 1 DECORATOR_BARE 4 DECORATOR_CALL 3 METHOD_CALL + 6 METHOD_REFERENCE 13 SIMPLE_CALL --- unresolved reasons --- @@ -25,7 +27,7 @@ distinct call sites emitted: 26 3 known_edge --- reconciling rows against the conserved site count --- - edge rows 26 + edge rows 32 minus extra rows from multi-target sites 0 - = tier/site pairs 26 + = tier/site pairs 32 engine's conserved site total 26 diff --git a/graph/test/python/expected/39-model-construction-hooks.edges b/graph/test/python/expected/39-model-construction-hooks.edges index 3c8b20b60..705a286e3 100644 --- a/graph/test/python/expected/39-model-construction-hooks.edges +++ b/graph/test/python/expected/39-model-construction-hooks.edges @@ -14,5 +14,7 @@ boundary_lib SIMPLE_CALL models.Widget.check_size -> builtin:max boundary_lib SIMPLE_CALL models.Widget.dump_size -> builtin:str boundary_lib SIMPLE_CALL service.make_gizmo -> builtin:object.__init__ boundary_lib SIMPLE_CALL service.make_plain -> builtin:object.__init__ +callback_registered METHOD_REFERENCE models.Gizmo. -> models.new_tags +callback_registered METHOD_REFERENCE models.Widget. -> models.new_id known_edge SELF_CALL models.Gadget.__init__ -> models.Gadget.check_size known_edge SIMPLE_CALL service.make_gadget -> models.Gadget.__init__ diff --git a/graph/test/python/expected/39-model-construction-hooks.tiers b/graph/test/python/expected/39-model-construction-hooks.tiers index a4e4b8aff..d1f820899 100644 --- a/graph/test/python/expected/39-model-construction-hooks.tiers +++ b/graph/test/python/expected/39-model-construction-hooks.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 21 +distinct call sites emitted: 23 --- by tier: edge ROWS, and the distinct SITES they cover --- 8 rows 8 sites ambiguous_unknown 11 rows 11 sites boundary_lib + 2 rows 2 sites callback_registered 2 rows 2 sites known_edge --- edge rows by call kind --- @@ -10,6 +11,7 @@ distinct call sites emitted: 21 3 DECORATOR_BARE 3 DECORATOR_CALL 2 METHOD_CALL + 2 METHOD_REFERENCE 1 SELF_CALL 9 SIMPLE_CALL @@ -27,7 +29,7 @@ distinct call sites emitted: 21 2 known_edge --- reconciling rows against the conserved site count --- - edge rows 21 + edge rows 23 minus extra rows from multi-target sites 0 - = tier/site pairs 21 + = tier/site pairs 23 engine's conserved site total 21 diff --git a/graph/test/python/expected/41-di-marker-forms.edges b/graph/test/python/expected/41-di-marker-forms.edges index 0a449dee2..3cc836455 100644 --- a/graph/test/python/expected/41-di-marker-forms.edges +++ b/graph/test/python/expected/41-di-marker-forms.edges @@ -8,3 +8,11 @@ boundary_lib SIMPLE_CALL handlers. -> external:APIRouter boundary_lib SIMPLE_CALL handlers. -> external:Depends boundary_lib SIMPLE_CALL handlers. -> external:FastAPI boundary_lib SIMPLE_CALL other. -> external:Depends +callback_registered METHOD_REFERENCE deps. -> deps.load_cart +callback_registered METHOD_REFERENCE deps.ShapeFilter. -> deps.load_shape_size +callback_registered METHOD_REFERENCE handlers. -> deps.check_app +callback_registered METHOD_REFERENCE handlers. -> deps.check_quota +callback_registered METHOD_REFERENCE handlers. -> deps.check_region +callback_registered METHOD_REFERENCE handlers. -> deps.load_owner +callback_registered METHOD_REFERENCE handlers. -> deps.load_user +callback_registered METHOD_REFERENCE other. -> other.check_other diff --git a/graph/test/python/expected/41-di-marker-forms.tiers b/graph/test/python/expected/41-di-marker-forms.tiers index 85ad730a4..e2ec4f880 100644 --- a/graph/test/python/expected/41-di-marker-forms.tiers +++ b/graph/test/python/expected/41-di-marker-forms.tiers @@ -1,12 +1,14 @@ -distinct call sites emitted: 32 +distinct call sites emitted: 40 --- by tier: edge ROWS, and the distinct SITES they cover --- 7 rows 7 sites ambiguous_unknown 25 rows 25 sites boundary_lib + 8 rows 8 sites callback_registered --- edge rows by call kind --- 7 DECORATOR_APPLICATION 8 DECORATOR_CALL + 8 METHOD_REFERENCE 17 SIMPLE_CALL --- unresolved reasons --- @@ -18,7 +20,7 @@ distinct call sites emitted: 32 25 boundary_lib --- reconciling rows against the conserved site count --- - edge rows 32 + edge rows 40 minus extra rows from multi-target sites 0 - = tier/site pairs 32 + = tier/site pairs 40 engine's conserved site total 32 diff --git a/graph/test/python/expected/42-signal-forms.edges b/graph/test/python/expected/42-signal-forms.edges index 456f0118c..f8aa33ff1 100644 --- a/graph/test/python/expected/42-signal-forms.edges +++ b/graph/test/python/expected/42-signal-forms.edges @@ -4,6 +4,7 @@ ambiguous_unknown METHOD_CALL orders.close -> - ambiguous_unknown METHOD_CALL orders.ship -> - boundary_lib DECORATOR_CALL handlers. -> external:receiver boundary_lib SIMPLE_CALL signals. -> builtin:object.__init__ +callback_registered METHOD_REFERENCE handlers. -> handlers.on_closed known_edge DECORATOR_CALL handlers. -> handlers.remember known_edge METHOD_CALL orders.not_a_signal -> orders.Outbox.asend known_edge METHOD_CALL orders.pay -> signals.Signal.asend diff --git a/graph/test/python/expected/42-signal-forms.tiers b/graph/test/python/expected/42-signal-forms.tiers index e780b18a8..f608211b9 100644 --- a/graph/test/python/expected/42-signal-forms.tiers +++ b/graph/test/python/expected/42-signal-forms.tiers @@ -1,14 +1,16 @@ -distinct call sites emitted: 22 +distinct call sites emitted: 23 --- by tier: edge ROWS, and the distinct SITES they cover --- 8 rows 8 sites ambiguous_unknown 9 rows 9 sites boundary_lib + 1 rows 1 sites callback_registered 5 rows 5 sites known_edge --- edge rows by call kind --- 5 DECORATOR_APPLICATION 5 DECORATOR_CALL 7 METHOD_CALL + 1 METHOD_REFERENCE 5 SIMPLE_CALL --- unresolved reasons --- @@ -22,7 +24,7 @@ distinct call sites emitted: 22 5 known_edge --- reconciling rows against the conserved site count --- - edge rows 22 + edge rows 23 minus extra rows from multi-target sites 0 - = tier/site pairs 22 + = tier/site pairs 23 engine's conserved site total 22 diff --git a/graph/test/python/expected/project-name-collision.tiers b/graph/test/python/expected/project-name-collision.tiers index ee405f873..d47e460f7 100644 --- a/graph/test/python/expected/project-name-collision.tiers +++ b/graph/test/python/expected/project-name-collision.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 91 +distinct call sites emitted: 92 --- by tier: edge ROWS, and the distinct SITES they cover --- 1 rows 1 sites ambiguous_unknown 34 rows 34 sites boundary_lib + 1 rows 1 sites callback_registered 49 rows 49 sites known_edge 25 rows 7 sites multi_inferred @@ -12,6 +13,7 @@ distinct call sites emitted: 91 1 DECORATOR_BARE 1 DECORATOR_CALL 25 METHOD_CALL + 1 METHOD_REFERENCE 5 SELF_CALL 58 SIMPLE_CALL 6 SUPER_CALL @@ -27,7 +29,7 @@ distinct call sites emitted: 91 7 multi_inferred --- reconciling rows against the conserved site count --- - edge rows 109 + edge rows 110 minus extra rows from multi-target sites 18 - = tier/site pairs 91 + = tier/site pairs 92 engine's conserved site total 91 diff --git a/graph/test/python/expected/project-two-service-fastapi.tiers b/graph/test/python/expected/project-two-service-fastapi.tiers index 0b54e3f31..d50f26912 100644 --- a/graph/test/python/expected/project-two-service-fastapi.tiers +++ b/graph/test/python/expected/project-two-service-fastapi.tiers @@ -1,8 +1,9 @@ -distinct call sites emitted: 229 +distinct call sites emitted: 239 --- by tier: edge ROWS, and the distinct SITES they cover --- 10 rows 10 sites ambiguous_unknown 71 rows 71 sites boundary_lib + 10 rows 10 sites callback_registered 126 rows 126 sites known_edge 59 rows 22 sites multi_inferred @@ -12,6 +13,7 @@ distinct call sites emitted: 229 8 DECORATOR_BARE 9 DECORATOR_CALL 82 METHOD_CALL + 10 METHOD_REFERENCE 2 PROPERTY_READ 27 SELF_CALL 104 SIMPLE_CALL @@ -29,8 +31,8 @@ distinct call sites emitted: 229 22 multi_inferred --- reconciling rows against the conserved site count --- - edge rows 266 + edge rows 276 minus extra rows from multi-target sites 37 - = tier/site pairs 229 + = tier/site pairs 239 engine's conserved site total 227 of which 2 are PROPERTY_READ, an edge with no call site (README decision 5) diff --git a/graph/test/python/tools/literal_gate.py b/graph/test/python/tools/literal_gate.py index 0e7af6f71..b42121347 100644 --- a/graph/test/python/tools/literal_gate.py +++ b/graph/test/python/tools/literal_gate.py @@ -13,7 +13,9 @@ provenance "client" / "lib" -- assigned by which fact file a row is staged into, not by anything in the source. confidence tiers "known_edge", "multi_inferred", "boundary_lib", - "ambiguous_unknown" -- the engine's own output vocabulary. + "ambiguous_unknown", "callback_registered" (a method + reference, call-edge-generation/method-reference.dl) -- the + engine's own output vocabulary. diagnostic labels "no_rule", "escape_hatch", ... These are OUTPUT arguments of call_unresolvable / site_reason / expr_type_untypable and are never matched against input, so they cannot bias resolution. @@ -48,7 +50,7 @@ raise SystemExit(f"literal gate: engine dir not found at {ENGINE} -- refusing to pass vacuously") PROVENANCE = {"client", "lib", "external", "builtin"} -TIERS = {"known_edge", "multi_inferred", "boundary_lib", "ambiguous_unknown"} +TIERS = {"known_edge", "multi_inferred", "boundary_lib", "ambiguous_unknown", "callback_registered"} SCAFFOLD = {"", "-", ".", "/", "true", "false", "lib:", "builtin:", "external:", "untyped_receiver:", "builtin:object.__init__", "_total_sites"} # "/" joins "." above: both are separators fixed by a specification (a dotted name, a URL diff --git a/graph/test/python/torture/expected/torture.edges b/graph/test/python/torture/expected/torture.edges index 557d3208c..54c66e115 100644 --- a/graph/test/python/torture/expected/torture.edges +++ b/graph/test/python/torture/expected/torture.edges @@ -350,6 +350,11 @@ boundary_lib SUBSCRIPT_CALL f30_builtin_elements.an_annotated_local -> builtin:s boundary_lib SUBSCRIPT_CALL f30_builtin_elements.annotated_parameter_indexed -> builtin:str.upper boundary_lib SUBSCRIPT_CALL f30_builtin_elements.through_any_object -> builtin:str.upper boundary_lib SUPER_CALL f41_class_attribute_absent.ModelMeta.__new__ -> builtin:type.__new__ +callback_registered METHOD_REFERENCE f12_value_flow.method_alias_on_class -> f12_value_flow.Widget.make +callback_registered METHOD_REFERENCE f12_value_flow.method_alias_on_instance -> f12_value_flow.Widget.emit +callback_registered METHOD_REFERENCE rxfacade. -> rxaccel.boost +callback_registered METHOD_REFERENCE rxfacade. -> rxlib.fallback +callback_registered METHOD_REFERENCE rxfacade. -> rxlib.slow known_edge CHAINED_CALL f01_inheritance.client_chain -> f01_inheritance.LocalBase.kind known_edge CHAINED_CALL f01_inheritance.client_chain -> f01_inheritance.LocalLeaf.kind known_edge CHAINED_CALL f02_callables.attribute_callables -> f02_callables.HoldsCallables.run diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py index 5aa67cc24..98764b89a 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py @@ -112,7 +112,7 @@ # C#: a call the language writes for a statement: `using` -> Dispose, `foreach` -> GetEnumerator, `{ a, b }` -> Add 'using_dispose': 'call', 'foreach_enumerator': 'call', 'collection_add': 'call', # the callable is named, not called at that line — it runs when whoever took it runs it - 'ref': 'method-ref', + 'ref': 'method-ref', 'METHOD_REFERENCE': 'method-ref', # a declaration handed to a decorator, which is what wires most framework handlers up 'DECORATOR_APPLICATION': 'decorator', 'DECORATOR_ATTRIBUTE': 'decorator', 'DECORATOR_BARE': 'decorator', 'DECORATOR_CALL': 'decorator', diff --git a/tests/cases/python/method-reference-reaches-its-target/case.json b/tests/cases/python/method-reference-reaches-its-target/case.json new file mode 100644 index 000000000..d96498e7c --- /dev/null +++ b/tests/cases/python/method-reference-reaches-its-target/case.json @@ -0,0 +1,230 @@ +{ + "lang": "python", + "src": ".", + "checks": [ + { + "why": "a bound method handed to an executor as a VALUE (`loop.run_in_executor(None, svc.cancel, oid)`) is called by the executor, by no call site the graph sees. Its receiver is typed exactly as `svc.cancel()` would be, so the function that queues it reaches it, labelled as handed over, never as a call", + "run": [ + "path", + "executor_svc", + "OrderService.cancel" + ], + "want": [ + "executor_svc → OrderService.cancel: 1 of 1 target(s) reached" + ], + "avoid": [ + "the two are independent in this graph" + ] + }, + { + "why": "every resolved shape that hands cancel over is a dependent of it: a positional argument, a keyword argument, functools.partial, a receiver the function constructed itself", + "run": [ + "impact", + "OrderService.cancel", + "--limit", + "50" + ], + "want": [ + "executor_svc", + "keyword_register", + "partial_cancel", + "constructed_receiver", + "handed over as a value" + ], + "avoid": [ + "untyped_receiver", + "by_string" + ] + }, + { + "why": "control: a same-named cancel on an unrelated class gains none of OrderService.cancel's hand-offs", + "run": [ + "impact", + "LedgerService.cancel", + "--limit", + "50" + ], + "avoid": [ + "executor_svc", + "keyword_register", + "partial_cancel", + "constructed_receiver", + "handed over as a value" + ] + }, + { + "why": "and the closure continues from the hand-off: the test that drives executor_svc reaches cancel", + "run": [ + "impact", + "OrderService.cancel", + "--tests" + ], + "want": [ + "test_queue.py" + ], + "avoid": [ + "test_ledger.py" + ] + }, + { + "why": "a bound method passed as a sort key, and one read inside a lambda: the lambda is the reader", + "run": [ + "impact", + "OrderService.rank", + "--limit", + "50" + ], + "want": [ + "sort_key", + "lambda_holder" + ], + "avoid": [ + "ledger_rank" + ] + }, + { + "why": "control: a same-named method on an UNRELATED class keeps its own reader and gains none of OrderService's", + "run": [ + "impact", + "LedgerService.rank", + "--limit", + "50" + ], + "want": [ + "ledger_rank" + ], + "avoid": [ + "sort_key", + "lambda_holder" + ] + }, + { + "why": "map() over a bound method and over the method read off its CLASS (an unbound reference)", + "run": [ + "impact", + "OrderService.convert", + "--limit", + "50" + ], + "want": [ + "map_convert", + "unbound_reference" + ] + }, + { + "why": "threading.Thread(target=...), a list of callbacks, a signal handler, a view_func keyword", + "run": [ + "path", + "thread_target", + "OrderService.run_job" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a callback stored in a list literal", + "run": [ + "path", + "callback_list", + "OrderService.on_event" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a signal handler", + "run": [ + "path", + "signal_connect", + "OrderService.handle" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a view function registered by keyword", + "run": [ + "path", + "url_rule", + "OrderService.index" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "self-references: submitted to the instance's own pool, used as a sort key, and returned (an escaping reference)", + "run": [ + "path", + "Worker.submit_bound_self", + "Worker._do_work" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a self method as a sort key", + "run": [ + "path", + "Worker.sort_by_self_key", + "Worker._rank" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a returned bound method escapes to whoever called escapes()", + "run": [ + "path", + "Worker.escapes", + "Worker._handle" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "a plain module function handed to a pool is now the engine's resolved reference, not a name match", + "run": [ + "path", + "submit_plain_function", + "top_level_job" + ], + "want": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "control: an UNTYPED receiver's reference resolves to nothing, exactly like `thing.cancel()`, so there is no edge to OrderService.cancel", + "run": [ + "path", + "untyped_receiver", + "OrderService.cancel" + ], + "expect_error": true, + "avoid": [ + "1 of 1 target(s) reached" + ] + }, + { + "why": "control: the method's name in a string, an attribute read OFF the method object (`svc.audit.__name__`), str() of it, and an identity test on it do not run it", + "run": [ + "impact", + "OrderService.audit", + "--limit", + "50" + ], + "avoid": [ + "by_string", + "reads_its_name", + "prints_it", + "compares_it" + ] + } + ] +} diff --git a/tests/cases/python/method-reference-reaches-its-target/orders.py b/tests/cases/python/method-reference-reaches-its-target/orders.py new file mode 100644 index 000000000..44a34d5f0 --- /dev/null +++ b/tests/cases/python/method-reference-reaches-its-target/orders.py @@ -0,0 +1,163 @@ +"""Methods and functions handed over as REFERENCES: named as a value, called later by whoever holds them.""" +import functools +import signal +import threading +from concurrent.futures import ThreadPoolExecutor + + +class OrderService: + def cancel(self, oid): + return oid + + def rank(self, x): + return x + + def convert(self, row): + return row + + def on_event(self, e): + return e + + def handle(self, sig, frame): + return sig + + def run_job(self): + return 1 + + def index(self): + return "ok" + + def audit(self, oid): + return oid + + +class LedgerService: + """Same method names as OrderService, and no relation to it.""" + + def cancel(self, oid): + return oid + + def rank(self, x): + return x + + +def top_level_job(item): + return item + + +class Worker: + def __init__(self): + self.pool = ThreadPoolExecutor() + + def _do_work(self, item): + return item + + def _rank(self, x): + return x + + def _handle(self, e): + return e + + def submit_bound_self(self, item): + return self.pool.submit(self._do_work, item) + + def sort_by_self_key(self, xs): + return sorted(xs, key=self._rank) + + def escapes(self): + return self._handle + + +def submit_plain_function(pool, item): + return pool.submit(top_level_job, item) + + +def executor_svc(loop, svc: OrderService, oid): + return loop.run_in_executor(None, svc.cancel, oid) + + +def thread_target(svc: OrderService): + t = threading.Thread(target=svc.run_job) + t.start() + return t + + +def callback_list(handler: OrderService): + callbacks = [handler.on_event] + return callbacks + + +def register(handler): + return handler + + +def keyword_register(svc: OrderService): + return register(handler=svc.cancel) + + +def sort_key(svc: OrderService, xs): + return sorted(xs, key=svc.rank) + + +def map_convert(svc: OrderService, rows): + return list(map(svc.convert, rows)) + + +def unbound_reference(rows): + return list(map(OrderService.convert, rows)) + + +def partial_cancel(svc: OrderService, oid): + return functools.partial(svc.cancel, oid) + + +def signal_connect(svc: OrderService): + signal.signal(signal.SIGTERM, svc.handle) + + +def url_rule(app, views: OrderService): + app.add_url_rule("/x", view_func=views.index) + + +def constructed_receiver(oid): + svc = OrderService() + return functools.partial(svc.cancel, oid) + + +def lambda_holder(svc: OrderService, xs): + return sorted(xs, key=lambda x: svc.rank) + + +# ── near-miss controls ─────────────────────────────────────────────────────── +def untyped_receiver(executor, thing, oid): + """`thing` has no type: `thing.cancel` may be any class's cancel, so it resolves to none.""" + return executor.submit(thing.cancel, oid) + + +def ledger_rank(ledger: LedgerService, xs): + """LedgerService.rank, not OrderService.rank.""" + return sorted(xs, key=ledger.rank) + + +def by_string(svc: OrderService): + """The method's NAME in a string is not a reference to it.""" + return getattr_name("audit") + + +def getattr_name(n): + return n + + +def reads_its_name(svc: OrderService): + """Reading an attribute OF the method object does not run it.""" + return svc.audit.__name__ + + +def prints_it(svc: OrderService): + """str() of the method object runs the function object's __str__, not the method.""" + return str(svc.audit) + + +def compares_it(svc: OrderService, cb): + """An identity test on the method object does not run it.""" + return cb is svc.audit diff --git a/tests/cases/python/method-reference-reaches-its-target/tests/test_ledger.py b/tests/cases/python/method-reference-reaches-its-target/tests/test_ledger.py new file mode 100644 index 000000000..5ba83160c --- /dev/null +++ b/tests/cases/python/method-reference-reaches-its-target/tests/test_ledger.py @@ -0,0 +1,5 @@ +from orders import LedgerService, ledger_rank + + +def test_ledger_rank(): + assert ledger_rank(LedgerService(), [2, 1]) == [1, 2] diff --git a/tests/cases/python/method-reference-reaches-its-target/tests/test_queue.py b/tests/cases/python/method-reference-reaches-its-target/tests/test_queue.py new file mode 100644 index 000000000..8d8de3896 --- /dev/null +++ b/tests/cases/python/method-reference-reaches-its-target/tests/test_queue.py @@ -0,0 +1,9 @@ +from unittest import mock + +from orders import OrderService, executor_svc + + +def test_queues_the_cancel(): + loop = mock.Mock() + executor_svc(loop, OrderService(), "o-1") + loop.run_in_executor.assert_called_once() From 6ec89f4334d69103276a00df8331d36c688c4942 Mon Sep 17 00:00:00 2001 From: swapnil Date: Thu, 15 Oct 2026 00:28:01 -0700 Subject: [PATCH 10/18] parser(ts): a plain JSX tag is an element row, never a call `
` produced no expression row in the TypeScript parser, so nothing in the IR said a handler or a ref callback sat on a plain tag: an attribute's value existed only as an unattached embedded expression. The JavaScript parser records the same tag as an element. An intrinsic tag (`
`, ``, ``) is now a JSX_INTRINSIC_ELEMENT row with no child and no call site, rooted wherever it sits (a return, a ternary branch, another element's children, a function inside an attribute). It is never added to the call nodes, so nothing can resolve it into JSX.IntrinsicElements. Component tags keep their JSX_ELEMENT / JSX_SELF_CLOSING rows and JSX_COMPONENT_CALL sites. --- .../expressions/TsExpressionKind.ts | 14 ++++- .../extractors/ts-expression-extractor.ts | 12 +++-- .../extractors/ts-expression-walker.ts | 10 ++-- parser/src/schema/typescript/schema.json | 1 + parser/src/test/typescript-tests.ts | 54 +++++++++++++++++-- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/parser/src/enums/typescript/expressions/TsExpressionKind.ts b/parser/src/enums/typescript/expressions/TsExpressionKind.ts index 6d42c2088..ebb5b03d2 100644 --- a/parser/src/enums/typescript/expressions/TsExpressionKind.ts +++ b/parser/src/enums/typescript/expressions/TsExpressionKind.ts @@ -33,8 +33,10 @@ * A JSX element whose tag names a component IS a call to it, with the whole * props object as argument 0: JSX_ELEMENT / JSX_SELF_CLOSING, 1:1 with a call * site, with the tag as its METHOD_NAME child. Attribute values and children are - * rooted as their own JSX_EMBEDDED_EXPRESSION trees. An intrinsic element emits - * no row. + * rooted as their own JSX_EMBEDDED_EXPRESSION trees. An intrinsic element + * (`
`, ``, ``) is JSX_INTRINSIC_ELEMENT: a row with no + * children and no call site, so it exists only as the element its attributes + * belong to. * * Schema §4.14 c0, §4.15.1. */ @@ -99,6 +101,14 @@ export enum TsExpressionKind { JSX_ELEMENT = 'JSX_ELEMENT', /** `` naming a component. 1:1 with a call site. */ JSX_SELF_CLOSING = 'JSX_SELF_CLOSING', + /** + * `
`, ``, `` -- a tag that names no + * declaration. NOT a call: no call site, no METHOD_NAME child, so nothing can + * resolve it into `JSX.IntrinsicElements`. The row exists so that a plain + * tag's attributes (an event handler, a `ref` callback) have an element to + * belong to, as the JavaScript parser's JSX_INTRINSIC_ELEMENT does. + */ + JSX_INTRINSIC_ELEMENT = 'JSX_INTRINSIC_ELEMENT', /** `delete a`, `typeof a`, `void a` — `operatorString` says which. */ DELETE_TYPEOF_VOID = 'DELETE_TYPEOF_VOID', /** `a, b` — the comma operator. */ diff --git a/parser/src/parsers/typescript/extractors/ts-expression-extractor.ts b/parser/src/parsers/typescript/extractors/ts-expression-extractor.ts index f4183b369..5c86cf7c6 100644 --- a/parser/src/parsers/typescript/extractors/ts-expression-extractor.ts +++ b/parser/src/parsers/typescript/extractors/ts-expression-extractor.ts @@ -47,7 +47,9 @@ import { isVueTemplateTag } from '@/utils/vue-sfc'; * * A JSX element naming a component is a call (§4.15.1): a JSX_ELEMENT or * JSX_SELF_CLOSING row with its tag as the METHOD_NAME child, and a - * JSX_COMPONENT_CALL site. `JSX_ATTRIBUTE_VALUE` and `JSX_CHILD` stay reserved: + * JSX_COMPONENT_CALL site. An intrinsic element (`
`) is a + * JSX_INTRINSIC_ELEMENT row with no child and no call site. + * `JSX_ATTRIBUTE_VALUE` and `JSX_CHILD` stay reserved: * attribute values and children are rooted as their own trees by the walker, and * the gate asserts the emptiness so that switching them on shows up as a failure. */ @@ -453,10 +455,12 @@ export function expressionKindOf(node: ts.Node): TsExpressionKind | undefined { case ts.SyntaxKind.JsxElement: case ts.SyntaxKind.JsxSelfClosingElement: { // `` IS a call to Badge. `
` is an intrinsic element that no - // project declaration can bind, so it produces no row -- a call site that - // can only ever resolve into `JSX.IntrinsicElements` would be noise. + // project declaration can bind: a call site that can only ever resolve + // into `JSX.IntrinsicElements` would be noise, so it is its own kind, + // which is never a call (see `emit`) and carries no tag child. The row is + // what a plain tag's attributes -- `onClick={h}`, `ref={r}` -- belong to. if (jsxTagReference(node) === undefined) { - return undefined; + return TsExpressionKind.JSX_INTRINSIC_ELEMENT; } return ts.isJsxElement(node) ? TsExpressionKind.JSX_ELEMENT : TsExpressionKind.JSX_SELF_CLOSING; } diff --git a/parser/src/parsers/typescript/extractors/ts-expression-walker.ts b/parser/src/parsers/typescript/extractors/ts-expression-walker.ts index 1b82d1018..5fda834bf 100644 --- a/parser/src/parsers/typescript/extractors/ts-expression-walker.ts +++ b/parser/src/parsers/typescript/extractors/ts-expression-walker.ts @@ -2,7 +2,6 @@ import * as ts from 'typescript'; import { ExpressionOwner, - jsxTagReference, TsExpressionExtractor, } from '@/parsers/typescript/extractors/ts-expression-extractor'; import { TsExpressionOwnerKind, TsRootContext } from '@/enums/typescript/expressions'; @@ -513,11 +512,12 @@ export class TsExpressionWalker { // attribute values, children, and spreads at once, wherever the JSX sits. // // A component ELEMENT is a call in its own right: `` calls Badge. - // One that is already a row -- `cond ? : `, `x && ` -- was - // reached by the worklist as an operand; any other is rooted here, which - // is what reaches an element nested in another element's children. + // An intrinsic one (`
`) is a JSX_INTRINSIC_ELEMENT row and no call, + // the element its attributes belong to. One that is already a row -- + // `cond ? : `, `x &&
` -- was reached by the worklist as an + // operand; any other is rooted here, which is what reaches an element + // nested in another element's children. if ((ts.isJsxElement(child) || ts.isJsxSelfClosingElement(child)) - && jsxTagReference(child) !== undefined && !this.options.extractor.rowByNode.has(nodeId(child, this.sf))) { this.root(child, TsRootContext.JSX_EMBEDDED_EXPRESSION); return; diff --git a/parser/src/schema/typescript/schema.json b/parser/src/schema/typescript/schema.json index 590c68a26..b91264822 100644 --- a/parser/src/schema/typescript/schema.json +++ b/parser/src/schema/typescript/schema.json @@ -829,6 +829,7 @@ "FUNCTION_EXPRESSION", "IDENTIFIER_REFERENCE", "JSX_ELEMENT", + "JSX_INTRINSIC_ELEMENT", "JSX_SELF_CLOSING", "LITERAL", "NEW_EXPRESSION", diff --git a/parser/src/test/typescript-tests.ts b/parser/src/test/typescript-tests.ts index fd295003d..d10428fd9 100644 --- a/parser/src/test/typescript-tests.ts +++ b/parser/src/test/typescript-tests.ts @@ -1414,7 +1414,9 @@ function irCompleteness(): number { * The component element is a call too: `` is `Badge({...})`, a * JSX_COMPONENT_CALL site, wherever it sits -- a child, a ternary branch, the * body of a `.map` arrow. The CONTROL is the intrinsic element: `
` and - * `` name no declaration and must produce no call site at all. + * `` name no declaration and must produce no call site at all. It is + * still an element, a childless JSX_INTRINSIC_ELEMENT row, so that + * `', + ' {flag && }', + ' ', '
', ' );', '}', @@ -1493,6 +1500,15 @@ async function jsxBraceExpressionsWalked(): Promise { } } let expected = 0; + // Expression rows by position, so an element's row can be asserted. + const exprRows = relation(outputDir, 'all-typescript-expressions.csv') + .filter((r) => (modulePath.get(r.tsModuleLinkHash ?? '') ?? '').endsWith('panel.tsx')); + const exprKindsAt = new Map(); + for (const r of exprRows) { + const key = `panel.tsx:${r.startLine}:${r.startColumn}`; + exprKindsAt.set(key, [...(exprKindsAt.get(key) ?? []), r.kind ?? '']); + } + const parents = new Set(exprRows.map((r) => r.parentExpressionHash ?? '')); let components = 0; let intrinsics = 0; const walk = (node: ts.Node): void => { @@ -1507,12 +1523,43 @@ async function jsxBraceExpressionsWalked(): Promise { failures.push(`no JSX_COMPONENT_CALL at ${key} for \`${node.getText(sf).slice(0, 44)}\`` + ` (found ${kindAt.get(key) ?? 'nothing'})`); } + // The control: a component keeps its call-shaped row, never the intrinsic kind. + const want = ts.isJsxElement(node) ? 'JSX_ELEMENT' : 'JSX_SELF_CLOSING'; + const kinds = exprKindsAt.get(key) ?? []; + if (!kinds.includes(want) || kinds.includes('JSX_INTRINSIC_ELEMENT')) { + failures.push(`component at ${key} has expression kind(s) [${kinds.join(', ')}], ` + + `expected ${want}`); + } } else { intrinsics += 1; if (emitted.has(key)) { failures.push(`intrinsic \`<${tag.getText(sf)}>\` at ${key} became a call site ` + `(${kindAt.get(key)}) -- it names no declaration`); } + // ...but it IS an element: one JSX_INTRINSIC_ELEMENT row, with no child + // (no tag reference for anything to resolve), so its attributes have an + // element to belong to, as in the JavaScript parser. + const rows = exprRows.filter((r) => `panel.tsx:${r.startLine}:${r.startColumn}` === key + && r.kind === 'JSX_INTRINSIC_ELEMENT'); + if (rows.length !== 1) { + failures.push(`intrinsic \`<${tag.getText(sf)}>\` at ${key} has ${rows.length} ` + + 'JSX_INTRINSIC_ELEMENT row(s), expected 1'); + } else if (parents.has(rows[0]!.tsExpressionUniqueHash ?? '')) { + failures.push(`intrinsic \`<${tag.getText(sf)}>\` at ${key} has a child row -- ` + + 'its tag references nothing, so nothing hangs off it'); + } + // An attribute's value is still its own rooted tree. + const attributes = ts.isJsxElement(node) ? node.openingElement.attributes : node.attributes; + for (const attribute of attributes.properties) { + if (ts.isJsxAttribute(attribute) && attribute.initializer + && ts.isJsxExpression(attribute.initializer) && attribute.initializer.expression) { + const v = sf.getLineAndCharacterOfPosition(attribute.initializer.expression.getStart(sf)); + const vkey = `panel.tsx:${v.line + 1}:${v.character + 1}`; + if ((exprKindsAt.get(vkey) ?? []).length === 0) { + failures.push(`attribute \`${attribute.getText(sf)}\` at ${vkey} has no value row`); + } + } + } } } if (ts.isCallExpression(node) || ts.isNewExpression(node)) { @@ -1545,7 +1592,8 @@ async function jsxBraceExpressionsWalked(): Promise { console.log(` ${expected} call/new node(s) in JSX attributes, children, spreads and a ` + `nested map; ${expected - missingCalls} emitted; ${components} component element(s) as ` + - `calls, ${intrinsics} intrinsic element(s) as none; ${RESERVED_TSX_VALUES.size} reserved ` + + `calls, ${intrinsics} intrinsic element(s) as JSX_INTRINSIC_ELEMENT rows and no call; ` + + `${RESERVED_TSX_VALUES.size} reserved ` + 'value(s) still empty'); for (const f of failures.slice(0, 10)) console.log(` ${f}`); fs.rmSync(root, { recursive: true, force: true }); @@ -4721,7 +4769,7 @@ const CHECKS: Check[] = [ { name: 'tsc-adjudicated resolution', proves: 'every resolved call target equals getResolvedSignature', run: tscAdjudicatedResolution }, { name: 'type-only isolation', proves: 'no type-only construct reaches the call graph', run: typeOnlyIsolation }, { name: 'TSX reserved but empty', proves: 'the JSX attribute and child edge roles carry no rows', run: tsxReservedButEmpty }, - { name: 'JSX brace expressions are walked', proves: 'a call inside a JSX brace is an ordinary call site, a component element is a JSX_COMPONENT_CALL, an intrinsic element is none', run: jsxBraceExpressionsWalked }, + { name: 'JSX brace expressions are walked', proves: 'a call inside a JSX brace is an ordinary call site, a component element is a JSX_COMPONENT_CALL, an intrinsic element is a childless JSX_INTRINSIC_ELEMENT row and no call', run: jsxBraceExpressionsWalked }, { name: 'destructuring records its source', proves: 'a bound name carries the property or index it binds, so a renamed or positional binding is recoverable', run: destructuringRecordsItsSource }, { name: 'no emitted value can split a row', proves: 'no value contains a character a consumer treats as a line break, so a row cannot tear', run: noValueCanSplitARow }, { name: 'signatures link their return type', proves: 'a call through any callable shape reaches a result type, so a chain does not stop at it', run: signaturesLinkTheirReturnType }, From 47dacac63f27d1faa028791e27b41ed87e53b35f Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:38:02 -0700 Subject: [PATCH 11/18] parser(ts): a plain JSX element's row names its tag A JSX_INTRINSIC_ELEMENT row said a plain tag was there but not which one: ``, `
` and `