From b7e541e684105e13e0c6a88b4c26a13e41ae0c6a Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Sun, 15 Nov 2026 17:59:27 -0600 Subject: [PATCH 1/7] docs: add interactive Python runtime architecture GitHub Pages Add a code-grounded, multi-repository interactive Architecture Atlas for the Azure Functions Python runtime and wire up automated GitHub Pages builds: - Extract live Python AST, Rust (R2P2), and FunctionRpc.proto symbols across python-worker, python-library, python-extensions, and language-worker-protobuf (eng/docs/extract_arch.py + eng/docs/phases_spec.py). - Model all 3 worker generations (<= 3.12 monolith, 3.13-3.14 proxy_worker + runtimes split, 3.15+ R2P2 Rust/PyO3 worker) across all 7 lifecycle phases (Init, Specialization, Metadata, Functions Load, Invocation, Worker Status, and Worker Logs). - Render algorithmic compound swimlane topologies, multi-actor sequence flows, cross-era phase comparisons, and deterministic Mermaid exports (docs/site/). - Add unit tests (eng/docs/tests/test_extract_arch.py) and GitHub Pages CI/CD (.github/workflows/architecture-pages.yml). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/architecture-pages.yml | 107 ++ .gitignore | 5 + docs/site/.nojekyll | 0 docs/site/README.md | 24 + docs/site/assets/app.js | 1202 ++++++++++++++++++++ docs/site/assets/layoutEngine.js | 395 +++++++ docs/site/assets/styles.css | 757 +++++++++++++ docs/site/index.html | 19 + eng/docs/extract_arch.py | 1309 ++++++++++++++++++++++ eng/docs/phases_spec.py | 792 +++++++++++++ eng/docs/tests/test_extract_arch.py | 93 ++ 11 files changed, 4703 insertions(+) create mode 100644 .github/workflows/architecture-pages.yml create mode 100644 docs/site/.nojekyll create mode 100644 docs/site/README.md create mode 100644 docs/site/assets/app.js create mode 100644 docs/site/assets/layoutEngine.js create mode 100644 docs/site/assets/styles.css create mode 100644 docs/site/index.html create mode 100644 eng/docs/extract_arch.py create mode 100644 eng/docs/phases_spec.py create mode 100644 eng/docs/tests/test_extract_arch.py diff --git a/.github/workflows/architecture-pages.yml b/.github/workflows/architecture-pages.yml new file mode 100644 index 000000000..e2d273adc --- /dev/null +++ b/.github/workflows/architecture-pages.yml @@ -0,0 +1,107 @@ +name: Architecture GitHub Pages + +on: + push: + branches: [dev] + paths: + - "workers/**" + - "runtimes/**" + - "eng/docs/**" + - "docs/site/**" + - ".github/workflows/architecture-pages.yml" + pull_request: + branches: [dev] + paths: + - "workers/**" + - "runtimes/**" + - "eng/docs/**" + - "docs/site/**" + - ".github/workflows/architecture-pages.yml" + repository_dispatch: + types: [sibling-repo-updated] + schedule: + # Daily cross-repo sync with python-library, python-extensions, and language-worker-protobuf + - cron: "0 8 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: "pages-architecture-${{ github.ref }}" + cancel-in-progress: true + +jobs: + build: + name: Build & Validate Architecture Site + runs-on: ubuntu-latest + steps: + - name: Checkout python-worker + uses: actions/checkout@v4 + + - name: Checkout azure-functions-python-library + uses: actions/checkout@v4 + with: + repository: Azure/azure-functions-python-library + path: _sibling_repos/python-library + depth: 1 + + - name: Checkout azure-functions-python-extensions + uses: actions/checkout@v4 + with: + repository: Azure/azure-functions-python-extensions + path: _sibling_repos/python-extensions + depth: 1 + + - name: Checkout azure-functions-language-worker-protobuf + uses: actions/checkout@v4 + with: + repository: Azure/azure-functions-language-worker-protobuf + path: _sibling_repos/language-worker-protobuf + depth: 1 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Run architecture extractor unit tests + run: python3 -m unittest discover -s eng/docs/tests -v + + - name: Extract multi-repo architecture graph + run: python3 eng/docs/extract_arch.py --sibling-root _sibling_repos + + - name: Upload preview artifact + uses: actions/upload-artifact@v4 + with: + name: architecture-site + path: ./docs/site + include-hidden-files: true + + - name: Upload GitHub Pages artifact + if: github.event_name != 'pull_request' + uses: actions/upload-pages-artifact@v3 + with: + path: ./docs/site + + deploy: + name: Deploy to GitHub Pages + if: github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Configure Pages + uses: actions/configure-pages@v5 + with: + enablement: true + continue-on-error: true + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index ccab2a249..c461878b4 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,8 @@ tests/**/extensions.csproj __blobstorage__/* __queuestorage__/* __azurite* + +# Generated architecture site artifacts +docs/site/assets/architecture-graph.json +docs/site/assets/architecture-data.js +_sibling_repos/ diff --git a/docs/site/.nojekyll b/docs/site/.nojekyll new file mode 100644 index 000000000..e69de29bb diff --git a/docs/site/README.md b/docs/site/README.md new file mode 100644 index 000000000..45b123674 --- /dev/null +++ b/docs/site/README.md @@ -0,0 +1,24 @@ +# Azure Functions Python Runtime — Architecture Atlas (`docs/site`) + +Interactive GitHub Pages site documenting the Azure Functions Python runtime architecture across: + +- **4 Linked Repositories**: + - [`Azure/azure-functions-python-worker`](https://github.com/Azure/azure-functions-python-worker) + - [`Azure/azure-functions-python-library`](https://github.com/Azure/azure-functions-python-library) + - [`Azure/azure-functions-python-extensions`](https://github.com/Azure/azure-functions-python-extensions) + - [`Azure/azure-functions-language-worker-protobuf`](https://github.com/Azure/azure-functions-language-worker-protobuf) +- **3 Worker Generations**: Python `≤ 3.12` (`workers/azure_functions_worker`), Python `3.13–3.14` (`workers/proxy_worker` + `runtimes/`), and Python `3.15+` (`workers/r2p2` + `PyO3` + `runtimes/`). +- **7 Lifecycle Phases per Generation**: Worker Init, Specialization (Consumption), Metadata & Indexing, Functions Load (V1), Invocation, Worker Status, and Worker Logs (`RpcLog`). + +## Local Preview + +```bash +python3 eng/docs/extract_arch.py +python3 -m http.server 8765 --bind 127.0.0.1 --directory docs/site +``` + +Then open `http://127.0.0.1:8765/`. + +## GitHub Pages Setup + +`.github/workflows/architecture-pages.yml` builds and validates the site on pull requests and deploys to `https://azure.github.io/azure-functions-python-worker/` on `dev`. Ensure **Settings → Pages → Build and deployment → Source** is set to **GitHub Actions**. diff --git a/docs/site/assets/app.js b/docs/site/assets/app.js new file mode 100644 index 000000000..5111cd16f --- /dev/null +++ b/docs/site/assets/app.js @@ -0,0 +1,1202 @@ +/** + * Interactive DeepWiki-style Architecture Portal for Azure Functions Python Worker. + * Renders: + * - Evolution Overview (side-by-side matrix + 4-repo map) + * - 4-Repo Ecosystem Topology + 3 Era Architecture Pages (<=3.12, 3.13-3.14, 3.15+ R2P2) + * - Auto-laid-out Compound Swimlane SVG Canvas (pan/zoom, LR/TB toggle, animated packet traces) + * - Interactive Multi-Actor Sequence Diagram + * - Exportable Deterministic Mermaid Source + * - Dependency Isolation & Auto-Update CI Pipeline Inspector + */ +(function () { + const graph = window.__ARCH_GRAPH__; + if (!graph) return; + + const EDGE_PALETTE = { + "grpc-wire": { color: "#38bdf8", label: "gRPC Wire (FunctionRpc)" }, + "thread-queue": { color: "#a78bfa", label: "Thread / Async Channel" }, + "inproc-contract": { color: "#34d399", label: "In-Proc Python Contract" }, + "pyo3-hot": { color: "#fb923c", label: "PyO3 Hot Tuple FFI" }, + "pyo3-control": { color: "#f472b6", label: "PyO3 Control / Extensions" }, + "syspath-mutation": { color: "#fbbf24", label: "Import / EntryPoint" }, + "exec-dispatch": { color: "#60a5fa", label: "Handler Execution" }, + }; + + const CONTAINER_TONES = { + proto: { stroke: "#fbbf24", fill: "rgba(120, 53, 15, 0.14)", head: "rgba(146, 64, 14, 0.32)", text: "#fde68a" }, + host: { stroke: "#38bdf8", fill: "rgba(12, 74, 110, 0.14)", head: "rgba(14, 116, 144, 0.30)", text: "#bae6fd" }, + python: { stroke: "#64748b", fill: "rgba(15, 23, 42, 0.50)", head: "rgba(30, 41, 59, 0.80)", text: "#e2e8f0" }, + "rust-proc": { stroke: "#fb923c", fill: "rgba(67, 20, 7, 0.22)", head: "rgba(124, 45, 18, 0.42)", text: "#fed7aa" }, + rust: { stroke: "#fb923c", fill: "rgba(124, 45, 18, 0.16)", head: "rgba(154, 52, 18, 0.36)", text: "#ffedd5" }, + transport: { stroke: "#a78bfa", fill: "rgba(76, 29, 149, 0.15)", head: "rgba(91, 33, 182, 0.32)", text: "#ddd6fe" }, + bridge: { stroke: "#f472b6", fill: "rgba(131, 24, 67, 0.16)", head: "rgba(157, 23, 77, 0.34)", text: "#fbcfe8" }, + runtime: { stroke: "#34d399", fill: "rgba(6, 78, 59, 0.16)", head: "rgba(6, 95, 70, 0.34)", text: "#a7f3d0" }, + lib: { stroke: "#34d399", fill: "rgba(6, 78, 59, 0.12)", head: "rgba(4, 120, 87, 0.28)", text: "#6ee7b7" }, + ext: { stroke: "#c084fc", fill: "rgba(88, 28, 135, 0.16)", head: "rgba(107, 33, 168, 0.34)", text: "#e9d5ff" }, + }; + + const state = { + page: "py315_r2p2", // overview | ecosystem | py312 | py313_314 | py315_r2p2 | dep_iso | pipeline + diagramMode: "topology", // topology | sequence | compare_phase | mermaid + orientation: "LR", // LR | TB + phaseId: "init", // all | init | specialization | metadata | function_load | invocation | worker_status | worker_logs + flowId: "idle", + stepIdx: -1, // -1 = full phase view; 0..N-1 = specific step hop + selectedNodeId: null, + selectedEdgeId: null, + zoom: 0.86, + panX: 18, + panY: 16, + copied: false, + sidebarCollapsed: false, + mobileMenuOpen: false, + }; + + function syncHashToState() { + const raw = (window.location.hash || "").replace(/^#\/?/, ""); + if (!raw) return; + const [p, ph, m] = raw.split("/"); + if (p) state.page = p; + if (ph) state.phaseId = ph; + if (m) state.diagramMode = m; + } + + function writeStateToHash() { + const next = `#${state.page}/${state.phaseId}/${state.diagramMode}`; + if (window.location.hash !== next) { + history.replaceState(null, "", next); + } + } + + function esc(str) { + return String(str ?? "") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); + } + + function mdInline(str) { + return esc(str) + .replace(/\*\*([^*]+)\*\*/g, "$1") + .replace(/`([^`]+)`/g, "$1"); + } + + function currentEra() { + return graph.eras.find((e) => e.id === state.page) || null; + } + + function currentPhase(era) { + if (!era || !era.phases || !era.phases.length || state.phaseId === "all") return null; + return era.phases.find((p) => p.id === state.phaseId) || era.phases[0] || null; + } + + let needsAutoFit = true; + + function selectPage(pageId) { + state.page = pageId; + state.flowId = "idle"; + state.stepIdx = -1; + state.selectedEdgeId = null; + if (pageId === "ecosystem" && state.diagramMode === "compare_phase") { + state.diagramMode = "topology"; + } + const era = currentEra(); + const ph = currentPhase(era); + state.selectedNodeId = ph?.activeNodes?.[0] || era?.nodes?.[0]?.id || null; + needsAutoFit = true; + writeStateToHash(); + render(); + } + + function fitDiagramToStage(era) { + if (!era) return; + const ph = currentPhase(era); + const layout = window.ArchLayoutEngine.layoutCompoundGraph( + era, + state.orientation, + ph ? ph.topologyEdges : null + ); + const stageEl = document.getElementById("diagram-stage"); + const availW = Math.max(480, (stageEl ? stageEl.clientWidth : window.innerWidth - 650) - 36); + const availH = Math.max(320, (stageEl ? stageEl.clientHeight : window.innerHeight - 260) - 64); + const scale = Math.min(1.0, availW / layout.width, availH / layout.height); + state.zoom = Math.max(0.14, Number(scale.toFixed(3))); + state.panX = Math.round(((stageEl ? stageEl.clientWidth : availW) - layout.width * state.zoom) / 2); + state.panY = Math.max(14, Math.round(((stageEl ? stageEl.clientHeight : availH) - layout.height * state.zoom) / 2 - 8)); + const vp = document.getElementById("viewport-g"); + if (vp) vp.setAttribute("transform", `translate(${state.panX}, ${state.panY}) scale(${state.zoom})`); + } + + function renderTopbar() { + return ` +
+
+ +
λ
+
+ Azure Functions Python Runtime + Architecture Atlas +
+
+
+ + + + @${esc(graph.git.shortSha)} + +
+
+ `; + } + + function renderSidebar() { + const navEraItems = [ + { id: "overview", title: "Evolution Matrix", sub: "≤3.12 vs 3.13-14 vs 3.15+", tag: "Compare" }, + { id: "ecosystem", title: "4-Repo Ecosystem", sub: "Proto · Worker · SDK · Ext", tag: "All" }, + { id: "py312", title: "Python ≤ 3.12", sub: "workers/azure_functions_worker", tag: "Monolith" }, + { id: "py313_314", title: "Python 3.13 – 3.14", sub: "proxy_worker + runtimes/", tag: "Split" }, + { id: "py315_r2p2", title: "Python 3.15+ (R2P2)", sub: "workers/r2p2 + PyO3 + runtimes/", tag: "Rust" }, + ]; + const deepDiveItems = [ + { id: "dep_iso", title: "Dependency Isolation", sub: "sys.path & Namespace Eviction", tag: "Internals" }, + { id: "pipeline", title: "Commit Auto-Sync Engine", sub: "AST + Drift Diff + Pages CI", tag: "DeepWiki" }, + ]; + + return ` + + `; + } + + function renderTopologySVG(era, activePhase, activeStep) { + const layout = window.ArchLayoutEngine.layoutCompoundGraph( + era, + state.orientation, + activePhase ? activePhase.topologyEdges : null + ); + const phaseNodeSet = activePhase ? new Set(activePhase.activeNodes) : null; + const stepNodeSet = activeStep ? new Set(activeStep.activeNodes) : null; + const stepEdgeSet = activeStep ? new Set(activeStep.activeEdges) : null; + + const markers = Object.entries(EDGE_PALETTE) + .map( + ([k, v]) => ` + + + + ` + ) + .join(""); + + const containersSvg = layout.containers + .map((c) => { + const t = CONTAINER_TONES[c.tone] || CONTAINER_TONES.python; + const r = c.isNested ? 9 : 13; + return ` + + + + ${esc(c.label.toUpperCase())} + + `; + }) + .join(""); + + const edgesSvg = layout.edges + .map((e) => { + const pal = EDGE_PALETTE[e.kind] || EDGE_PALETTE["inproc-contract"]; + const isStepActive = stepEdgeSet ? stepEdgeSet.has(e.id) : false; + const isPhaseFlowing = Boolean(activePhase) && !stepEdgeSet; + const isSelected = state.selectedEdgeId === e.id; + const isDim = stepEdgeSet && !isStepActive; + const opacity = isDim ? 0.16 : 0.96; + const strokeW = isStepActive || isSelected ? 3.3 : isPhaseFlowing ? 2.3 : 1.9; + const labelW = Math.max(78, e.label.length * 6.3 + 16); + + return ` + + + + ${ + isStepActive + ? ` + + + + ` + : "" + } + + + + ${esc(e.label)} + + + + `; + }) + .join(""); + + const nodesSvg = layout.nodes + .map((p) => { + const n = p.node; + const repo = graph.repos[n.repo] || graph.repos["python-worker"]; + const isBypassedInPhase = phaseNodeSet ? !phaseNodeSet.has(n.id) : false; + const isStepActive = stepNodeSet ? stepNodeSet.has(n.id) : false; + const isDim = isBypassedInPhase || (stepNodeSet && !isStepActive); + const isSel = state.selectedNodeId === n.id; + const borderCol = isStepActive ? "#fb923c" : isSel ? "#38bdf8" : !isBypassedInPhase && phaseNodeSet ? "#3b527e" : "#24324d"; + const borderW = isStepActive || isSel ? 2.4 : 1.4; + const shortFile = n.file.split("/").slice(-2).join("/"); + const lineTag = n.line > 1 ? `:${n.line}` : ""; + + return ` + + + + + + + ${esc(repo.shortName)} + + ${esc(isBypassedInPhase ? "BYPASSED" : n.badge.slice(0, 22))} + + + + ${esc(n.label.length > 31 ? n.label.slice(0, 30) + "…" : n.label)} + + + + + ${esc((shortFile + lineTag).slice(0, 33))} + + ${ + n.extractedSymbols && n.extractedSymbols.length + ? `${n.extractedSymbols.length} AST` + : "" + } + + `; + }) + .join(""); + + return ` +
+ + ${markers} + + ${containersSvg} + ${edgesSvg} + ${nodesSvg} + + + +
+ + + + +
+ +
+ ${Object.values(EDGE_PALETTE) + .slice(0, 5) + .map( + (p) => ` + + + ${esc(p.label)} + + ` + ) + .join("")} +
+
+ `; + } + + function renderSequenceSVG(flow) { + if (!flow) return `
Select a lifecycle flow above.
`; + const colW = 188; + const padX = 115; + const startY = 96; + const rowH = 68; + const width = Math.max(760, padX * 2 + (flow.actors.length - 1) * colW); + const height = startY + flow.steps.length * rowH + 46; + const actorX = new Map(flow.actors.map((a, i) => [a.id, padX + i * colW])); + + return ` +
+
+
+ ${esc(flow.title)} +
${esc(flow.summary)}
+
+ Click any message row to synchronize active path +
+
+ + + + + + + + + + + + + ${flow.actors + .map((a) => { + const x = actorX.get(a.id); + return ` + + + ${esc(a.label)} + `; + }) + .join("")} + ${flow.steps + .map((st, idx) => { + const x1 = actorX.get(st.from); + const x2 = actorX.get(st.to); + const y = startY + idx * rowH; + const isAct = state.stepIdx === idx; + const stroke = isAct ? "#fb923c" : st.isReturn ? "#34d399" : "#38bdf8"; + const mk = isAct ? "url(#sq-act)" : st.isReturn ? "url(#sq-ret)" : "url(#sq-req)"; + const mid = x1 === x2 ? x1 + 60 : (x1 + x2) / 2; + return ` + + + + ${idx + 1} + + ${esc(st.message)} + ${esc(st.payload)} + + `; + }) + .join("")} + +
+
+ `; + } + + function renderMermaidTab(era, activePhase) { + const topoMmd = activePhase ? activePhase.topologyMermaid : era.topologyMermaid; + const seqTarget = activePhase || era.flows[0]; + return ` +
+
+
+ Auto-Compiled Mermaid Source — ${esc(activePhase ? activePhase.title : era.title)} +
Deterministically compiled from architecture-graph.json for PRs & design docs.
+
+ +
+
${esc(topoMmd)}
+ ${ + seqTarget + ? ` +
Sequence Diagram — ${esc(seqTarget.title)}
+
${esc(seqTarget.sequenceMermaid)}
+ ` + : "" + } +
+ `; + } + + function renderComparePhaseTab(phaseId) { + const targetId = phaseId === "all" ? "init" : phaseId; + const cat = (graph.phaseCatalog || []).find((p) => p.id === targetId) || { short: targetId, proto: "" }; + const workerEras = graph.eras.filter((e) => e.phases && e.phases.length); + + return ` +
+
+ ${esc(cat.proto)} +

Cross-Era Comparison — ${esc(cat.short)}

+
How Python ≤ 3.12, 3.13–3.14, and 3.15+ (R2P2) each handle this exact lifecycle phase.
+
+
+ ${workerEras + .map((e) => { + const ph = e.phases.find((p) => p.id === targetId); + if (!ph) return ""; + return ` +
+
+ ${esc(e.shortTitle)} + +
+
${esc(ph.proto)}
+

${mdInline(ph.summary)}

+
⚡ ${mdInline(ph.bypassedNote)}
+
Sequence Hops (${ph.steps.length})
+
    + ${ph.steps + .map( + (s) => `
  1. ${esc(s.message)}
    ${esc(s.payload)}
  2. ` + ) + .join("")} +
+
+ `; + }) + .join("")} +
+
+ `; + } + + function renderInspector(era, activePhase) { + const edgePool = activePhase ? activePhase.topologyEdges : era.edges; + const selEdge = state.selectedEdgeId ? edgePool.find((e) => e.id === state.selectedEdgeId) : null; + if (selEdge) { + const srcNode = era.nodes.find((n) => n.id === selEdge.source); + const tgtNode = era.nodes.find((n) => n.id === selEdge.target); + const pal = EDGE_PALETTE[selEdge.kind] || EDGE_PALETTE["inproc-contract"]; + return ` + + `; + } + + const node = era.nodes.find((n) => n.id === state.selectedNodeId) || era.nodes[0]; + if (!node) return ``; + const repo = graph.repos[node.repo] || graph.repos["python-worker"]; + const inbound = edgePool.filter((e) => e.target === node.id); + const outbound = edgePool.filter((e) => e.source === node.id); + const isBypassed = activePhase ? !activePhase.activeNodes.includes(node.id) : false; + + return ` + + `; + } + + function renderEraWorkspace(era) { + const hasPhases = Boolean(era.phases && era.phases.length); + const activePhase = currentPhase(era); + const seqSource = activePhase || era.flows[0]; + const stepList = activePhase ? activePhase.steps : []; + const activeStep = + activePhase && state.stepIdx >= 0 && state.stepIdx < stepList.length + ? stepList[state.stepIdx] + : null; + + return ` +
+
+
+
+

${esc(era.title)}

+ ${esc(era.pythonVersions)} +
+ ${ + !activePhase + ? `

${mdInline(era.summary)}

` + : "" + } +
+
+ + + ${ + hasPhases + ? `` + : "" + } + +
+
+ + ${ + hasPhases + ? ` +
+ Lifecycle Phase: + + ${(graph.phaseCatalog || []) + .map( + (p) => ` + + ` + ) + .join("")} +
+ ` + : "" + } + + ${ + activePhase + ? ` +
+
+
+ ${esc(activePhase.title)} + ${esc(activePhase.proto)} + ${esc(activePhase.scope)} +
+
${mdInline(activePhase.summary)}
+
⚡ ${mdInline(activePhase.bypassedNote)}
+
+ +
+ ${ + state.stepIdx >= 0 + ? ` + + Hop ${state.stepIdx + 1} / ${stepList.length} + + + ` + : ` + + ` + } +
+
+ ${ + activeStep + ? ` +
+
+ Hop ${state.stepIdx + 1}: ${esc(activeStep.message)} + [${esc(activeStep.payload)}] +
+
+ ` + : "" + } + ` + : "" + } + + ${ + state.diagramMode === "topology" + ? `
+ ${renderTopologySVG(era, activePhase, activeStep)} + ${renderInspector(era, activePhase)} +
` + : state.diagramMode === "sequence" + ? renderSequenceSVG(seqSource) + : state.diagramMode === "compare_phase" + ? renderComparePhaseTab(state.phaseId) + : renderMermaidTab(era, activePhase) + } +
+ `; + } + + function renderOverviewPage() { + const threeEras = graph.eras.filter((e) => e.id !== "ecosystem"); + return ` +
+
+ Architecture Evolution · Python 3.7 → 3.15+ +

Azure Functions Python Runtime — Three-Generation Evolution

+

+ While preserving the exact same FunctionRpc.proto Host contract and customer + azure-functions programming models (V1 function.json & V2 @app decorators), + the worker evolved from a single CPython monolith (≤3.12), to a decoupled Python Proxy + PyPI Runtime split (3.13–3.14), + to R2P2 — a native Rust tonic/prost shell embedding CPython via PyO3 (3.15+). +

+ +
+ ${threeEras + .map( + (e) => ` +
+
+ ${esc(e.shortTitle)} + ${esc(e.pythonVersions)} +
+
${esc(e.packagePath)}
+

${esc(e.tagline)}

+
    + ${e.tradeoffs + .map( + (t) => + `
  • ${ + t.kind === "strength" ? "✓" : "⚠" + } ${esc(t.text)}
  • ` + ) + .join("")} +
+ +
+ ` + ) + .join("")} +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Architectural DimensionPython ≤ 3.12 (Monolith)Python 3.13 – 3.14 (Proxy Split)Python 3.15+ (R2P2 Rust Proxy)
Launcher & ProcessprodV4/worker.py → CPythonproxyV4/worker.py → CPythonCompiled r2p2 Rust binary + embedded libpython3.15 (GIL)
gRPC & Wire CodecPython grpcio + google.protobufPython grpcio + worker protobuf + uvloopNative Rust tonic + prost + tokio (0 Python protobuf)
Proxy ↔ Runtime SeamNone (internal module imports)In-proc WorkerRequest + injected protosPyO3 FFI: invoke_native Datum tuples (hot) + protos_adapter dicts (control)
Day-1 New Python SupportBlocked on grpcio/protobuf wheelsBlocked on grpcio/protobuf wheelsUnblocked — Rust owns wire; runtime is pure Python
WorkerStatus Scale ProbeHandled inside Python DispatcherFast-path in proxy_workerZero-GIL pure Rust reply in bridge.rs
Measured GIL ThroughputBaseline (~339 req/s, p50 145ms)Parity (~339 req/s; PyPI agility win)+122% (754 req/s, p50 64ms)
+
+
+ `; + } + + function renderDepIsoPage() { + return ` +
+
+ Runtime Internals · Dependency Isolation +

Dependency Isolation & Namespace-Safe Promotion Across Eras

+

+ In Flex Consumption and Linux Consumption, warm placeholder containers boot before customer packages are mounted, + then specialize on FunctionEnvironmentReloadRequest (or WorkerInitRequest). +

+ +
+
+ Python ≤ 3.14 (DependencyManager) +

+ proxy_worker/utils/dependency.py boots with use_worker_dependencies() so worker + grpcio and google.protobuf C-extensions load from the worker directory, then switches + sys.path to [cx_deps, cx_working_dir, worker_deps]. +

+ Historical Pain (D-003..D-005): Because google.protobuf uses a process-global C descriptor pool, + once loaded at boot it could still collide with customer-pinned protobuf versions. +

+
+
+ Python 3.15+ R2P2 (bridge.py + Zero Protobuf) +

+ Because Rust (tonic/prost) owns the wire and protos_adapter.py is pure Python, + neither grpcio nor google.protobuf ever enters sys.modules! +

+ On specialization, _clear_modules_from_path() (workers/r2p2/bridge/bridge.py:L328) + evicts file-backed worker packages & cached descendants and detaches stale child attributes from retained parents, + while preserving shared PEP 420 namespace parents (azure, azurefunctions) so customer extensions stay attached. +

+
+
+ +

sys.path Priority Order After Specialization

+
1. Customer Packages (.python_packages/lib/site-packages)   # Customer-pinned azure-functions / extensions win!
+2. Worker Bundled Runtime Dependencies (workers_dir)        # Fallback azure_functions_runtime (v2 / v1)
+3. Customer Function App Directory (function_app_directory) # function_app.py & user modules
+   [Protected in R2P2: bridge, protos_adapter, runtime prefixes]
+
+
+ `; + } + + function renderPipelinePage() { + const invRows = Object.entries(graph.packageInventories) + .map( + ([pkg, files]) => ` + + ${esc(pkg)} + ${files.length} source files + ${files.reduce((s, f) => s + f.lines, 0).toLocaleString()} LOC + ${esc(files[0]?.path || "")} … + + ` + ) + .join(""); + + return ` +
+
+ DeepWiki Auto-Sync Pipeline · Commit-Driven +

How Diagrams & Pages Stay Live on Every Commit

+

+ Rather than brittle hand-positioned boxes or unconstrained LLM prose, the documentation pipeline separates + deterministic code-graph extraction + algorithmic compound layout (every commit) from + drift-gated AI narrative synthesis (only when structural contracts change). +

+ +
+
+ 1. Deterministic AST + Rust + Proto Extractor +
eng/docs/extract_arch.py · < 0.5s
+

+ Parses Python ast, Rust workers/r2p2/src/*.rs, and FunctionRpc.proto, + and links sibling contracts in python-library, python-extensions, and language-worker-protobuf. + Computes SHA-256 structuralFingerprint (currently ${esc(graph.structuralFingerprint)}). +

+
+
+ 2. Algorithmic Compound Diagram Engine +
site/assets/layoutEngine.js
+

+ Computes topological layer ranks, nested container swimlanes, port ordering, and Bezier/orthogonal routes + automatically from JSON — plus deterministic Mermaid compilation. Adding a module or edge never breaks layout. +

+
+
+ 3. Drift-Gated AI Refresher + Pages Deploy +
.github/workflows/architecture-pages.yml
+

+ Every push to dev deploys live AST line anchors & inventories to GitHub Pages. + When structuralFingerprint changes (new router case, FFI signature, or module), GitHub Models + (models: read) updates affected explanations with citation validation. +

+
+
+ +

Live Scanned Package Inventory (Commit ${esc(graph.git.shortSha)})

+ + + + + ${invRows} +
SubsystemTracked FilesTotal LinesSample Entry
+
+
+ `; + } + + function render() { + const root = document.getElementById("root"); + const era = currentEra(); + let mainHtml = ""; + if (state.page === "overview") mainHtml = renderOverviewPage(); + else if (state.page === "dep_iso") mainHtml = renderDepIsoPage(); + else if (state.page === "pipeline") mainHtml = renderPipelinePage(); + else if (era) mainHtml = renderEraWorkspace(era); + + const shellCls = [ + "app-shell", + state.sidebarCollapsed ? "sidebar-collapsed" : "", + state.mobileMenuOpen ? "sidebar-open-mobile" : "", + ] + .filter(Boolean) + .join(" "); + + root.innerHTML = ` +
+ ${renderTopbar()} + ${renderSidebar()} + ${mainHtml} +
+ `; + if (needsAutoFit && era && state.diagramMode === "topology") { + needsAutoFit = false; + fitDiagramToStage(era); + } + bindEvents(); + } + + function bindEvents() { + const toggleBtn = document.getElementById("btn-toggle-nav"); + if (toggleBtn) { + toggleBtn.addEventListener("click", () => { + if (window.innerWidth <= 1120) { + state.mobileMenuOpen = !state.mobileMenuOpen; + } else { + state.sidebarCollapsed = !state.sidebarCollapsed; + } + needsAutoFit = true; + render(); + }); + } + + const quickSel = document.getElementById("quick-era-select"); + if (quickSel) { + quickSel.addEventListener("change", (ev) => { + state.mobileMenuOpen = false; + selectPage(ev.target.value); + }); + } + + document.querySelectorAll("[data-nav]").forEach((el) => { + el.addEventListener("click", () => { + state.mobileMenuOpen = false; + selectPage(el.getAttribute("data-nav")); + }); + }); + + document.querySelectorAll("[data-mode]").forEach((el) => { + el.addEventListener("click", () => { + state.diagramMode = el.getAttribute("data-mode"); + if (state.diagramMode === "topology") needsAutoFit = true; + writeStateToHash(); + render(); + }); + }); + + document.querySelectorAll("[data-phase]").forEach((el) => { + el.addEventListener("click", () => { + state.phaseId = el.getAttribute("data-phase"); + state.stepIdx = -1; + state.selectedEdgeId = null; + const era = currentEra(); + const ph = currentPhase(era); + if (ph && ph.activeNodes?.length) { + state.selectedNodeId = ph.activeNodes[0]; + } + needsAutoFit = true; + writeStateToHash(); + render(); + }); + }); + + document.querySelectorAll("[data-jump-era]").forEach((el) => { + el.addEventListener("click", () => { + state.page = el.getAttribute("data-jump-era"); + state.phaseId = el.getAttribute("data-jump-phase") || state.phaseId; + state.diagramMode = "topology"; + state.stepIdx = -1; + needsAutoFit = true; + writeStateToHash(); + render(); + }); + }); + + const btnStart = document.getElementById("btn-start-trace"); + if (btnStart) { + btnStart.addEventListener("click", () => { + state.stepIdx = 0; + render(); + }); + } + + const prev = document.getElementById("step-prev"); + if (prev) prev.addEventListener("click", () => { state.stepIdx = Math.max(0, state.stepIdx - 1); render(); }); + const next = document.getElementById("step-next"); + if (next) next.addEventListener("click", () => { state.stepIdx += 1; render(); }); + const reset = document.getElementById("step-reset"); + if (reset) reset.addEventListener("click", () => { state.stepIdx = -1; render(); }); + + document.querySelectorAll("[data-node]").forEach((el) => { + el.addEventListener("click", (ev) => { + ev.stopPropagation(); + state.selectedNodeId = el.getAttribute("data-node"); + state.selectedEdgeId = null; + render(); + }); + }); + + document.querySelectorAll("[data-edge]").forEach((el) => { + el.addEventListener("click", (ev) => { + ev.stopPropagation(); + state.selectedEdgeId = el.getAttribute("data-edge"); + render(); + }); + }); + + document.querySelectorAll("[data-seq-step]").forEach((el) => { + el.addEventListener("click", () => { + state.stepIdx = Number(el.getAttribute("data-seq-step")); + render(); + }); + }); + + const zin = document.getElementById("btn-zoom-in"); + if (zin) zin.addEventListener("click", () => { state.zoom = Math.min(1.6, +(state.zoom + 0.12).toFixed(2)); render(); }); + const zout = document.getElementById("btn-zoom-out"); + if (zout) zout.addEventListener("click", () => { state.zoom = Math.max(0.35, +(state.zoom - 0.12).toFixed(2)); render(); }); + const zfit = document.getElementById("btn-zoom-fit"); + if (zfit) zfit.addEventListener("click", () => { fitDiagramToStage(currentEra()); render(); }); + const bOrient = document.getElementById("btn-orient"); + if (bOrient) { + bOrient.addEventListener("click", () => { + state.orientation = state.orientation === "LR" ? "TB" : "LR"; + fitDiagramToStage(currentEra()); + render(); + }); + } + + const copyBtn = document.getElementById("btn-copy-mermaid"); + if (copyBtn) { + copyBtn.addEventListener("click", () => { + const era = currentEra(); + const ph = currentPhase(era); + const mmd = ph ? ph.topologyMermaid : era?.topologyMermaid; + if (mmd && navigator.clipboard) { + navigator.clipboard.writeText(mmd); + state.copied = true; + render(); + setTimeout(() => { state.copied = false; render(); }, 1500); + } + }); + } + + // Drag-to-pan + wheel-to-zoom on topology stage + const stage = document.getElementById("diagram-stage"); + const vp = document.getElementById("viewport-g"); + if (stage && vp) { + let dragging = false; + let sx = 0, sy = 0, ox = state.panX, oy = state.panY; + stage.addEventListener("mousedown", (ev) => { + if (ev.target.closest("button") || ev.target.closest(".svg-node") || ev.target.closest(".svg-edge")) return; + dragging = true; + sx = ev.clientX; + sy = ev.clientY; + ox = state.panX; + oy = state.panY; + }); + window.addEventListener("mousemove", (ev) => { + if (!dragging) return; + state.panX = ox + (ev.clientX - sx); + state.panY = oy + (ev.clientY - sy); + vp.setAttribute("transform", `translate(${state.panX}, ${state.panY}) scale(${state.zoom})`); + }); + window.addEventListener("mouseup", () => { dragging = false; }); + stage.addEventListener( + "wheel", + (ev) => { + ev.preventDefault(); + const factor = ev.deltaY < 0 ? 1.07 : 0.93; + state.zoom = Math.max(0.32, Math.min(1.8, +(state.zoom * factor).toFixed(3))); + vp.setAttribute("transform", `translate(${state.panX}, ${state.panY}) scale(${state.zoom})`); + }, + { passive: false } + ); + } + } + + syncHashToState(); + window.addEventListener("hashchange", () => { + syncHashToState(); + needsAutoFit = true; + render(); + }); + const initEra = currentEra(); + const initPhase = currentPhase(initEra); + state.selectedNodeId = initPhase?.activeNodes?.[0] || initEra?.nodes?.[0]?.id || null; + writeStateToHash(); + render(); +})(); diff --git a/docs/site/assets/layoutEngine.js b/docs/site/assets/layoutEngine.js new file mode 100644 index 000000000..fa19b7e95 --- /dev/null +++ b/docs/site/assets/layoutEngine.js @@ -0,0 +1,395 @@ +/** + * Deterministic Compound Sugiyama / Swimlane Architecture Layout Engine. + * + * Takes an `era` semantic graph (`containers`, `nodes`, `edges`) and computes: + * 1. Topological rank per leaf node from directed edges (cycle-safe longest-path). + * 2. Compound container columns/rows (root process/repo -> sub-layer -> component). + * 3. Barycenter ordering to minimize edge crossings. + * 4. Dynamic container bounding boxes with zero hardcoded coordinates. + * 5. Multi-port orthogonal rounded SVG edge paths (`M ... L ... Q ...`) + non-overlapping label positions. + */ +(function () { + const NODE_W = 252; + const NODE_H = 92; + + function computeRanks(nodes, edges) { + const ids = nodes.map((n) => n.id); + const incoming = new Map(ids.map((id) => [id, []])); + for (const e of edges) { + if (incoming.has(e.target) && e.source !== e.target) { + incoming.get(e.target).push(e.source); + } + } + const memo = new Map(); + const visiting = new Set(); + + function rankOf(id) { + if (memo.has(id)) return memo.get(id); + if (visiting.has(id)) return 0; + visiting.add(id); + let r = 0; + for (const pred of incoming.get(id) || []) { + r = Math.max(r, rankOf(pred) + 1); + } + visiting.delete(id); + memo.set(id, r); + return r; + } + + for (const id of ids) rankOf(id); + return memo; + } + + function layoutCompoundGraph(era, orientation, edgesOverride) { + const isLR = orientation !== "TB"; + const activeEdgeList = edgesOverride || era.edges; + const containersById = new Map(era.containers.map((c) => [c.id, c])); + const childContainers = new Map(); + for (const c of era.containers) { + const pid = c.parentId || "__root__"; + if (!childContainers.has(pid)) childContainers.set(pid, []); + childContainers.get(pid).push(c); + } + + const leavesByContainer = new Map(); + for (const n of era.nodes) { + if (!leavesByContainer.has(n.containerId)) leavesByContainer.set(n.containerId, []); + leavesByContainer.get(n.containerId).push(n); + } + + const ranks = computeRanks(era.nodes, era.edges); + const nodePositions = new Map(); + const containerBoxes = []; + + // Order root containers by average topological rank of their descendant nodes + function descendantLeaves(cid) { + const own = [...(leavesByContainer.get(cid) || [])]; + for (const sub of childContainers.get(cid) || []) { + own.push(...descendantLeaves(sub.id)); + } + return own; + } + + const rootContainers = [...(childContainers.get("__root__") || [])].sort((a, b) => { + const la = descendantLeaves(a.id); + const lb = descendantLeaves(b.id); + const avgA = la.length ? la.reduce((s, n) => s + (ranks.get(n.id) || 0), 0) / la.length : 0; + const avgB = lb.length ? lb.reduce((s, n) => s + (ranks.get(n.id) || 0), 0) / lb.length : 0; + return avgA - avgB; + }); + + const PAD_TOP = 44; + const PAD_SIDE = 22; + const PAD_BOTTOM = 22; + const COL_GAP = isLR ? 132 : 44; + const ROW_GAP = isLR ? 34 : 84; + const SUB_GAP = isLR ? 108 : 64; + const ROOT_GAP = isLR ? 138 : 92; + + // Layout leaf nodes inside a leaf-bearing container as a mini layered grid + function layoutLeafGroup(leaves, originX, originY) { + if (!leaves.length) return { w: 180, h: 90 }; + const uniqueRanks = [...new Set(leaves.map((n) => ranks.get(n.id) || 0))].sort((a, b) => a - b); + const rankIndex = new Map(uniqueRanks.map((r, i) => [r, i])); + + const buckets = new Map(); + for (const n of leaves) { + const ri = rankIndex.get(ranks.get(n.id) || 0) || 0; + if (!buckets.has(ri)) buckets.set(ri, []); + buckets.get(ri).push(n); + } + + let maxCrossCount = 1; + for (const arr of buckets.values()) { + maxCrossCount = Math.max(maxCrossCount, arr.length); + } + + const numLayers = uniqueRanks.length; + const totalPrimary = isLR + ? numLayers * NODE_W + (numLayers - 1) * COL_GAP + : numLayers * NODE_H + (numLayers - 1) * ROW_GAP; + const totalCross = isLR + ? maxCrossCount * NODE_H + (maxCrossCount - 1) * ROW_GAP + : maxCrossCount * NODE_W + (maxCrossCount - 1) * COL_GAP; + + for (const [layerIdx, arr] of buckets.entries()) { + const spanCross = isLR + ? arr.length * NODE_H + (arr.length - 1) * ROW_GAP + : arr.length * NODE_W + (arr.length - 1) * COL_GAP; + const crossOffset = (totalCross - spanCross) / 2; + + arr.forEach((node, idx) => { + const x = isLR + ? originX + layerIdx * (NODE_W + COL_GAP) + : originX + crossOffset + idx * (NODE_W + COL_GAP); + const y = isLR + ? originY + crossOffset + idx * (NODE_H + ROW_GAP) + : originY + layerIdx * (NODE_H + ROW_GAP); + nodePositions.set(node.id, { x, y, w: NODE_W, h: NODE_H, node }); + }); + } + + return { + w: isLR ? totalPrimary : totalCross, + h: isLR ? totalCross : totalPrimary, + }; + } + + let cursorPrimary = 36; + const rootMetrics = []; + + // First pass: measure each root container + for (const rc of rootContainers) { + const subs = [...(childContainers.get(rc.id) || [])].sort((a, b) => { + const la = descendantLeaves(a.id); + const lb = descendantLeaves(b.id); + const avgA = la.length ? la.reduce((s, n) => s + (ranks.get(n.id) || 0), 0) / la.length : 0; + const avgB = lb.length ? lb.reduce((s, n) => s + (ranks.get(n.id) || 0), 0) / lb.length : 0; + return avgA - avgB; + }); + rootMetrics.push({ rc, subs }); + } + + // Second pass: place containers and compute bounding boxes + const placedRoots = []; + for (const { rc, subs } of rootMetrics) { + if (subs.length === 0) { + const leaves = leavesByContainer.get(rc.id) || []; + const startX = isLR ? cursorPrimary + PAD_SIDE : 36 + PAD_SIDE; + const startY = isLR ? 36 + PAD_TOP : cursorPrimary + PAD_TOP; + const inner = layoutLeafGroup(leaves, startX, startY); + const boxW = inner.w + PAD_SIDE * 2; + const boxH = inner.h + PAD_TOP + PAD_BOTTOM; + const box = { + id: rc.id, + label: rc.label, + tone: rc.tone, + isNested: false, + x: isLR ? cursorPrimary : 36, + y: isLR ? 36 : cursorPrimary, + w: boxW, + h: boxH, + leafIds: leaves.map((l) => l.id), + subIds: [], + }; + placedRoots.push(box); + cursorPrimary += (isLR ? boxW : boxH) + ROOT_GAP; + } else { + let subCursor = (isLR ? cursorPrimary : 36) + PAD_SIDE; + if (!isLR) subCursor = cursorPrimary + PAD_TOP; + + const subBoxes = []; + let maxSubCross = 0; + + for (const sub of subs) { + const leaves = leavesByContainer.get(sub.id) || []; + const sx = isLR ? subCursor + PAD_SIDE : 36 + PAD_SIDE * 2; + const sy = isLR ? 36 + PAD_TOP * 2 : subCursor + PAD_TOP; + const inner = layoutLeafGroup(leaves, sx, sy); + const sw = inner.w + PAD_SIDE * 2; + const sh = inner.h + PAD_TOP + PAD_BOTTOM; + const sBox = { + id: sub.id, + label: sub.label, + tone: sub.tone, + isNested: true, + x: isLR ? subCursor : 36 + PAD_SIDE, + y: isLR ? 36 + PAD_TOP : subCursor, + w: sw, + h: sh, + leafIds: leaves.map((l) => l.id), + }; + subBoxes.push(sBox); + maxSubCross = Math.max(maxSubCross, isLR ? sh : sw); + subCursor += (isLR ? sw : sh) + SUB_GAP; + } + + // Vertically/horizontally center sibling sub-containers inside their parent root + for (const sBox of subBoxes) { + const delta = isLR ? (maxSubCross - sBox.h) / 2 : (maxSubCross - sBox.w) / 2; + if (delta > 0) { + if (isLR) sBox.y += delta; + else sBox.x += delta; + for (const lid of sBox.leafIds) { + const np = nodePositions.get(lid); + if (np) { + if (isLR) np.y += delta; + else np.x += delta; + } + } + } + containerBoxes.push(sBox); + } + + const totalSubSpan = + subBoxes.reduce((acc, b) => acc + (isLR ? b.w : b.h), 0) + + Math.max(0, subBoxes.length - 1) * SUB_GAP; + const boxW = isLR ? totalSubSpan + PAD_SIDE * 2 : maxSubCross + PAD_SIDE * 2; + const boxH = isLR ? maxSubCross + PAD_TOP + PAD_BOTTOM : totalSubSpan + PAD_TOP + PAD_BOTTOM; + + const rBox = { + id: rc.id, + label: rc.label, + tone: rc.tone, + isNested: false, + x: isLR ? cursorPrimary : 36, + y: isLR ? 36 : cursorPrimary, + w: boxW, + h: boxH, + leafIds: descendantLeaves(rc.id).map((l) => l.id), + subIds: subBoxes.map((b) => b.id), + }; + placedRoots.push(rBox); + cursorPrimary += (isLR ? boxW : boxH) + ROOT_GAP; + } + } + + // Center all root containers along the cross axis so the whole diagram is balanced + const maxRootCross = Math.max(...placedRoots.map((r) => (isLR ? r.h : r.w)), 240); + const subById = new Map(containerBoxes.map((b) => [b.id, b])); + for (const rBox of placedRoots) { + const delta = isLR ? (maxRootCross - rBox.h) / 2 : (maxRootCross - rBox.w) / 2; + if (delta > 0) { + if (isLR) rBox.y += delta; + else rBox.x += delta; + for (const sid of rBox.subIds) { + const sb = subById.get(sid); + if (sb) { + if (isLR) sb.y += delta; + else sb.x += delta; + } + } + for (const lid of rBox.leafIds) { + const np = nodePositions.get(lid); + if (np) { + if (isLR) np.y += delta; + else np.x += delta; + } + } + } + } + + const allContainers = [...placedRoots, ...containerBoxes]; + + // Port distribution + bidirectional smooth cubic routing for activeEdgeList + const outPorts = new Map(); + const inPorts = new Map(); + for (const e of activeEdgeList) { + if (!outPorts.has(e.source)) outPorts.set(e.source, []); + outPorts.get(e.source).push(e.id); + if (!inPorts.has(e.target)) inPorts.set(e.target, []); + inPorts.get(e.target).push(e.id); + } + + for (const [, eids] of outPorts.entries()) { + eids.sort((a, b) => { + const ea = activeEdgeList.find((x) => x.id === a); + const eb = activeEdgeList.find((x) => x.id === b); + const pa = nodePositions.get(ea.target); + const pb = nodePositions.get(eb.target); + return isLR ? (pa?.y || 0) - (pb?.y || 0) : (pa?.x || 0) - (pb?.x || 0); + }); + } + for (const [, eids] of inPorts.entries()) { + eids.sort((a, b) => { + const ea = activeEdgeList.find((x) => x.id === a); + const eb = activeEdgeList.find((x) => x.id === b); + const pa = nodePositions.get(ea.source); + const pb = nodePositions.get(eb.source); + return isLR ? (pa?.y || 0) - (pb?.y || 0) : (pa?.x || 0) - (pb?.x || 0); + }); + } + + const pairKeys = new Set(activeEdgeList.map((e) => `${e.source}->${e.target}`)); + + const routedEdges = activeEdgeList.map((e, idx) => { + const s = nodePositions.get(e.source); + const t = nodePositions.get(e.target); + if (!s || !t) return null; + + const hasReverse = pairKeys.has(`${e.target}->${e.source}`); + const sList = outPorts.get(e.source) || [e.id]; + const tList = inPorts.get(e.target) || [e.id]; + const sSlot = (sList.indexOf(e.id) + 1) / (sList.length + 1); + const tSlot = (tList.indexOf(e.id) + 1) / (tList.length + 1); + + let x1, y1, x2, y2, path, labelX, labelY; + if (isLR) { + const sameColumn = Math.abs(s.x - t.x) < 20; + if (sameColumn) { + const xOff = hasReverse ? (s.y < t.y ? -22 : 22) : 0; + x1 = s.x + s.w * 0.5 + xOff; + y1 = s.y < t.y ? s.y + s.h : s.y; + x2 = t.x + t.w * 0.5 + xOff; + y2 = s.y < t.y ? t.y : t.y + t.h; + path = `M ${x1} ${y1} L ${x2} ${y2}`; + labelX = x1; + labelY = (y1 + y2) / 2; + } else if (s.x < t.x) { + // Forward Left -> Right + x1 = s.x + s.w; + x2 = t.x; + const laneShift = hasReverse ? -18 : 0; + y1 = s.y + 18 + sSlot * (s.h - 36) + laneShift; + y2 = t.y + 18 + tSlot * (t.h - 36) + laneShift; + const dx = Math.max(48, Math.abs(x2 - x1) * 0.45); + const bow = hasReverse ? -24 : 0; + path = `M ${x1} ${y1} C ${x1 + dx} ${y1 + bow}, ${x2 - dx} ${y2 + bow}, ${x2} ${y2}`; + const tParam = 0.48 + ((idx % 3) - 1) * 0.06; + labelX = x1 + (x2 - x1) * tParam; + labelY = y1 + (y2 - y1) * tParam + bow * 0.72; + } else { + // Reverse Right -> Left (e.g. RpcLog Worker -> Host or Response bounce-back) + x1 = s.x; + x2 = t.x + t.w; + const laneShift = hasReverse ? 18 : 0; + y1 = s.y + 18 + sSlot * (s.h - 36) + laneShift; + y2 = t.y + 18 + tSlot * (t.h - 36) + laneShift; + const dx = Math.max(48, Math.abs(x1 - x2) * 0.45); + const bow = hasReverse ? 26 : 0; + path = `M ${x1} ${y1} C ${x1 - dx} ${y1 + bow}, ${x2 + dx} ${y2 + bow}, ${x2} ${y2}`; + const tParam = 0.5 + ((idx % 3) - 1) * 0.06; + labelX = x1 + (x2 - x1) * tParam; + labelY = y1 + (y2 - y1) * tParam + bow * 0.72; + } + } else { + const forward = s.y <= t.y; + const laneShift = hasReverse ? (forward ? -28 : 28) : 0; + x1 = s.x + 24 + sSlot * (s.w - 48) + laneShift; + y1 = forward ? s.y + s.h : s.y; + x2 = t.x + 24 + tSlot * (t.w - 48) + laneShift; + y2 = forward ? t.y : t.y + t.h; + const dy = Math.max(42, Math.abs(y2 - y1) * 0.45); + path = forward + ? `M ${x1} ${y1} C ${x1} ${y1 + dy}, ${x2} ${y2 - dy}, ${x2} ${y2}` + : `M ${x1} ${y1} C ${x1} ${y1 - dy}, ${x2} ${y2 + dy}, ${x2} ${y2}`; + labelX = (x1 + x2) / 2; + labelY = (y1 + y2) / 2; + } + + return { + ...e, + x1, + y1, + x2, + y2, + path, + labelX, + labelY, + }; + }).filter(Boolean); + + const totalW = Math.max(...allContainers.map((c) => c.x + c.w), 800) + 48; + const totalH = Math.max(...allContainers.map((c) => c.y + c.h), 480) + 48; + + return { + width: totalW, + height: totalH, + containers: allContainers, + nodes: [...nodePositions.values()], + edges: routedEdges, + }; + } + + window.ArchLayoutEngine = { layoutCompoundGraph }; +})(); diff --git a/docs/site/assets/styles.css b/docs/site/assets/styles.css new file mode 100644 index 000000000..6030d81bb --- /dev/null +++ b/docs/site/assets/styles.css @@ -0,0 +1,757 @@ +:root { + --bg: #070b14; + --bg-elev: #0d1424; + --bg-card: #111b2e; + --bg-hover: #17243d; + --border: #1e2d4a; + --border-strong: #314870; + --text: #f1f5f9; + --text-muted: #94a3b8; + --text-dim: #64748b; + --accent: #38bdf8; + --rust: #fb923c; + --green: #34d399; + --purple: #c084fc; + --amber: #fbbf24; + --pink: #f472b6; + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; + --sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Inter, Roboto, sans-serif; +} + +* { + box-sizing: border-box; +} + +html, +body { + margin: 0; + padding: 0; + background: var(--bg); + color: var(--text); + font-family: var(--sans); + height: 100%; + overflow: hidden; +} + +.app-shell { + display: grid; + grid-template-columns: 248px 1fr; + grid-template-rows: auto 1fr; + height: 100vh; +} + +.app-shell.sidebar-collapsed { + grid-template-columns: 0px 1fr; +} + +.app-shell.sidebar-collapsed .sidebar { + display: none; +} + +/* Topbar */ +.topbar { + grid-column: 1 / -1; + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 14px; + background: rgba(13, 20, 36, 0.96); + border-bottom: 1px solid var(--border); + gap: 10px; + flex-wrap: wrap; +} + +.brand { + display: flex; + align-items: center; + gap: 10px; +} + +.brand-mark { + width: 28px; + height: 28px; + border-radius: 7px; + background: linear-gradient(135deg, #0ea5e9, #fb923c); + display: grid; + place-items: center; + font-weight: 800; + font-size: 13px; + color: #070b14; +} + +.brand-title { + font-size: 14px; + font-weight: 700; + letter-spacing: -0.01em; +} + +.brand-sub { + font-size: 11.5px; + color: var(--text-muted); + margin-left: 6px; +} + +.topbar-meta { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +.meta-chip { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 4px 9px; + border-radius: 999px; + background: var(--bg-card); + border: 1px solid var(--border); + font-size: 11px; + font-family: var(--mono); + color: var(--text-muted); + text-decoration: none; +} + +.meta-chip:hover { + border-color: var(--accent); + color: var(--text); +} + +.pulse-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--green); + box-shadow: 0 0 8px var(--green); +} + +/* Sidebar */ +.sidebar { + background: var(--bg-elev); + border-right: 1px solid var(--border); + padding: 14px 12px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 18px; +} + +.nav-group-title { + font-size: 10.5px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--text-dim); + padding: 0 8px 6px; +} + +.nav-btn { + width: 100%; + text-align: left; + background: transparent; + border: 1px solid transparent; + color: var(--text-muted); + padding: 8px 10px; + border-radius: 8px; + cursor: pointer; + display: flex; + flex-direction: column; + gap: 2px; + margin-bottom: 3px; + transition: all 0.14s ease; +} + +.nav-btn:hover { + background: var(--bg-card); + color: var(--text); +} + +.nav-btn.active { + background: rgba(56, 189, 248, 0.11); + border-color: rgba(56, 189, 248, 0.42); + color: #f8fafc; +} + +.nav-btn-row { + display: flex; + align-items: center; + justify-content: space-between; + font-size: 12.5px; + font-weight: 650; +} + +.nav-btn-sub { + font-size: 10.5px; + color: var(--text-dim); + font-family: var(--mono); +} + +.nav-tag { + font-size: 9.5px; + padding: 1px 6px; + border-radius: 99px; + font-family: var(--mono); + background: rgba(255, 255, 255, 0.06); + color: var(--text-muted); +} + +.repo-legend-list { + display: flex; + flex-direction: column; + gap: 6px; + padding: 0 4px; +} + +.repo-legend-item { + display: flex; + align-items: center; + justify-content: space-between; + padding: 6px 8px; + border-radius: 7px; + background: rgba(17, 27, 46, 0.7); + border: 1px solid var(--border); + text-decoration: none; + color: var(--text); + font-size: 11px; +} + +.repo-legend-item:hover { + border-color: var(--border-strong); + background: var(--bg-hover); +} + +.repo-dot-wrap { + display: flex; + align-items: center; + gap: 7px; + font-family: var(--mono); + font-size: 10.5px; +} + +.repo-dot { + width: 8px; + height: 8px; + border-radius: 50%; + flex-shrink: 0; +} + +/* Main workspace */ +.workspace { + display: flex; + flex-direction: column; + overflow: hidden; + background: radial-gradient(circle at 20% 0%, rgba(14, 165, 233, 0.06), transparent 45%); +} + +.era-header { + padding: 10px 16px 8px; + border-bottom: 1px solid var(--border); + background: rgba(13, 20, 36, 0.65); + display: flex; + justify-content: space-between; + align-items: center; + gap: 12px; + flex-wrap: wrap; +} + +.era-title-row { + display: flex; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} + +.era-h1 { + margin: 0; + font-size: 18px; + font-weight: 750; + letter-spacing: -0.015em; +} + +.status-pill { + font-size: 10.5px; + font-family: var(--mono); + padding: 2px 8px; + border-radius: 99px; + border: 1px solid rgba(56, 189, 248, 0.4); + background: rgba(56, 189, 248, 0.1); + color: #7dd3fc; +} + +.era-tagline { + margin: 5px 0 0; + font-size: 12.5px; + color: var(--text-muted); + max-width: 820px; + line-height: 1.45; +} + +.mode-tabs { + display: flex; + gap: 5px; + background: var(--bg); + padding: 4px; + border-radius: 9px; + border: 1px solid var(--border); + flex-shrink: 0; +} + +.mode-tab { + background: transparent; + border: none; + color: var(--text-muted); + font-size: 11.5px; + font-weight: 600; + padding: 6px 11px; + border-radius: 6px; + cursor: pointer; +} + +.mode-tab.active { + background: var(--bg-card); + color: var(--text); + box-shadow: inset 0 0 0 1px var(--border-strong); +} + +/* Flow stepper toolbar */ +.diagram-toolbar { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 18px; + background: var(--bg-elev); + border-bottom: 1px solid var(--border); + gap: 12px; + flex-wrap: wrap; +} + +.flow-controls { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +.toolbar-label { + font-size: 11px; + font-weight: 700; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.flow-select { + background: var(--bg-card); + color: var(--text); + border: 1px solid var(--border-strong); + border-radius: 6px; + padding: 5px 9px; + font-size: 12px; + font-weight: 600; +} + +.ctrl-btn { + background: var(--bg-card); + color: var(--text); + border: 1px solid var(--border); + border-radius: 6px; + padding: 5px 10px; + font-size: 11.5px; + font-weight: 600; + cursor: pointer; +} + +.ctrl-btn:hover:not(:disabled) { + border-color: var(--accent); + background: var(--bg-hover); +} + +.ctrl-btn:disabled { + opacity: 0.4; + cursor: default; +} + +.ctrl-btn.primary { + background: rgba(251, 146, 60, 0.16); + border-color: rgba(251, 146, 60, 0.55); + color: #fed7aa; +} + +.active-step-banner { + display: flex; + align-items: center; + gap: 10px; + padding: 4px 10px; + border-radius: 6px; + background: rgba(251, 146, 60, 0.11); + border: 1px solid rgba(251, 146, 60, 0.35); + font-size: 11.5px; +} + +.active-step-payload { + font-family: var(--mono); + color: #fdba74; + font-size: 11px; +} + +/* Split Diagram Canvas + Code Inspector */ +.canvas-split { + flex: 1; + display: grid; + grid-template-columns: 1fr 365px; + min-height: 0; + overflow: hidden; +} + +.diagram-stage { + position: relative; + overflow: hidden; + background-color: #060a12; + background-image: radial-gradient(rgba(148, 163, 184, 0.14) 1px, transparent 1px); + background-size: 22px 22px; + cursor: grab; +} + +.diagram-stage:active { + cursor: grabbing; +} + +.zoom-hud { + position: absolute; + top: 12px; + right: 12px; + display: flex; + gap: 6px; + background: rgba(13, 20, 36, 0.92); + border: 1px solid var(--border); + padding: 5px; + border-radius: 8px; + z-index: 10; +} + +.edge-legend-hud { + position: absolute; + bottom: 12px; + left: 12px; + display: flex; + gap: 12px; + flex-wrap: wrap; + background: rgba(13, 20, 36, 0.92); + border: 1px solid var(--border); + padding: 6px 11px; + border-radius: 8px; + font-size: 10.5px; + color: var(--text-muted); + z-index: 10; +} + +.edge-legend-item { + display: inline-flex; + align-items: center; + gap: 5px; +} + +.edge-swatch { + width: 14px; + height: 2.5px; + border-radius: 2px; +} + +/* Inspector Drawer */ +.inspector { + background: var(--bg-elev); + border-left: 1px solid var(--border); + padding: 16px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 14px; +} + +.insp-card { + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: 10px; + padding: 12px 14px; +} + +.insp-kicker { + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--text-dim); + margin-bottom: 5px; +} + +.insp-title { + font-size: 15px; + font-weight: 750; + margin: 0 0 6px; +} + +.insp-role { + font-size: 12px; + color: var(--text-muted); + line-height: 1.48; + margin: 0; +} + +.gh-deep-link { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + margin-top: 10px; + padding: 8px 10px; + background: rgba(56, 189, 248, 0.09); + border: 1px solid rgba(56, 189, 248, 0.36); + border-radius: 7px; + color: #7dd3fc; + text-decoration: none; + font-family: var(--mono); + font-size: 11px; + word-break: break-all; +} + +.gh-deep-link:hover { + background: rgba(56, 189, 248, 0.16); +} + +.sym-list { + display: flex; + flex-direction: column; + gap: 5px; + margin-top: 6px; +} + +.sym-item { + display: flex; + justify-content: space-between; + align-items: center; + gap: 6px; + padding: 5px 8px; + border-radius: 6px; + background: rgba(7, 11, 20, 0.65); + border: 1px solid var(--border); + font-family: var(--mono); + font-size: 10.5px; + color: #e2e8f0; + text-decoration: none; +} + +.sym-item:hover { + border-color: var(--accent); +} + +.sym-line { + color: var(--text-dim); + flex-shrink: 0; +} + +.detail-bullets { + margin: 6px 0 0; + padding-left: 17px; + font-size: 12px; + color: #cbd5e1; + line-height: 1.5; +} + +.detail-bullets li { + margin-bottom: 5px; +} + +/* Comparison / Overview & Pipeline Pages */ +.scroll-page { + flex: 1; + overflow-y: auto; + padding: 22px 26px 40px; +} + +.compare-grid { + display: grid; + grid-template-columns: repeat(3, minmax(260px, 1fr)); + gap: 16px; + margin-top: 16px; +} + +.era-card { + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: 12px; + padding: 16px; + display: flex; + flex-direction: column; + gap: 10px; +} + +.era-card:hover { + border-color: var(--border-strong); +} + +.matrix-table { + width: 100%; + border-collapse: collapse; + margin-top: 20px; + font-size: 12.5px; + background: var(--bg-card); + border-radius: 10px; + overflow: hidden; + border: 1px solid var(--border); +} + +.matrix-table th, +.matrix-table td { + padding: 10px 13px; + border-bottom: 1px solid var(--border); + text-align: left; + vertical-align: top; +} + +.matrix-table th { + background: rgba(30, 45, 74, 0.45); + color: #e2e8f0; + font-size: 11.5px; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.code-pre { + background: #050811; + border: 1px solid var(--border); + border-radius: 9px; + padding: 14px; + font-family: var(--mono); + font-size: 11.5px; + color: #cbd5e1; + overflow-x: auto; + line-height: 1.5; +} + +/* 7-Phase Selector Bar */ +.phase-bar { + display: flex; + align-items: center; + gap: 6px; + padding: 6px 16px; + background: rgba(9, 15, 29, 0.94); + border-bottom: 1px solid var(--border); + overflow-x: auto; + flex-shrink: 0; +} + +.phase-pill { + background: var(--bg-card); + border: 1px solid var(--border); + color: var(--text-muted); + border-radius: 7px; + padding: 5px 10px; + font-size: 11.5px; + font-weight: 650; + cursor: pointer; + white-space: nowrap; + display: inline-flex; + align-items: center; + gap: 6px; + transition: all 0.14s ease; +} + +.phase-pill:hover { + border-color: var(--border-strong); + color: var(--text); +} + +.phase-pill.active { + background: rgba(56, 189, 248, 0.15); + border-color: var(--accent); + color: #f8fafc; + box-shadow: 0 0 12px rgba(56, 189, 248, 0.14); +} + +.phase-num { + display: inline-grid; + place-items: center; + width: 17px; + height: 17px; + border-radius: 4px; + font-size: 10px; + font-family: var(--mono); + background: rgba(255, 255, 255, 0.08); + color: #cbd5e1; +} + +.phase-pill.active .phase-num { + background: var(--accent); + color: #060a12; + font-weight: 800; +} + +.phase-context-strip { + padding: 7px 16px; + background: rgba(17, 27, 46, 0.75); + border-bottom: 1px solid var(--border); + display: flex; + justify-content: space-between; + align-items: center; + gap: 12px; + flex-wrap: wrap; + flex-shrink: 0; +} + +@media (max-width: 1120px) { + .app-shell { + grid-template-columns: 1fr; + } + .sidebar { + display: none; + } + .app-shell.sidebar-open-mobile .sidebar { + display: flex; + position: fixed; + top: 48px; + left: 0; + bottom: 0; + width: 260px; + z-index: 60; + box-shadow: 12px 0 32px rgba(0, 0, 0, 0.65); + } + .workspace { + overflow-y: auto; + } + .canvas-split { + display: flex; + flex-direction: column; + flex: none; + overflow: visible; + } + .diagram-stage { + height: 440px; + min-height: 440px; + } + .inspector { + border-left: none; + border-top: 1px solid var(--border); + overflow: visible; + } + .compare-grid { + grid-template-columns: 1fr !important; + } +} + +.phase-bypassed-note { + font-size: 11px; + color: #fbbf24; + font-family: var(--mono); + margin-top: 4px; +} + +@keyframes dashFlow { + to { + stroke-dashoffset: -28; + } +} + +.edge-animated { + stroke-dasharray: 8 6; + animation: dashFlow 0.85s linear infinite; +} diff --git a/docs/site/index.html b/docs/site/index.html new file mode 100644 index 000000000..eed571757 --- /dev/null +++ b/docs/site/index.html @@ -0,0 +1,19 @@ + + + + + + Azure Functions Python Runtime — Architecture Atlas + + + +
+
+ Run python3 eng/docs/extract_arch.py to generate live architecture data. +
+
+ + + + + diff --git a/eng/docs/extract_arch.py b/eng/docs/extract_arch.py new file mode 100644 index 000000000..354faeda5 --- /dev/null +++ b/eng/docs/extract_arch.py @@ -0,0 +1,1309 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +""" +Multi-Repository Code-to-Architecture Graph Extractor for Azure Functions Python. + +Synthesizes live AST/Rust/Proto facts from `Azure/azure-functions-python-worker` +and cross-links the 4-repository Python Functions runtime ecosystem: + 1. Azure/azure-functions-python-worker (workers + runtimes v1/v2) + 2. Azure/azure-functions-python-library (azure-functions PyPI SDK) + 3. Azure/azure-functions-python-extensions (HTTP v2, SdkType deferred bindings, Agent runtimes) + 4. Azure/azure-functions-language-worker-protobuf (FunctionRpc.proto + settlement.proto) + +Outputs: + - site/assets/architecture-graph.json (machine-readable graph + drift fingerprint) + - site/assets/architecture-data.js (zero-fetch bundle for local preview & GitHub Pages) +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from phases_spec import ( + PHASE_CATALOG, + get_py312_phases, + get_py313_phases, + get_py315_phases, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +JSON_OUT = REPO_ROOT / "docs" / "site" / "assets" / "architecture-graph.json" +JS_OUT = REPO_ROOT / "docs" / "site" / "assets" / "architecture-data.js" + +REPOS: dict[str, dict[str, Any]] = { + "python-worker": { + "id": "python-worker", + "fullName": "Azure/azure-functions-python-worker", + "shortName": "python-worker", + "url": "https://github.com/Azure/azure-functions-python-worker", + "defaultBranch": "dev", + "color": "#38bdf8", + "pypi": ["azure-functions-runtime", "azure-functions-runtime-v1"], + "role": "Worker transports (monolith, proxy_worker, R2P2 Rust) and v1/v2 execution runtimes.", + }, + "python-library": { + "id": "python-library", + "fullName": "Azure/azure-functions-python-library", + "shortName": "python-library", + "url": "https://github.com/Azure/azure-functions-python-library", + "defaultBranch": "dev", + "color": "#34d399", + "pypi": ["azure-functions"], + "role": "Customer-facing Python SDK: V2 @app decorators, Blueprints, Datum converters, Trigger/Binding types, Out[T], Context.", + }, + "python-extensions": { + "id": "python-extensions", + "fullName": "Azure/azure-functions-python-extensions", + "shortName": "python-extensions", + "url": "https://github.com/Azure/azure-functions-python-extensions", + "defaultBranch": "dev", + "color": "#c084fc", + "pypi": [ + "azurefunctions-extensions-base", + "azurefunctions-extensions-http-fastapi", + "azurefunctions-extensions-bindings-blob", + "azurefunctions-extensions-bindings-servicebus", + "azurefunctions-agents-extensions-*", + ], + "role": "Extension contracts: WebApp HTTP v2 streaming, SdkType deferred bindings, Settlement gRPC client, and Custom/Agent Runtime hooks.", + }, + "language-worker-protobuf": { + "id": "language-worker-protobuf", + "fullName": "Azure/azure-functions-language-worker-protobuf", + "shortName": "worker-protobuf", + "url": "https://github.com/Azure/azure-functions-language-worker-protobuf", + "defaultBranch": "dev", + "color": "#fbbf24", + "pypi": [], + "role": "Canonical gRPC wire schema (`FunctionRpc.proto`, `settlement.proto`, `ClaimsIdentityRpc.proto`) shared by Host and all language workers.", + }, +} + + +def git_info(repo_dir: Path = REPO_ROOT) -> dict[str, str]: + def run(args: list[str], default: str = "") -> str: + try: + return subprocess.check_output( + ["git", *args], cwd=repo_dir, stderr=subprocess.DEVNULL, text=True + ).strip() + except Exception: + return default + + return { + "sha": run(["rev-parse", "HEAD"], "dev"), + "shortSha": run(["rev-parse", "--short", "HEAD"], "dev"), + "branch": run(["rev-parse", "--abbrev-ref", "HEAD"], "dev"), + "subject": run(["log", "-1", "--pretty=%s"], ""), + "committedAt": run(["log", "-1", "--pretty=%cI"], ""), + } + + +def extract_python_file(rel_path: str, base_dir: Path = REPO_ROOT) -> dict[str, Any]: + full = base_dir / rel_path + if not full.exists(): + return {"path": rel_path, "exists": False, "lines": 0, "symbols": [], "routerCases": []} + + src = full.read_text(encoding="utf-8", errors="replace") + lines = src.count("\n") + 1 + symbols: list[dict[str, Any]] = [] + match_cases: list[str] = [] + + try: + tree = ast.parse(src, filename=rel_path) + doc = ast.get_docstring(tree) or "" + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = [a.arg for a in node.args.args if a.arg != "self"] + symbols.append({ + "name": node.name, + "kind": "async def" if isinstance(node, ast.AsyncFunctionDef) else "def", + "line": node.lineno, + "signature": f"{node.name}({', '.join(args[:4])}{', ...' if len(args) > 4 else ''})", + }) + elif isinstance(node, ast.ClassDef): + methods = [ + n.name for n in node.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and not n.name.startswith("__") + ] + symbols.append({ + "name": node.name, + "kind": "class", + "line": node.lineno, + "signature": f"class {node.name}", + "methods": methods[:6], + }) + for node in ast.walk(tree): + if isinstance(node, ast.Match): + for case in node.cases: + if isinstance(case.pattern, ast.MatchValue) and isinstance(case.pattern.value, ast.Constant): + val = str(case.pattern.value.value) + if val.endswith("_request") or val.endswith("_response"): + match_cases.append(val) + except SyntaxError: + doc = "" + + return { + "path": rel_path, + "exists": True, + "lines": lines, + "docSummary": doc.strip().splitlines()[0] if doc.strip() else "", + "symbols": symbols[:14], + "routerCases": sorted(set(match_cases)), + } + + +def extract_rust_file(rel_path: str) -> dict[str, Any]: + full = REPO_ROOT / rel_path + if not full.exists(): + return {"path": rel_path, "exists": False, "lines": 0, "symbols": [], "pyo3Calls": []} + + src = full.read_text(encoding="utf-8", errors="replace") + src_lines = src.splitlines() + symbols: list[dict[str, Any]] = [] + pyo3_calls: list[str] = [] + + fn_re = re.compile(r"^\s*(pub(?:\(crate\))?\s+)?(async\s+)?fn\s+([A-Za-z0-9_]+)\s*\(([^)]*)\)") + struct_re = re.compile(r"^\s*(pub\s+)?(struct|enum)\s+([A-Za-z0-9_]+)") + call_method_re = re.compile(r'call_method1\(\s*"([A-Za-z0-9_]+)"') + + for idx, line in enumerate(src_lines, start=1): + m_fn = fn_re.match(line) + if m_fn: + is_pub, is_async, name, params = m_fn.groups() + if name.startswith("test_"): + continue + kind = f"{'pub ' if is_pub else ''}{'async ' if is_async else ''}fn".strip() + short_params = ", ".join( + p.strip().split(":")[0].strip() for p in params.split(",") if p.strip() + ) + symbols.append({ + "name": name, + "kind": kind, + "line": idx, + "signature": f"{name}({short_params})", + }) + continue + m_st = struct_re.match(line) + if m_st: + _, kw, name = m_st.groups() + symbols.append({"name": name, "kind": kw, "line": idx, "signature": name}) + + for m in call_method_re.finditer(src): + pyo3_calls.append(m.group(1)) + + return { + "path": rel_path, + "exists": True, + "lines": len(src_lines), + "symbols": symbols[:16], + "pyo3Calls": sorted(set(pyo3_calls)), + } + + +def extract_proto_messages(rel_path: str) -> list[dict[str, Any]]: + full = REPO_ROOT / rel_path + if not full.exists(): + return [] + src = full.read_text(encoding="utf-8", errors="replace") + m = re.search(r"message\s+StreamingMessage\s*\{(.*?)\n\}", src, flags=re.S) + if not m: + return [] + oneof = re.search(r"oneof\s+content\s*\{(.*?)\}", m.group(1), flags=re.S) + if not oneof: + return [] + fields = [] + for fm in re.finditer(r"^\s*([A-Za-z0-9_]+)\s+([a-z0-9_]+)\s*=\s*(\d+)\s*;", oneof.group(1), flags=re.M): + msg_type, field_name, tag = fm.groups() + fields.append({"type": msg_type, "field": field_name, "tag": int(tag)}) + return fields + + +def scan_package_files(rel_dir: str, suffixes: tuple[str, ...]) -> list[dict[str, Any]]: + base = REPO_ROOT / rel_dir + if not base.exists(): + return [] + out = [] + for p in sorted(base.rglob("*")): + if not p.is_file() or p.suffix not in suffixes: + continue + rel = p.relative_to(REPO_ROOT).as_posix() + if any(seg in rel for seg in ("/_thirdparty/", "/_vendored/", "/protos/", "__pycache__", "/target/")): + continue + lines = p.read_text(encoding="utf-8", errors="replace").count("\n") + 1 + out.append({"path": rel, "lines": lines}) + return out + + +def sym_line(meta: dict[str, Any], name: str, fallback: int = 1) -> int: + for s in meta.get("symbols", []): + if s["name"] == name: + return s["line"] + return fallback + + +def compile_topology_mermaid(era: dict[str, Any], edges_override: list[dict[str, Any]] | None = None) -> str: + lines = ["flowchart LR"] + by_parent: dict[str | None, list[dict[str, Any]]] = {} + for c in era["containers"]: + by_parent.setdefault(c.get("parentId"), []).append(c) + nodes_by_c: dict[str, list[dict[str, Any]]] = {} + for n in era["nodes"]: + nodes_by_c.setdefault(n["containerId"], []).append(n) + + def emit(c: dict[str, Any], ind: str) -> None: + cid = c["id"].replace("-", "_") + lines.append(f'{ind}subgraph {cid}["{c["label"]}"]') + for sub in by_parent.get(c["id"], []): + emit(sub, ind + " ") + for n in nodes_by_c.get(c["id"], []): + nid = n["id"].replace("-", "_") + repo_short = REPOS.get(n["repo"], {}).get("shortName", n["repo"]) + lines.append(f'{ind} {nid}["{n["label"]}
({repo_short})"]') + lines.append(f"{ind}end") + + for rc in by_parent.get(None, []): + emit(rc, " ") + for e in (edges_override if edges_override is not None else era["edges"]): + s = e["source"].replace("-", "_") + t = e["target"].replace("-", "_") + arr = "-.->" if e.get("dashed") else "-->" + lines.append(f' {s} {arr}|"{e["label"]}"| {t}') + return "\n".join(lines) + + +def compile_sequence_mermaid(flow: dict[str, Any]) -> str: + lines = ["sequenceDiagram"] + for a in flow["actors"]: + lines.append(f' participant {a["id"].replace("-", "_")} as {a["label"]}') + for st in flow["steps"]: + fr = st["from"].replace("-", "_") + to = st["to"].replace("-", "_") + arr = "-->>" if st.get("isReturn") else "->>" + lines.append(f' {fr}{arr}{to}: {st["message"]}') + return "\n".join(lines) + + +def resolve_github_url(node: dict[str, Any], repo_refs: dict[str, str]) -> str: + repo_id = node["repo"] + repo = REPOS[repo_id] + ref = repo_refs.get(repo_id) or repo["defaultBranch"] + line_sfx = f"#L{node['line']}" if node.get("line", 0) > 1 else "" + return f"{repo['url']}/blob/{ref}/{node['file']}{line_sfx}" + + +def build_graph(sibling_root: Path | None = None) -> dict[str, Any]: + git = git_info(REPO_ROOT) + repo_dirs: dict[str, Path] = {"python-worker": REPO_ROOT} + repo_refs: dict[str, str] = { + "python-worker": git["sha"] if len(git["sha"]) >= 7 else "dev" + } + if sibling_root and sibling_root.exists(): + for rid in ("python-library", "python-extensions", "language-worker-protobuf"): + cand = sibling_root / rid + if cand.exists(): + repo_dirs[rid] = cand + s_git = git_info(cand) + if len(s_git["sha"]) >= 7: + repo_refs[rid] = s_git["sha"] + + py_paths = [ + "workers/python/prodV4/worker.py", + "workers/python/proxyV4/worker.py", + "workers/azure_functions_worker/main.py", + "workers/azure_functions_worker/dispatcher.py", + "workers/azure_functions_worker/loader.py", + "workers/azure_functions_worker/http_v2.py", + "workers/azure_functions_worker/extension.py", + "workers/azure_functions_worker/utils/dependency.py", + "workers/proxy_worker/start_worker.py", + "workers/proxy_worker/dispatcher.py", + "workers/proxy_worker/logging.py", + "workers/proxy_worker/utils/dependency.py", + "workers/r2p2/bridge/bridge.py", + "workers/r2p2/bridge/protos_adapter.py", + "runtimes/v2/azure_functions_runtime/handle_event.py", + "runtimes/v2/azure_functions_runtime/native_invocation.py", + "runtimes/v2/azure_functions_runtime/loader.py", + "runtimes/v2/azure_functions_runtime/http_v2.py", + "runtimes/v2/azure_functions_runtime/otel.py", + "runtimes/v1/azure_functions_runtime_v1/handle_event.py", + "runtimes/v1/azure_functions_runtime_v1/native_invocation.py", + ] + rs_paths = [ + "workers/r2p2/src/main.rs", + "workers/r2p2/src/bridge.rs", + "workers/r2p2/src/control.rs", + "workers/r2p2/src/convert.rs", + "workers/r2p2/src/codec.rs", + "workers/r2p2/src/log.rs", + "workers/r2p2/src/pb.rs", + ] + + extracted: dict[str, dict[str, Any]] = {p: extract_python_file(p) for p in py_paths} + for p in rs_paths: + extracted[p] = extract_rust_file(p) + + proto_messages = extract_proto_messages("workers/r2p2/proto/FunctionRpc.proto") + inventories = { + "azure_functions_worker": scan_package_files("workers/azure_functions_worker", (".py",)), + "proxy_worker": scan_package_files("workers/proxy_worker", (".py",)), + "r2p2_rust": scan_package_files("workers/r2p2/src", (".rs",)), + "r2p2_bridge": scan_package_files("workers/r2p2/bridge", (".py",)), + "runtime_v2": scan_package_files("runtimes/v2/azure_functions_runtime", (".py",)), + "runtime_v1": scan_package_files("runtimes/v1/azure_functions_runtime_v1", (".py",)), + } + + mono_disp = extracted["workers/azure_functions_worker/dispatcher.py"] + proxy_disp = extracted["workers/proxy_worker/dispatcher.py"] + r2p2_main = extracted["workers/r2p2/src/main.rs"] + r2p2_bridge_rs = extracted["workers/r2p2/src/bridge.rs"] + r2p2_bridge_py = extracted["workers/r2p2/bridge/bridge.py"] + rt_v2_event = extracted["runtimes/v2/azure_functions_runtime/handle_event.py"] + rt_v2_native = extracted["runtimes/v2/azure_functions_runtime/native_invocation.py"] + + eras = [ + # ===================================================================== + # VIEW 0: 4-Repository Ecosystem Topology + # ===================================================================== + { + "id": "ecosystem", + "shortTitle": "4-Repo Ecosystem", + "title": "Cross-Repository Architecture (Wire, Worker, SDK & Extensions)", + "pythonVersions": "All Supported Versions (3.7 → 3.15+)", + "packagePath": "4 Linked GitHub Repositories", + "launcherPath": "Host <-> Protobuf <-> Worker <-> Library & Extensions", + "status": "Canonical Multi-Repo Map", + "tagline": "How language-worker-protobuf, python-worker, python-library, and python-extensions compose the Python Functions runtime.", + "summary": ( + "The Azure Functions Python stack spans four specialized repositories: " + "`language-worker-protobuf` defines the cross-language gRPC wire contract; " + "`python-worker` implements the transport shells (monolith, proxy_worker, R2P2) and " + "execution runtimes (v1/v2); `python-library` provides the `azure-functions` decorator " + "model and `Datum` type converters; and `python-extensions` powers HTTP v2 ASGI streaming, " + "SDK-type deferred bindings, Settlement gRPC, and pluggable Agent Runtimes." + ), + "tradeoffs": [ + {"kind": "strength", "text": "Wire contract (`FunctionRpc.proto`) stays identical while worker transport evolves from Python `grpcio` to Rust `tonic`."}, + {"kind": "strength", "text": "`python-library` and `python-extensions` ship on PyPI independently of Azure Host images."}, + {"kind": "strength", "text": "Clean extension hooks (`WebApp`, `SdkType`, `RuntimeTrackerMeta`) let FastAPI, Blob SDK types, and Agent Frameworks plug in without forking the worker."}, + ], + "erasDiff": { + "transport": "Defined in language-worker-protobuf; implemented in python-worker", + "boundary": "Datum & Decorator metadata (python-library) + Base metaclasses (python-extensions)", + "runtimeLoading": "python-worker runtimes/v1,v2 or python-extensions RuntimeTrackerMeta", + "hotPathOverhead": "Datum bindings take R2P2 Hot Path; Extensions (HTTP-v2 / SdkType) take Control Path", + }, + "containers": [ + {"id": "ce-proto", "label": "Azure/azure-functions-language-worker-protobuf", "parentId": None, "tone": "proto"}, + {"id": "ce-worker", "label": "Azure/azure-functions-python-worker", "parentId": None, "tone": "rust-proc"}, + {"id": "ce-w-shell", "label": "Worker Transport Shells (workers/)", "parentId": "ce-worker", "tone": "transport"}, + {"id": "ce-w-rt", "label": "Execution Runtimes (runtimes/v1 & v2)", "parentId": "ce-worker", "tone": "runtime"}, + {"id": "ce-lib", "label": "Azure/azure-functions-python-library", "parentId": None, "tone": "lib"}, + {"id": "ce-ext", "label": "Azure/azure-functions-python-extensions", "parentId": None, "tone": "ext"}, + ], + "nodes": [ + { + "id": "ne-funcrpc", + "repo": "language-worker-protobuf", + "containerId": "ce-proto", + "label": "FunctionRpc.proto", + "badge": "EventStream + StreamingMessage", + "role": "Canonical bidirectional gRPC protocol between the .NET Script Host and all Python workers.", + "file": "src/proto/FunctionRpc.proto", + "line": 1, + "details": [ + "Compiled via `grpcio-tools` for ≤3.14 workers and via `prost-build`/`tonic` (`workers/r2p2/build.rs`) for 3.15+.", + "Defines `WorkerInit`, `FunctionsMetadata`, `FunctionLoad`, `InvocationRequest`, `RpcLog`, `WorkerStatus`.", + ], + }, + { + "id": "ne-settle", + "repo": "language-worker-protobuf", + "containerId": "ce-proto", + "label": "settlement.proto & ClaimsIdentity", + "badge": "ServiceBus Settlement", + "role": "gRPC settlement service allowing Python functions to complete/abandon/dead-letter ServiceBus messages.", + "file": "src/proto/servicebus/settlement.proto", + "line": 1, + "details": [ + "Consumed by `grpcClientType.py` in `python-extensions` and `get_settlement_client()` in `python-worker`.", + ], + }, + { + "id": "ne-shells", + "repo": "python-worker", + "containerId": "ce-w-shell", + "label": "Transports: Monolith / Proxy / R2P2", + "badge": "≤3.12 | 3.13-14 | 3.15+", + "role": "Version-selected worker shell managing Host gRPC stream, logging, and sys.path dependency isolation.", + "file": "workers/r2p2/src/main.rs", + "line": sym_line(r2p2_main, "main", 230), + "details": [ + "≤3.12: `workers/azure_functions_worker` (monolith).", + "3.13–3.14: `workers/proxy_worker` (Python transport shell).", + "3.15+: `workers/r2p2` (Rust `tonic`+`prost` + `PyO3` embedded CPython).", + ], + }, + { + "id": "ne-runtimes", + "repo": "python-worker", + "containerId": "ce-w-rt", + "label": "azure-functions-runtime (v2 & v1)", + "badge": "handle_event + native_invocation", + "role": "PyPI-published execution engines that index apps, decode/encode bindings, and run sync/async handlers.", + "file": "runtimes/v2/azure_functions_runtime/handle_event.py", + "line": sym_line(rt_v2_event, "worker_init_request", 58), + "details": [ + "`runtimes/v2`: V2 decorator model, HTTP v2 coordinator, OpenTelemetry.", + "`runtimes/v1`: V1 `function.json` folder model.", + "Exposes both `handle_event.py` (5-verb contract) and `native_invocation.py` (R2P2 zero-protobuf seam).", + ], + }, + { + "id": "ne-lib-dec", + "repo": "python-library", + "containerId": "ce-lib", + "label": "Decorators & Blueprints", + "badge": "FunctionApp / @app.*", + "role": "Defines V2 Python programming model (`@app.route`, `@app.queue_trigger`, `Blueprint`, MCP & Connector triggers).", + "file": "azure/functions/__init__.py", + "line": 1, + "details": [ + "Inspected by `runtimes/v2/azure_functions_runtime/loader.py` (`index_function_app`) during WorkerInit/Metadata.", + "Includes `azure/functions/mcp.py`, `connectors.py`, `warmup.py`, `decorators/`.", + ], + }, + { + "id": "ne-lib-datum", + "repo": "python-library", + "containerId": "ce-lib", + "label": "meta.py Datum & Converters", + "badge": "Datum / InConverter / Out[T]", + "role": "Protobuf-agnostic `Datum` value container and trigger/binding converters (`HttpRequest`, `QueueMessage`, `Blob`).", + "file": "azure/functions/meta.py", + "line": 1, + "details": [ + "`Datum` is the exact currency constructed by R2P2's `native_invocation.py` — enabling zero-protobuf execution!", + "`azure/functions/extension/` also defines `AppExtensionBase` & `FuncExtensionBase`.", + ], + }, + { + "id": "ne-ext-base", + "repo": "python-extensions", + "containerId": "ce-ext", + "label": "extensions-base (WebApp, SdkType, Runtime)", + "badge": "web.py | sdkType.py | runtime.py", + "role": "Metaclass registries bridging HTTP v2 ASGI servers, SDK-type deferred bindings, and custom Agent Runtimes.", + "file": "azurefunctions-extensions-base/azurefunctions/extensions/base/runtime.py", + "line": 1, + "details": [ + "`runtime.py` (`RuntimeTrackerMeta`): discovered when `PYTHON_ENABLE_AGENT_RUNTIME=1`.", + "`web.py` (`WebApp`): drives `http_v2.py` loopback streaming for FastAPI/ASGI.", + "`sdkType.py` (`SdkType`): drives deferred bindings (`requires_control_path() == true` in R2P2).", + ], + }, + { + "id": "ne-ext-pkgs", + "repo": "python-extensions", + "containerId": "ce-ext", + "label": "FastAPI, SDK Bindings & Agent Runtimes", + "badge": "http-fastapi | bindings-* | agents-*", + "role": "Concrete extension packages for FastAPI HTTP streaming, Blob/Cosmos/EventHub/ServiceBus SDK types, and Agents.", + "file": "azurefunctions-extensions-http-fastapi/README.md", + "line": 1, + "details": [ + "`azurefunctions-extensions-http-fastapi`: Request/Response streaming (`StreamingResponse`, SSE).", + "`azurefunctions-extensions-bindings-{blob,cosmosdb,eventhub,servicebus}`: rich Azure SDK client bindings.", + "`azurefunctions-agents-extensions-{agent-framework,copilot-sdk}`: hosted agent runtimes.", + ], + }, + ], + "edges": [ + {"id": "ee-1", "source": "ne-funcrpc", "target": "ne-shells", "kind": "grpc-wire", "label": "prost / grpcio codegen"}, + {"id": "ee-2", "source": "ne-shells", "target": "ne-runtimes", "kind": "pyo3-hot", "label": "WorkerRequest / PyO3 FFI"}, + {"id": "ee-3", "source": "ne-runtimes", "target": "ne-lib-dec", "kind": "inproc-contract", "label": "index_function_app()"}, + {"id": "ee-4", "source": "ne-runtimes", "target": "ne-lib-datum", "kind": "inproc-contract", "label": "Datum <-> Binding decode"}, + {"id": "ee-5", "source": "ne-runtimes", "target": "ne-ext-base", "kind": "pyo3-control", "label": "http_v2 / deferred SdkType"}, + {"id": "ee-6", "source": "ne-shells", "target": "ne-ext-base", "kind": "syspath-mutation", "label": "entry_points(azurefunctions.runtimes)", "dashed": True}, + {"id": "ee-7", "source": "ne-settle", "target": "ne-ext-base", "kind": "grpc-wire", "label": "grpcClientType settlement", "dashed": True}, + {"id": "ee-8", "source": "ne-ext-base", "target": "ne-ext-pkgs", "kind": "exec-dispatch", "label": "metaclass registration"}, + ], + "flows": [ + { + "id": "eco_flow", + "title": "End-to-End Cross-Repo Request Lifecycle", + "summary": "How a single HTTP/SDK request traverses contracts defined across all 4 repositories.", + "actors": [ + {"id": "proto", "label": "worker-protobuf"}, + {"id": "shell", "label": "python-worker (Shell)"}, + {"id": "rt", "label": "python-worker (Runtime)"}, + {"id": "lib", "label": "python-library"}, + {"id": "ext", "label": "python-extensions"}, + ], + "steps": [ + {"from": "proto", "to": "shell", "message": "FunctionRpc.proto StreamingMessage frame", "activeNodes": ["ne-funcrpc", "ne-shells"], "activeEdges": ["ee-1"], "payload": "Shared wire contract across 3.7 -> 3.15+"}, + {"from": "shell", "to": "rt", "message": "Dispatch WorkerInit / Invocation across seam", "activeNodes": ["ne-shells", "ne-runtimes"], "activeEdges": ["ee-2"], "payload": "WorkerRequest (3.13) or PyO3 Tuple/Dict (3.15)"}, + {"from": "rt", "to": "lib", "message": "Index @app decorators & decode Datum inputs", "activeNodes": ["ne-runtimes", "ne-lib-dec", "ne-lib-datum"], "activeEdges": ["ee-3", "ee-4"], "payload": "azure.functions.FunctionApp + meta.Datum"}, + {"from": "rt", "to": "ext", "message": "Hydrate SdkType (BlobClient) or stream WebApp (FastAPI)", "activeNodes": ["ne-runtimes", "ne-ext-base", "ne-ext-pkgs"], "activeEdges": ["ee-5", "ee-8"], "payload": "azurefunctions.extensions.base"}, + {"from": "ext", "to": "proto", "message": "Optional ServiceBus Settlement gRPC call to Host", "isReturn": True, "activeNodes": ["ne-ext-base", "ne-settle"], "activeEdges": ["ee-7"], "payload": "settlement.proto Complete/Abandon"}, + ], + }, + ], + }, + # ===================================================================== + # ERA 1: Python <= 3.12 (Monolithic Worker) + # ===================================================================== + { + "id": "py312", + "shortTitle": "Python ≤ 3.12", + "title": "Monolithic Python Worker (≤ 3.12)", + "pythonVersions": "3.7 – 3.12", + "packagePath": "workers/azure_functions_worker", + "launcherPath": "workers/python/prodV4/worker.py", + "status": "Maintenance (Legacy Monolith)", + "tagline": "Single-package CPython worker owning gRPC transport, protobuf, indexing, bindings, and invocation.", + "summary": ( + "For Python 3.12 and below, `azure_functions_worker` is a single monolithic package. " + "Transport (`grpcio` + `google.protobuf` compiled from `language-worker-protobuf`), " + "dependency isolation, V1/V2 function indexing (`python-library`), HTTP v2 streaming " + "(`python-extensions`), and execution all live in one coupled tree." + ), + "tradeoffs": [ + {"kind": "strength", "text": "Direct single-package Python calls; simplest mental model."}, + {"kind": "pain", "text": "Coupled releases: any binding/HTTP/OTel fix requires shipping a new Host worker bundle for every Python version."}, + {"kind": "pain", "text": "Worker `grpcio` and `google.protobuf` C-extensions live in the same interpreter as customer code."}, + ], + "erasDiff": { + "transport": "Python grpcio + google.protobuf (compiled from FunctionRpc.proto)", + "boundary": "Monolithic internal imports inside azure_functions_worker", + "runtimeLoading": "Bundled directly inside worker (no PyPI runtime split)", + "hotPathOverhead": "grpcio thread -> asyncio -> Python protobuf -> Datum -> handler", + }, + "containers": [ + {"id": "c312-host", "label": "Host & Wire Contract", "parentId": None, "tone": "proto"}, + {"id": "c312-worker", "label": "Monolithic Worker Process (workers/azure_functions_worker)", "parentId": None, "tone": "python"}, + {"id": "c312-trans", "label": "gRPC Transport & Isolation", "parentId": "c312-worker", "tone": "transport"}, + {"id": "c312-eng", "label": "In-Tree Indexing, Bindings & Executor", "parentId": "c312-worker", "tone": "runtime"}, + {"id": "c312-eco", "label": "SDK, Extensions & Customer App", "parentId": None, "tone": "lib"}, + ], + "nodes": [ + { + "id": "n312-proto", + "repo": "language-worker-protobuf", + "containerId": "c312-host", + "label": "Host + FunctionRpc.proto", + "badge": "EventStream Server", + "role": "Functions Host spawns `prodV4/worker.py` and streams `StreamingMessage` frames over HTTP/2.", + "file": "src/proto/FunctionRpc.proto", + "line": 1, + "details": [ + "Compiled into `workers/azure_functions_worker/protos/` via `grpcio-tools`.", + ], + }, + { + "id": "n312-boot", + "repo": "python-worker", + "containerId": "c312-trans", + "label": "prodV4/worker.py & main.py", + "badge": "Boot + DependencyManager", + "role": "Sets worker-first sys.path so bundled `grpcio`/`protobuf` import cleanly, then starts asyncio loop.", + "file": "workers/azure_functions_worker/main.py", + "line": sym_line(extracted["workers/azure_functions_worker/main.py"], "start_async", 1), + "details": [ + "`DependencyManager.initialize()` isolates worker deps during placeholder cold boot.", + ], + }, + { + "id": "n312-grpc", + "repo": "python-worker", + "containerId": "c312-trans", + "label": "gRPC Pump (__poll_grpc)", + "badge": "grpcio + _grpc_resp_queue", + "role": "Background OS thread reading `stub.EventStream` and posting tasks via `call_soon_threadsafe`.", + "file": "workers/azure_functions_worker/dispatcher.py", + "line": sym_line(mono_disp, "connect", 180), + "details": [ + "Outbound responses and `AsyncLoggingHandler` `RpcLog` records drain from `_grpc_resp_queue`.", + ], + }, + { + "id": "n312-disp", + "repo": "python-worker", + "containerId": "c312-trans", + "label": "Monolithic Dispatcher", + "badge": "dispatcher.py", + "role": "Central class owning message routing, logging switch, OTel/AppInsights setup, and `_sync_call_tp`.", + "file": "workers/azure_functions_worker/dispatcher.py", + "line": sym_line(mono_disp, "Dispatcher", 76), + "details": [ + "Directly imports `loader`, `bindings`, `http_v2`, `SharedMemoryManager`, and `ExtensionManager`.", + ], + }, + { + "id": "n312-dep", + "repo": "python-worker", + "containerId": "c312-trans", + "label": "DependencyManager", + "badge": "utils/dependency.py", + "role": "Manages worker-first placeholder boot vs customer `.python_packages` promotion on Init/Specialization.", + "file": "workers/azure_functions_worker/utils/dependency.py", + "line": sym_line(extracted["workers/azure_functions_worker/utils/dependency.py"], "DependencyManager", 1), + "details": [ + "Controls `use_worker_dependencies()` and `prioritize_customer_dependencies()`.", + ], + }, + { + "id": "n312-load", + "repo": "python-worker", + "containerId": "c312-eng", + "label": "Loader & ExtensionManager", + "badge": "loader.py + extension.py", + "role": "Indexes V2 `@app` decorators or V1 `function.json` files and invokes `AppExtensionBase` hooks.", + "file": "workers/azure_functions_worker/loader.py", + "line": sym_line(extracted["workers/azure_functions_worker/loader.py"], "index_function_app", 1), + "details": [ + "Populates `functions.Registry` with parameter binding metadata.", + ], + }, + { + "id": "n312-bind", + "repo": "python-worker", + "containerId": "c312-eng", + "label": "Bindings, SharedMem & http_v2", + "badge": "bindings/ + mmap + ASGI", + "role": "Converts `protos.TypedData` <-> `Datum`, manages shared-memory rings, and coordinates HTTP v2 streaming.", + "file": "workers/azure_functions_worker/http_v2.py", + "line": sym_line(extracted["workers/azure_functions_worker/http_v2.py"], "HttpV2Registry", 1), + "details": [ + "Calls into `python-library` converters and `python-extensions` `SdkType`/`WebApp`.", + ], + }, + { + "id": "n312-sdk", + "repo": "python-library", + "containerId": "c312-eco", + "label": "azure-functions SDK", + "badge": "FunctionApp + Datum + Out", + "role": "Customer decorators, trigger/binding models (`HttpRequest`, `QueueMessage`), and `Datum` converters.", + "file": "azure/functions/meta.py", + "line": 1, + "details": [ + "Shared by worker `bindings/meta.py` and customer `function_app.py`.", + ], + }, + { + "id": "n312-ext", + "repo": "python-extensions", + "containerId": "c312-eco", + "label": "azurefunctions-extensions-*", + "badge": "http-fastapi + bindings-*", + "role": "Optional HTTP v2 ASGI streaming (`WebApp`) and deferred SDK bindings (`BlobClient`, etc.).", + "file": "azurefunctions-extensions-base/azurefunctions/extensions/base/sdkType.py", + "line": 1, + "details": [ + "Hooks into `loader.py` and `http_v2.py` via base metaclasses.", + ], + }, + ], + "edges": [ + {"id": "e312-1", "source": "n312-proto", "target": "n312-grpc", "kind": "grpc-wire", "label": "EventStream (grpcio)"}, + {"id": "e312-2", "source": "n312-boot", "target": "n312-disp", "kind": "syspath-mutation", "label": "boot + DependencyManager"}, + {"id": "e312-3", "source": "n312-grpc", "target": "n312-disp", "kind": "thread-queue", "label": "call_soon_threadsafe / Queue"}, + {"id": "e312-4", "source": "n312-disp", "target": "n312-load", "kind": "inproc-contract", "label": "index & load"}, + {"id": "e312-5", "source": "n312-disp", "target": "n312-bind", "kind": "inproc-contract", "label": "TypedData <-> args"}, + {"id": "e312-6", "source": "n312-load", "target": "n312-sdk", "kind": "exec-dispatch", "label": "inspect @app / function.json"}, + {"id": "e312-7", "source": "n312-bind", "target": "n312-sdk", "kind": "exec-dispatch", "label": "Datum decode + invoke"}, + {"id": "e312-8", "source": "n312-bind", "target": "n312-ext", "kind": "pyo3-control", "label": "WebApp / SdkType", "dashed": True}, + ], + "flows": [ + { + "id": "init", + "title": "Cold Start & Monolithic Init", + "summary": "prodV4 boots monolith, opens grpcio EventStream, switches sys.path to customer packages, and indexes decorators.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "grpc", "label": "gRPC Thread"}, + {"id": "disp", "label": "Dispatcher"}, + {"id": "load", "label": "Loader"}, + {"id": "sdk", "label": "python-library"}, + ], + "steps": [ + {"from": "host", "to": "grpc", "message": "Spawn prodV4/worker.py + StartStream", "activeNodes": ["n312-proto", "n312-boot", "n312-grpc"], "activeEdges": ["e312-1", "e312-2"], "payload": "Worker grpcio/protobuf loaded first"}, + {"from": "host", "to": "disp", "message": "WorkerInitRequest(app_dir, capabilities)", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["e312-1", "e312-3"], "payload": "StreamingMessage.worker_init_request"}, + {"from": "disp", "to": "load", "message": "prioritize_customer_deps + index_function_app()", "activeNodes": ["n312-disp", "n312-load"], "activeEdges": ["e312-4"], "payload": "Customer .python_packages promoted"}, + {"from": "load", "to": "sdk", "message": "Read FunctionApp / Blueprints", "activeNodes": ["n312-load", "n312-sdk"], "activeEdges": ["e312-6"], "payload": "azure.functions decorators"}, + {"from": "disp", "to": "host", "message": "WorkerInitResponse(capabilities)", "isReturn": True, "activeNodes": ["n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["e312-3", "e312-1"], "payload": "SHARED_MEMORY + RAW_HTTP_BODY_BYTES"}, + ], + }, + { + "id": "invoke", + "title": "Invocation Lifecycle", + "summary": "Dispatcher decodes protobuf TypedData via bindings, runs sync threadpool or async coroutine, and encodes response.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "disp", "label": "Dispatcher"}, + {"id": "bind", "label": "Bindings / http_v2"}, + {"id": "sdk", "label": "SDK & User App"}, + ], + "steps": [ + {"from": "host", "to": "disp", "message": "InvocationRequest(function_id, input_data)", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["e312-1", "e312-3"], "payload": "google.protobuf InvocationRequest"}, + {"from": "disp", "to": "bind", "message": "from_incoming_proto() / SharedMemory", "activeNodes": ["n312-disp", "n312-bind"], "activeEdges": ["e312-5"], "payload": "TypedData -> Datum"}, + {"from": "bind", "to": "sdk", "message": "Decode SDK types + run handler (_sync_call_tp / await)", "activeNodes": ["n312-bind", "n312-sdk", "n312-ext"], "activeEdges": ["e312-7", "e312-8"], "payload": "HttpRequest / SdkType -> User fn"}, + {"from": "sdk", "to": "host", "message": "to_outgoing_proto() -> InvocationResponse", "isReturn": True, "activeNodes": ["n312-sdk", "n312-bind", "n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["e312-7", "e312-5", "e312-3", "e312-1"], "payload": "Queued onto _grpc_resp_queue"}, + ], + }, + ], + }, + # ===================================================================== + # ERA 2: Python 3.13 - 3.14 (Split Proxy Worker + Pluggable Runtimes) + # ===================================================================== + { + "id": "py313_314", + "shortTitle": "Python 3.13 – 3.14", + "title": "Split Proxy Worker + PyPI Runtimes (3.13 – 3.14)", + "pythonVersions": "3.13 – 3.14", + "packagePath": "workers/proxy_worker + runtimes/{v1,v2}", + "launcherPath": "workers/python/proxyV4/worker.py", + "status": "Production (Python 3.13 / 3.14)", + "tagline": "Thin transport shell (`proxy_worker`) calling decoupled PyPI engines (`azure-functions-runtime` v2/v1 or Agent Runtimes).", + "summary": ( + "Python 3.13 split the worker into two independently versioned layers. " + "`workers/proxy_worker` is a thin, stable transport shell owning `grpcio`, " + "`DependencyManager`, logging switch, and 4-tier `invocation_id` tracking. " + "It delegates execution across a 5-verb `WorkerRequest` seam (injecting `protos` at init) " + "to `runtimes/v2`, `runtimes/v1`, or a custom `python-extensions` Agent Runtime." + ), + "tradeoffs": [ + {"kind": "strength", "text": "Decoupled cadence: `azure-functions-runtime` updates ship via PyPI without waiting for Host worker deployments."}, + {"kind": "strength", "text": "Injected `protos` decouples runtime code from specific protobuf wheel builds; unlocks pluggable Agent Runtimes."}, + {"kind": "pain", "text": "`proxy_worker` still links CPython `grpcio` + `protobuf` wheels (delaying day-1 support for new Python releases) and contends on the GIL."}, + ], + "erasDiff": { + "transport": "Python proxy_worker (grpcio + worker protobuf + uvloop)", + "boundary": "In-process async WorkerRequest(name, request, properties={protos, host})", + "runtimeLoading": "reload_library_worker(): Agent EntryPoint -> V2 (function_app.py) -> V1", + "hotPathOverhead": "grpcio thread -> asyncio -> WorkerRequest -> runtime cached protos", + }, + "containers": [ + {"id": "c313-host", "label": "Host & Wire Contract", "parentId": None, "tone": "proto"}, + {"id": "c313-worker", "label": "Python Worker Process (proxyV4/worker.py)", "parentId": None, "tone": "python"}, + {"id": "c313-proxy", "label": "Layer 1: proxy_worker (Transport Shell)", "parentId": "c313-worker", "tone": "transport"}, + {"id": "c313-rt", "label": "Layer 2: Pluggable Runtime Engine (runtimes/)", "parentId": "c313-worker", "tone": "runtime"}, + {"id": "c313-eco", "label": "PyPI SDK, Extensions & Custom Runtimes", "parentId": None, "tone": "lib"}, + ], + "nodes": [ + { + "id": "n313-proto", + "repo": "language-worker-protobuf", + "containerId": "c313-host", + "label": "Host + FunctionRpc.proto", + "badge": "EventStream Server", + "role": "Host spawns `proxyV4/worker.py` (`python -m proxy_worker`) for Python 3.13/3.14.", + "file": "src/proto/FunctionRpc.proto", + "line": 1, + "details": [ + "Identical wire contract; `worker_status_request` answered directly by `proxy_worker`.", + ], + }, + { + "id": "n313-grpc", + "repo": "python-worker", + "containerId": "c313-proxy", + "label": "start_worker + gRPC Pump", + "badge": "uvloop + grpc_local-thread", + "role": "Locks worker deps first, enables `uvloop`, opens `grpcio` stream, and switches console -> `RpcLog`.", + "file": "workers/proxy_worker/dispatcher.py", + "line": sym_line(proxy_disp, "__poll_grpc", 279), + "details": [ + "4-tier `invocation_id` tracking: global -> `ContextEnabledTask` -> `invocation_id_cv` -> thread-local.", + ], + }, + { + "id": "n313-router", + "repo": "python-worker", + "containerId": "c313-proxy", + "label": "Proxy Router & WorkerRequest", + "badge": "reload_library_worker()", + "role": "Promotes customer deps, dynamically selects V2 / V1 / Agent runtime, and passes `WorkerRequest` + `protos`.", + "file": "workers/proxy_worker/dispatcher.py", + "line": sym_line(proxy_disp, "_dispatch_grpc_request", 329), + "details": [ + "Injects `properties={'protos': protos, 'host': host}` on Init & EnvironmentReload so runtimes stay protobuf-version agnostic.", + ], + }, + { + "id": "n313-rtv2", + "repo": "python-worker", + "containerId": "c313-rt", + "label": "Runtime V2 (azure_functions_runtime)", + "badge": "runtimes/v2/handle_event.py", + "role": "PyPI V2 engine: decorator indexing, `Datum` bindings, HTTP v2 streaming, OpenTelemetry, threadpool.", + "file": "runtimes/v2/azure_functions_runtime/handle_event.py", + "line": sym_line(rt_v2_event, "worker_init_request", 58), + "details": [ + "Selected when `function_app.py` (or configured script file) exists in app directory.", + ], + }, + { + "id": "n313-rtv1", + "repo": "python-worker", + "containerId": "c313-rt", + "label": "Runtime V1 (azure_functions_runtime_v1)", + "badge": "runtimes/v1/handle_event.py", + "role": "PyPI V1 engine supporting classic `function.json` apps across the exact same 5-verb contract.", + "file": "runtimes/v1/azure_functions_runtime_v1/handle_event.py", + "line": sym_line(extracted["runtimes/v1/azure_functions_runtime_v1/handle_event.py"], "worker_init_request", 45), + "details": [ + "Selected automatically when `function_app.py` is not present.", + ], + }, + { + "id": "n313-ext", + "repo": "python-extensions", + "containerId": "c313-eco", + "label": "Agent Runtimes, WebApp & SdkType", + "badge": "runtime.py | web.py | sdkType.py", + "role": "Custom `azurefunctions.runtimes` entry points (`PYTHON_ENABLE_AGENT_RUNTIME`), FastAPI HTTP v2, and SDK bindings.", + "file": "azurefunctions-extensions-base/azurefunctions/extensions/base/runtime.py", + "line": 1, + "details": [ + "`RuntimeTrackerMeta` + `entry_points(group='azurefunctions.runtimes')` can replace V1/V2 engine entirely!", + ], + }, + { + "id": "n313-lib", + "repo": "python-library", + "containerId": "c313-eco", + "label": "azure-functions SDK & User App", + "badge": "Decorators + Datum + Handler", + "role": "Customer `function_app.py` + `azure-functions` SDK types invoked on asyncio loop or threadpool.", + "file": "azure/functions/__init__.py", + "line": 1, + "details": [ + "Customer can pin newer `azure-functions` and `azure-functions-runtime` in `requirements.txt`.", + ], + }, + ], + "edges": [ + {"id": "e313-1", "source": "n313-proto", "target": "n313-grpc", "kind": "grpc-wire", "label": "EventStream (grpcio)"}, + {"id": "e313-2", "source": "n313-grpc", "target": "n313-router", "kind": "thread-queue", "label": "ContextEnabledTask / Queue"}, + {"id": "e313-3", "source": "n313-router", "target": "n313-rtv2", "kind": "inproc-contract", "label": "WorkerRequest (V2)"}, + {"id": "e313-4", "source": "n313-router", "target": "n313-rtv1", "kind": "inproc-contract", "label": "WorkerRequest (V1)", "dashed": True}, + {"id": "e313-5", "source": "n313-router", "target": "n313-ext", "kind": "syspath-mutation", "label": "Agent Runtime EntryPoint", "dashed": True}, + {"id": "e313-6", "source": "n313-rtv2", "target": "n313-ext", "kind": "pyo3-control", "label": "WebApp HTTP-v2 / SdkType"}, + {"id": "e313-7", "source": "n313-rtv2", "target": "n313-lib", "kind": "exec-dispatch", "label": "Datum decode + invoke"}, + {"id": "e313-8", "source": "n313-rtv1", "target": "n313-lib", "kind": "exec-dispatch", "label": "Datum decode + invoke", "dashed": True}, + ], + "flows": [ + { + "id": "init", + "title": "Cold Start, Runtime Discovery & protos Injection", + "summary": "Proxy switches to customer deps, resolves Agent Runtime vs V2 vs V1, and injects its `protos` module once.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "router", "label": "proxy_worker"}, + {"id": "ext", "label": "python-extensions"}, + {"id": "rt", "label": "Runtime v2/v1"}, + ], + "steps": [ + {"from": "host", "to": "router", "message": "StartStream + WorkerInitRequest(dir)", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["e313-1", "e313-2"], "payload": "uvloop + grpc_local-thread active"}, + {"from": "router", "to": "ext", "message": "Check PYTHON_ENABLE_AGENT_RUNTIME entry_points", "activeNodes": ["n313-router", "n313-ext"], "activeEdges": ["e313-5"], "payload": "group='azurefunctions.runtimes'"}, + {"from": "router", "to": "rt", "message": "Import V2/V1 + await worker_init_request(WorkerRequest)", "activeNodes": ["n313-router", "n313-rtv2"], "activeEdges": ["e313-3"], "payload": "properties={'protos': protos, 'host': host}"}, + {"from": "rt", "to": "host", "message": "Cache protos, index app, return WorkerInitResponse", "isReturn": True, "activeNodes": ["n313-rtv2", "n313-lib", "n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["e313-7", "e313-3", "e313-2", "e313-1"], "payload": "Capabilities negotiated"}, + ], + }, + { + "id": "invoke", + "title": "Standard & HTTP-v2 Invocation", + "summary": "Proxy stamps invocation_id and delegates `WorkerRequest` to runtime v2 (`python-library` + `python-extensions`).", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "router", "label": "proxy_worker"}, + {"id": "rt", "label": "Runtime v2"}, + {"id": "lib", "label": "SDK / Extensions / App"}, + ], + "steps": [ + {"from": "host", "to": "router", "message": "InvocationRequest(invocation_id, function_id)", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["e313-1", "e313-2"], "payload": "ContextEnabledTask stamps invocation_id"}, + {"from": "router", "to": "rt", "message": "await _library_worker.invocation_request(req)", "activeNodes": ["n313-router", "n313-rtv2"], "activeEdges": ["e313-3"], "payload": "Uses cached protos"}, + {"from": "rt", "to": "lib", "message": "Decode Datum / SdkType / WebApp + invoke user fn", "activeNodes": ["n313-rtv2", "n313-ext", "n313-lib"], "activeEdges": ["e313-6", "e313-7"], "payload": "Sync threadpool or async loop"}, + {"from": "lib", "to": "host", "message": "InvocationResponse -> _grpc_resp_queue", "isReturn": True, "activeNodes": ["n313-lib", "n313-rtv2", "n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["e313-7", "e313-3", "e313-2", "e313-1"], "payload": "StreamingMessage sent over grpcio"}, + ], + }, + ], + }, + # ===================================================================== + # ERA 3: Python 3.15+ (R2P2 — Rust to Python Proxy) + # ===================================================================== + { + "id": "py315_r2p2", + "shortTitle": "Python 3.15+ (R2P2)", + "title": "R2P2 — Rust Transport + Embedded CPython via PyO3 (3.15+)", + "pythonVersions": "3.15+ (GIL Build)", + "packagePath": "workers/r2p2 + runtimes/{v1,v2}", + "launcherPath": "workers/r2p2 (compiled native Rust binary)", + "status": "Next-Gen Architecture (Python 3.15+)", + "tagline": "Native Rust (`tonic` + `prost` + `tokio`) embedding CPython via `PyO3` — zero Python `grpcio`/`protobuf`, +122% req/s.", + "summary": ( + "R2P2 replaces the Python proxy worker with a native Rust binary that compiles " + "`FunctionRpc.proto` via `prost`/`tonic` and embeds `libpython` (GIL) via `PyO3`. " + "Standard `python-library` `Datum` invocations take a **Zero-Protobuf Hot Path** " + "(`convert.rs` tuples -> `native_invocation.py`), while control-plane and " + "`python-extensions` (`SdkType` deferred / `WebApp` HTTP-v2) calls route through " + "`control.rs` + `protos_adapter.py`." + ), + "tradeoffs": [ + {"kind": "strength", "text": "Day-1 Python 3.15+ support: zero wait on `grpcio` or `google.protobuf` C-extension wheels."}, + {"kind": "strength", "text": "+122% throughput (754 vs 339 req/s, p50 64ms vs 145ms) on GIL build via native `tonic`/`prost` + PyO3 datum tuples."}, + {"kind": "strength", "text": "Complete transport isolation: zero `google.protobuf` or `grpcio` loaded in `sys.modules`."}, + {"kind": "pain", "text": "Dual FFI paths (`invoke_native` hot path vs `handle_invocation_control` for HTTP-v2 / `SdkType` extensions)."}, + ], + "erasDiff": { + "transport": "Native Rust: tonic + prost + tokio multi-thread + BytesCodec", + "boundary": "PyO3 FFI (GIL): Hot `(name, Datum)` tuples + Control `PyDict` + `LogSink`", + "runtimeLoading": "Unchanged `runtimes/v2,v1` fed `protos_adapter.py` + `native_invocation.py`", + "hotPathOverhead": "Rust prost -> PyO3 tuple -> native_invocation.py -> python-library Datum (0 protobuf)", + }, + "containers": [ + {"id": "c315-host", "label": "Host & Wire Contract", "parentId": None, "tone": "proto"}, + {"id": "c315-proc", "label": "R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)", "parentId": None, "tone": "rust-proc"}, + {"id": "c315-rust", "label": "Layer 1: Native Rust Shell (workers/r2p2/src)", "parentId": "c315-proc", "tone": "rust"}, + {"id": "c315-ffi", "label": "PyO3 FFI Thin Waist (GIL)", "parentId": "c315-proc", "tone": "bridge"}, + {"id": "c315-py", "label": "Layer 2: Embedded Python Bridge & Runtime", "parentId": "c315-proc", "tone": "runtime"}, + {"id": "c315-eco", "label": "SDK, Extensions & Customer Function App", "parentId": None, "tone": "lib"}, + ], + "nodes": [ + { + "id": "n315-proto", + "repo": "language-worker-protobuf", + "containerId": "c315-host", + "label": "Host + FunctionRpc.proto", + "badge": "prost-build (build.rs)", + "role": "Exact same Host gRPC contract, compiled at Rust build time into native `prost` structs (`pb.rs`).", + "file": "src/proto/FunctionRpc.proto", + "line": 1, + "details": [ + "Vendored in `workers/r2p2/proto/` and compiled by `build.rs` with zero system `protoc` dependency.", + ], + }, + { + "id": "n315-tonic", + "repo": "python-worker", + "containerId": "c315-rust", + "label": "Tokio + Tonic EventStream", + "badge": "main.rs + BytesCodec", + "role": "Multi-thread tokio pump dialing Host (`TCP_NODELAY`), spawning concurrent tasks, and draining `mpsc`.", + "file": "workers/r2p2/src/main.rs", + "line": sym_line(r2p2_main, "main", 230), + "details": [ + "Shared lock-free `mpsc::unbounded_channel` carries both responses and Python `LogSink` frames.", + ], + }, + { + "id": "n315-router", + "repo": "python-worker", + "containerId": "c315-rust", + "label": "Prost Router & Status Fast-Path", + "badge": "process_message()", + "role": "Answers `WorkerStatus` in pure Rust (0 GIL) and splits Hot vs Control invocations via `control_path_cache`.", + "file": "workers/r2p2/src/main.rs", + "line": sym_line(r2p2_main, "process_message", 193), + "details": [ + "Memoizes `needs_control_path(function_id)` once per function after `FunctionLoad`.", + ], + }, + { + "id": "n315-hot-ffi", + "repo": "python-worker", + "containerId": "c315-ffi", + "label": "Hot FFI: bridge::invoke + convert.rs", + "badge": "PyO3 Datum Tuples", + "role": "Translates prost `TypedData` directly into `(tag, value)` Python tuples across `Python::attach`.", + "file": "workers/r2p2/src/bridge.rs", + "line": sym_line(r2p2_bridge_rs, "invoke", 261), + "details": [ + "Eliminates all Python protobuf object allocation on standard HTTP/Queue/Timer/Blob invocations.", + ], + }, + { + "id": "n315-ctrl-ffi", + "repo": "python-worker", + "containerId": "c315-ffi", + "label": "Control FFI: bridge::handle + LogSink", + "badge": "control.rs + #[pyclass] LogSink", + "role": "Converts Init/Load/Reload & HTTP-v2/deferred requests to `PyDict`s; exposes non-blocking Rust `LogSink`.", + "file": "workers/r2p2/src/bridge.rs", + "line": sym_line(r2p2_bridge_rs, "handle", 107), + "details": [ + "`LogSink.emit_log(dict)` encodes `RpcLog` in Rust so Python logging never touches protobuf or blocks.", + ], + }, + { + "id": "n315-bridge-py", + "repo": "python-worker", + "containerId": "c315-py", + "label": "bridge.py & protos_adapter.py", + "badge": "Protobuf-Free + Namespace Dep Iso", + "role": "Pure-Python `protos` stand-in + `_clear_modules_from_path` namespace-safe customer dependency promotion.", + "file": "workers/r2p2/bridge/bridge.py", + "line": sym_line(r2p2_bridge_py, "_clear_modules_from_path", 328), + "details": [ + "Preserves shared namespace parents while evicting worker-backed packages and stale parent attributes.", + ], + }, + { + "id": "n315-native-py", + "repo": "python-worker", + "containerId": "c315-py", + "label": "native_invocation.py (v2 & v1)", + "badge": "Hot Datum Execution Seam", + "role": "Decodes Rust datum tuples directly into `python-library` bindings using `_InvocShim` and runs user code.", + "file": "runtimes/v2/azure_functions_runtime/native_invocation.py", + "line": sym_line(rt_v2_native, "_decode_inputs", 82), + "details": [ + "Supports sync (`ThreadPoolExecutor`) & async (`asyncio`), `Context`, `Out[T]`, and W3C OpenTelemetry.", + ], + }, + { + "id": "n315-rt", + "repo": "python-worker", + "containerId": "c315-py", + "label": "Runtime handle_event + http_v2", + "badge": "Control & Streaming Seam", + "role": "Handles `WorkerInit`, `FunctionLoad`, `EnvironmentReload`, and fallback invocations for Extensions.", + "file": "runtimes/v2/azure_functions_runtime/handle_event.py", + "line": sym_line(rt_v2_event, "worker_init_request", 58), + "details": [ + "Operates seamlessly against `protos_adapter.py` without code duplication.", + ], + }, + { + "id": "n315-lib", + "repo": "python-library", + "containerId": "c315-eco", + "label": "azure-functions (Datum + Decorators)", + "badge": "Hot-Path Compatible", + "role": "Standard V2/V1 triggers & bindings (`HttpRequest`, `QueueMessage`, `TimerRequest`, `Out[T]`) + User App.", + "file": "azure/functions/meta.py", + "line": 1, + "details": [ + "Runs at full native speed via `native_invocation.py` with zero protobuf dependency!", + ], + }, + { + "id": "n315-ext", + "repo": "python-extensions", + "containerId": "c315-eco", + "label": "extensions (HTTP-v2 FastAPI & SdkType)", + "badge": "requires_control_path = true", + "role": "`WebApp` ASGI streaming (`http-fastapi`) and `SdkType` deferred bindings (`BlobClient`, `ServiceBus`).", + "file": "azurefunctions-extensions-base/azurefunctions/extensions/base/web.py", + "line": 1, + "details": [ + "Detected by `bridge.requires_control_path(fn_id)` and routed cleanly via `handle_invocation_control`.", + ], + }, + ], + "edges": [ + {"id": "e315-1", "source": "n315-proto", "target": "n315-tonic", "kind": "grpc-wire", "label": "EventStream (tonic + BytesCodec)"}, + {"id": "e315-2", "source": "n315-tonic", "target": "n315-router", "kind": "thread-queue", "label": "tokio::spawn + mpsc"}, + {"id": "e315-3", "source": "n315-router", "target": "n315-hot-ffi", "kind": "pyo3-hot", "label": "Hot Path: bridge::invoke()"}, + {"id": "e315-4", "source": "n315-router", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "Control / Extensions Path"}, + {"id": "e315-5", "source": "n315-hot-ffi", "target": "n315-native-py", "kind": "pyo3-hot", "label": "invoke_native(datum tuples)"}, + {"id": "e315-6", "source": "n315-ctrl-ffi", "target": "n315-bridge-py", "kind": "pyo3-control", "label": "handle_control(PyDict)"}, + {"id": "e315-7", "source": "n315-bridge-py", "target": "n315-rt", "kind": "inproc-contract", "label": "WorkerRequest + protos_adapter"}, + {"id": "e315-8", "source": "n315-native-py", "target": "n315-lib", "kind": "exec-dispatch", "label": "Datum decode + invoke"}, + {"id": "e315-9", "source": "n315-rt", "target": "n315-ext", "kind": "pyo3-control", "label": "WebApp stream / SdkType", "dashed": True}, + {"id": "e315-10", "source": "n315-rt", "target": "n315-lib", "kind": "inproc-contract", "label": "index @app decorators", "dashed": True}, + ], + "flows": [ + { + "id": "hot_invoke", + "title": "Hot Native Invocation (prost -> PyO3 Tuple -> python-library Datum)", + "summary": "Standard triggers bypass Python protobuf completely: prost decodes TypedData into PyO3 datum tuples for native_invocation.py.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "rust", "label": "R2P2 (tonic+prost)"}, + {"id": "ffi", "label": "bridge::invoke"}, + {"id": "native", "label": "native_invocation.py"}, + {"id": "lib", "label": "python-library + App"}, + ], + "steps": [ + {"from": "host", "to": "rust", "message": "StreamingMessage(InvocationRequest) bytes", "activeNodes": ["n315-proto", "n315-tonic", "n315-router"], "activeEdges": ["e315-1", "e315-2"], "payload": "needs_control_path(fn_id) == false (cached)"}, + {"from": "rust", "to": "ffi", "message": "prost decode -> convert::typed_data_to_tuple", "activeNodes": ["n315-router", "n315-hot-ffi"], "activeEdges": ["e315-3"], "payload": "Rust TypedData -> compact PyTuple"}, + {"from": "ffi", "to": "native", "message": "Python::attach -> invoke_native(fn_id, inputs, meta)", "activeNodes": ["n315-hot-ffi", "n315-native-py"], "activeEdges": ["e315-5"], "payload": "Zero Python protobuf objects created"}, + {"from": "native", "to": "lib", "message": "Decode azure.functions Datum + execute handler", "activeNodes": ["n315-native-py", "n315-lib"], "activeEdges": ["e315-8"], "payload": "HttpRequest / QueueMessage -> User fn"}, + {"from": "lib", "to": "host", "message": "Return tuple -> prost encode -> mpsc tx", "isReturn": True, "activeNodes": ["n315-lib", "n315-native-py", "n315-hot-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["e315-8", "e315-5", "e315-3", "e315-2", "e315-1"], "payload": "+122% throughput vs Python proxy"}, + ], + }, + { + "id": "ext_invoke", + "title": "Extensions Path (HTTP-v2 FastAPI Streaming & SdkType Bindings)", + "summary": "Functions using `python-extensions` (`WebApp` HTTP streaming or `SdkType` BlobClient) route through `handle_invocation_control`.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "rust", "label": "R2P2 Router"}, + {"id": "ctrl", "label": "bridge::handle + bridge.py"}, + {"id": "rt", "label": "Runtime + http_v2"}, + {"id": "ext", "label": "python-extensions"}, + ], + "steps": [ + {"from": "host", "to": "rust", "message": "InvocationRequest(function_id)", "activeNodes": ["n315-proto", "n315-tonic", "n315-router"], "activeEdges": ["e315-1", "e315-2"], "payload": "needs_control_path(fn_id) == true"}, + {"from": "rust", "to": "ctrl", "message": "control::invocation_request_to_py(PyDict)", "activeNodes": ["n315-router", "n315-ctrl-ffi", "n315-bridge-py"], "activeEdges": ["e315-4", "e315-6"], "payload": "protos_adapter wraps dict (still 0 google.protobuf)"}, + {"from": "ctrl", "to": "rt", "message": "await handle_event.invocation_request()", "activeNodes": ["n315-bridge-py", "n315-rt"], "activeEdges": ["e315-7"], "payload": "Standard runtime invocation path"}, + {"from": "rt", "to": "ext", "message": "Stream ASGI FastAPI chunks / hydrate SdkType", "activeNodes": ["n315-rt", "n315-ext"], "activeEdges": ["e315-9"], "payload": "azurefunctions.extensions.base"}, + {"from": "ext", "to": "host", "message": "Dict -> py_to_invocation_response -> prost bytes", "isReturn": True, "activeNodes": ["n315-ext", "n315-rt", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["e315-9", "e315-7", "e315-6", "e315-4", "e315-1"], "payload": "StreamingMessage(InvocationResponse)"}, + ], + }, + { + "id": "env_reload", + "title": "Placeholder Specialization & Namespace-Safe Dep Promotion", + "summary": "Evicts worker-backed modules while keeping shared `azure.*` / `azurefunctions.*` namespace parents attached.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "ctrl", "label": "bridge::handle"}, + {"id": "bpy", "label": "bridge.py Dep Engine"}, + {"id": "rt", "label": "Runtime + SDK"}, + ], + "steps": [ + {"from": "host", "to": "ctrl", "message": "FunctionEnvironmentReloadRequest(app_dir)", "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi"], "activeEdges": ["e315-1", "e315-2", "e315-4"], "payload": "Warm placeholder specializes to customer app"}, + {"from": "ctrl", "to": "bpy", "message": "_clear_modules_from_path + _prioritize_customer_deps", "activeNodes": ["n315-ctrl-ffi", "n315-bridge-py"], "activeEdges": ["e315-6"], "payload": "Retains namespace parents (`azure`, `azurefunctions`); evicts stale worker children"}, + {"from": "bpy", "to": "rt", "message": "Import customer-pinned SDK/runtime & index app", "activeNodes": ["n315-bridge-py", "n315-rt", "n315-lib"], "activeEdges": ["e315-7", "e315-10"], "payload": "sys.path: [customer_pkgs, worker_deps, app_dir]"}, + {"from": "rt", "to": "host", "message": "FunctionEnvironmentReloadResponse", "isReturn": True, "activeNodes": ["n315-rt", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["e315-7", "e315-6", "e315-4", "e315-1"], "payload": "Specialized & ready"}, + ], + }, + ], + }, + ] + + phases_by_era = { + "py312": get_py312_phases(), + "py313_314": get_py313_phases(), + "py315_r2p2": get_py315_phases(), + } + + for era in eras: + for node in era["nodes"]: + fmeta: dict[str, Any] = {} + if node["repo"] == "python-worker": + fmeta = extracted.get(node["file"], {}) + elif node["repo"] in repo_dirs and node["file"].endswith(".py"): + fmeta = extract_python_file(node["file"], repo_dirs[node["repo"]]) + node["fileLines"] = fmeta.get("lines", 0) + node["extractedSymbols"] = fmeta.get("symbols", [])[:10] + node["routerCases"] = fmeta.get("routerCases", []) + node["pyo3Calls"] = fmeta.get("pyo3Calls", []) + node["githubUrl"] = resolve_github_url(node, repo_refs) + era["topologyMermaid"] = compile_topology_mermaid(era) + for flow in era["flows"]: + flow["sequenceMermaid"] = compile_sequence_mermaid(flow) + + era_phases = phases_by_era.get(era["id"], []) + for ph in era_phases: + ph["topologyMermaid"] = compile_topology_mermaid(era, ph["topologyEdges"]) + ph["sequenceMermaid"] = compile_sequence_mermaid(ph) + era["phases"] = era_phases + + structural_payload = json.dumps( + { + "repos": list(REPOS.keys()), + "phases": [p["id"] for p in PHASE_CATALOG], + "inventories": {k: [x["path"] for x in v] for k, v in inventories.items()}, + "protoMessages": proto_messages, + "symbols": {k: [s["name"] for s in v.get("symbols", [])] for k, v in sorted(extracted.items())}, + }, + sort_keys=True, + ) + fingerprint = hashlib.sha256(structural_payload.encode("utf-8")).hexdigest()[:16] + + graph = { + "schemaVersion": "1.2.0", + "generatedAt": datetime.now(timezone.utc).isoformat(), + "structuralFingerprint": fingerprint, + "git": git, + "repoRefs": repo_refs, + "repos": REPOS, + "phaseCatalog": PHASE_CATALOG, + "protoStreamingMessages": proto_messages, + "packageInventories": inventories, + "eras": eras, + } + validate_graph(graph) + return graph + + +def validate_graph(graph: dict[str, Any]) -> None: + expected_phase_ids = [p["id"] for p in PHASE_CATALOG] + for era in graph["eras"]: + node_ids = {n["id"] for n in era["nodes"]} + container_ids = {c["id"] for c in era["containers"]} + for n in era["nodes"]: + if n["containerId"] not in container_ids: + raise ValueError(f"Node {n['id']} references missing container {n['containerId']}") + if n["repo"] not in REPOS: + raise ValueError(f"Node {n['id']} references unknown repo {n['repo']}") + if n["repo"] == "python-worker" and not (REPO_ROOT / n["file"]).exists(): + raise ValueError(f"Node {n['id']} points to missing python-worker file {n['file']}") + for e in era["edges"]: + if e["source"] not in node_ids or e["target"] not in node_ids: + raise ValueError(f"Edge {e['id']} in {era['id']} has invalid endpoint") + + if era["id"] in ("py312", "py313_314", "py315_r2p2"): + got_phases = [p["id"] for p in era.get("phases", [])] + if got_phases != expected_phase_ids: + raise ValueError(f"Era {era['id']} phases {got_phases} != {expected_phase_ids}") + for ph in era["phases"]: + ph_edge_ids = {e["id"] for e in ph["topologyEdges"]} + for e in ph["topologyEdges"]: + if e["source"] not in node_ids or e["target"] not in node_ids: + raise ValueError(f"Phase edge {e['id']} in {era['id']}.{ph['id']} invalid") + for nid in ph["activeNodes"]: + if nid not in node_ids: + raise ValueError(f"Phase {era['id']}.{ph['id']} activeNode {nid} missing") + actor_ids = {a["id"] for a in ph["actors"]} + for st in ph["steps"]: + if st["from"] not in actor_ids or st["to"] not in actor_ids: + raise ValueError(f"Step actor missing in {era['id']}.{ph['id']}") + for nid in st["activeNodes"]: + if nid not in node_ids: + raise ValueError(f"Step activeNode {nid} missing in {era['id']}.{ph['id']}") + for eid in st["activeEdges"]: + if eid not in ph_edge_ids: + raise ValueError(f"Step activeEdge {eid} missing in {era['id']}.{ph['id']}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Extract multi-repo architecture graph.") + parser.add_argument( + "--sibling-root", + type=Path, + default=None, + help="Optional directory containing python-library, python-extensions, language-worker-protobuf checkouts.", + ) + args = parser.parse_args() + + JSON_OUT.parent.mkdir(parents=True, exist_ok=True) + graph = build_graph(sibling_root=args.sibling_root) + raw_json = json.dumps(graph, indent=2) + JSON_OUT.write_text(raw_json + "\n", encoding="utf-8") + JS_OUT.write_text(f"window.__ARCH_GRAPH__ = {raw_json};\n", encoding="utf-8") + print( + f"[extract_arch] Generated {JSON_OUT.relative_to(REPO_ROOT)} & {JS_OUT.relative_to(REPO_ROOT)} " + f"(repos={len(graph['repos'])}, eras={len(graph['eras'])}, fingerprint={graph['structuralFingerprint']})" + ) + + +if __name__ == "__main__": + main() diff --git a/eng/docs/phases_spec.py b/eng/docs/phases_spec.py new file mode 100644 index 000000000..e0c27255f --- /dev/null +++ b/eng/docs/phases_spec.py @@ -0,0 +1,792 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +""" +Canonical 7-Phase Lifecycle Specifications across all 3 Python Worker Generations: + 1. init — Worker Initialization (StartStream + WorkerInitRequest) + 2. specialization — Worker Specialization (FunctionEnvironmentReloadRequest, Consumption only) + 3. metadata — Worker Metadata & Indexing (FunctionsMetadataRequest) + 4. function_load — Worker Functions Load (FunctionLoadRequest, V1 primary) + 5. invocation — Worker Invocation (InvocationRequest) + 6. worker_status — Worker Status Health/Scale Check (WorkerStatusRequest) + 7. worker_logs — Worker Logs Reverse Stream (RpcLog: Worker -> Host) + +Each phase defines its own phase-specific Compound Topology edges (`topologyEdges`, +`activeNodes`, `bypassedNote`) AND its own Multi-Actor Sequence Diagram (`actors`, `steps`). +""" + +from __future__ import annotations +from typing import Any + + +PHASE_CATALOG = [ + {"id": "init", "num": "1", "short": "Worker Init", "proto": "StartStream + WorkerInitRequest"}, + {"id": "specialization", "num": "2", "short": "Specialization", "proto": "FunctionEnvironmentReloadRequest"}, + {"id": "metadata", "num": "3", "short": "Metadata & Indexing", "proto": "FunctionsMetadataRequest"}, + {"id": "function_load", "num": "4", "short": "Functions Load (V1)", "proto": "FunctionLoadRequest"}, + {"id": "invocation", "num": "5", "short": "Invocation", "proto": "InvocationRequest"}, + {"id": "worker_status", "num": "6", "short": "Worker Status", "proto": "WorkerStatusRequest"}, + {"id": "worker_logs", "num": "7", "short": "Worker Logs", "proto": "RpcLog (Worker → Host)"}, +] + + +def get_py312_phases() -> list[dict[str, Any]]: + return [ + # 1. Worker Init + { + "id": "init", + "title": "1. Worker Initialization Flow", + "proto": "StartStream → WorkerInitRequest → WorkerInitResponse", + "scope": "All SKUs (Cold Start)", + "summary": ( + "Host spawns `prodV4/worker.py`, which boots `azure_functions_worker` with worker " + "dependencies isolated first (`grpcio` + `protobuf`). Once `EventStream` connects and sends " + "`StartStream`, `WorkerInitRequest` negotiates capabilities (`SHARED_MEMORY`, `RAW_HTTP_BODY_BYTES`, " + "`WORKER_STATUS`), initializes OTel/AppInsights, promotes customer `.python_packages` (on Dedicated/EP), " + "loads the binding registry, and optionally pre-indexes if `PYTHON_ENABLE_INIT_INDEXING=1`." + ), + "bypassedNote": "In Linux Consumption placeholder mode, customer packages are NOT mounted yet during Init (deferred to Phase 2 Specialization).", + "activeNodes": ["n312-proto", "n312-boot", "n312-grpc", "n312-disp", "n312-dep", "n312-bind"], + "topologyEdges": [ + {"id": "p312-i1", "source": "n312-proto", "target": "n312-boot", "kind": "syspath-mutation", "label": "1. spawn prodV4/worker.py"}, + {"id": "p312-i2", "source": "n312-boot", "target": "n312-grpc", "kind": "inproc-contract", "label": "2. use_worker_deps + connect()"}, + {"id": "p312-i3", "source": "n312-grpc", "target": "n312-proto", "kind": "grpc-wire", "label": "3. StartStream(worker_id)"}, + {"id": "p312-i4", "source": "n312-proto", "target": "n312-grpc", "kind": "grpc-wire", "label": "4. WorkerInitRequest"}, + {"id": "p312-i5", "source": "n312-grpc", "target": "n312-disp", "kind": "thread-queue", "label": "5. call_soon_threadsafe"}, + {"id": "p312-i6", "source": "n312-disp", "target": "n312-dep", "kind": "syspath-mutation", "label": "6. prioritize_customer_deps()"}, + {"id": "p312-i7", "source": "n312-disp", "target": "n312-bind", "kind": "inproc-contract", "label": "7. load_binding_registry()"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "boot", "label": "prodV4 + main.py"}, + {"id": "grpc", "label": "gRPC Thread"}, + {"id": "disp", "label": "Monolith Dispatcher"}, + {"id": "dep", "label": "DependencyManager"}, + ], + "steps": [ + {"from": "host", "to": "boot", "message": "Spawn python prodV4/worker.py --host --port --workerId", "payload": "DependencyManager.initialize() + use_worker_dependencies()", "activeNodes": ["n312-proto", "n312-boot"], "activeEdges": ["p312-i1"]}, + {"from": "boot", "to": "grpc", "message": "Dispatcher.connect() opens insecure grpcio channel", "payload": "Starts __poll_grpc background thread", "activeNodes": ["n312-boot", "n312-grpc"], "activeEdges": ["p312-i2"]}, + {"from": "grpc", "to": "host", "message": "Send StartStream(worker_id) + detach console logger", "isReturn": True, "payload": "Switches root logger to AsyncLoggingHandler", "activeNodes": ["n312-grpc", "n312-proto"], "activeEdges": ["p312-i3"]}, + {"from": "host", "to": "disp", "message": "WorkerInitRequest(capabilities, function_app_directory)", "payload": "Posted from gRPC thread to asyncio loop", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["p312-i4", "p312-i5"]}, + {"from": "disp", "to": "dep", "message": "should_load_cx_dependencies() -> prioritize_customer_deps()", "payload": "Dedicated: promotes customer .python_packages; Consumption: stays in placeholder", "activeNodes": ["n312-disp", "n312-dep"], "activeEdges": ["p312-i6"]}, + {"from": "disp", "to": "disp", "message": "Configure OTel/AppInsights + bindings.load_binding_registry()", "payload": "Optional load_function_metadata if PYTHON_ENABLE_INIT_INDEXING=1", "activeNodes": ["n312-disp", "n312-bind"], "activeEdges": ["p312-i7"]}, + {"from": "disp", "to": "host", "message": "WorkerInitResponse(capabilities, worker_metadata)", "isReturn": True, "payload": "SHARED_MEMORY_DATA_TRANSFER, WORKER_STATUS, RAW_HTTP_BODY_BYTES", "activeNodes": ["n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["p312-i5", "p312-i3"]}, + ], + }, + # 2. Worker Specialization + { + "id": "specialization", + "title": "2. Worker Specialization Flow (Consumption Placeholder)", + "proto": "FunctionEnvironmentReloadRequest → FunctionEnvironmentReloadResponse", + "scope": "Linux / Flex Consumption Only (Placeholder Mode)", + "summary": ( + "Warm standby containers boot before any customer app is assigned. When specialized, " + "the Host mounts the customer package and sends `FunctionEnvironmentReloadRequest` carrying " + "`function_app_directory` and `environment_variables`. The monolith clears `os.environ`, calls " + "`time.tzset()`, clears `sys.path_importer_cache`, promotes customer `.python_packages`, recreates " + "`_sync_call_tp` (`PYTHON_THREADPOOL_THREAD_COUNT`), and changes `cwd`." + ), + "bypassedNote": "Never sent on Dedicated/Premium cold boots or worker restarts (those go straight through WorkerInit).", + "activeNodes": ["n312-proto", "n312-grpc", "n312-disp", "n312-dep", "n312-load", "n312-bind", "n312-sdk"], + "topologyEdges": [ + {"id": "p312-s1", "source": "n312-proto", "target": "n312-grpc", "kind": "grpc-wire", "label": "1. FunctionEnvironmentReloadRequest"}, + {"id": "p312-s2", "source": "n312-grpc", "target": "n312-disp", "kind": "thread-queue", "label": "2. _dispatch_grpc_request"}, + {"id": "p312-s3", "source": "n312-disp", "target": "n312-dep", "kind": "syspath-mutation", "label": "3. reload os.environ + sys.path"}, + {"id": "p312-s4", "source": "n312-dep", "target": "n312-sdk", "kind": "syspath-mutation", "label": "4. reload customer azure.functions", "dashed": True}, + {"id": "p312-s5", "source": "n312-disp", "target": "n312-load", "kind": "inproc-contract", "label": "5. optional init indexing + http_v2"}, + {"id": "p312-s6", "source": "n312-disp", "target": "n312-bind", "kind": "inproc-contract", "label": "6. recreate _sync_call_tp + bindings"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "disp", "label": "Monolith Dispatcher"}, + {"id": "dep", "label": "DependencyManager"}, + {"id": "sdk", "label": "Customer Packages"}, + ], + "steps": [ + {"from": "host", "to": "disp", "message": "FunctionEnvironmentReloadRequest(app_dir, env_vars)", "payload": "Mounts /home/site/wwwroot into warm container", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["p312-s1", "p312-s2"]}, + {"from": "disp", "to": "disp", "message": "os.environ.clear() + apply env_vars + time.tzset()", "payload": "Recreates _sync_call_tp with customer PYTHON_THREADPOOL_THREAD_COUNT", "activeNodes": ["n312-disp", "n312-bind"], "activeEdges": ["p312-s6"]}, + {"from": "disp", "to": "dep", "message": "sys.path_importer_cache.clear() + prioritize_customer_deps()", "payload": "Evicts placeholder azure.functions so customer version loads", "activeNodes": ["n312-disp", "n312-dep", "n312-sdk"], "activeEdges": ["p312-s3", "p312-s4"]}, + {"from": "disp", "to": "disp", "message": "bindings.load_binding_registry() + os.chdir(app_dir)", "payload": "Initializes http_v2 server if streaming app", "activeNodes": ["n312-disp", "n312-load"], "activeEdges": ["p312-s5"]}, + {"from": "disp", "to": "host", "message": "FunctionEnvironmentReloadResponse(capabilities)", "isReturn": True, "payload": "Warm worker specialized & ready", "activeNodes": ["n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["p312-s2", "p312-s1"]}, + ], + }, + # 3. Metadata & Indexing + { + "id": "metadata", + "title": "3. Worker Metadata & Indexing Flow", + "proto": "FunctionsMetadataRequest → FunctionMetadataResponse", + "scope": "V2 Worker-Indexed Apps (Returns Fallback Flag for V1)", + "summary": ( + "Host asks the worker whether it owns function discovery. In `≤ 3.12`, `load_function_metadata()` " + "checks if `function_app.py` exists: for **V2**, it imports `function_app.py`, walks `@app` " + "decorators and `Blueprint`s (`python-library`) plus `WebApp`/`SdkType` hooks (`python-extensions`), " + "and returns `RawFunctionMetadata` (`use_default_metadata_indexing=False`). For **V1**, it returns " + "`use_default_metadata_indexing=True` so the Host scans `function.json` directories itself." + ), + "bypassedNote": "For V1 apps (`function.json`), no customer Python code is imported during Metadata — Host falls back to disk scanning.", + "activeNodes": ["n312-proto", "n312-grpc", "n312-disp", "n312-load", "n312-sdk", "n312-ext"], + "topologyEdges": [ + {"id": "p312-m1", "source": "n312-proto", "target": "n312-grpc", "kind": "grpc-wire", "label": "1. FunctionsMetadataRequest"}, + {"id": "p312-m2", "source": "n312-grpc", "target": "n312-disp", "kind": "thread-queue", "label": "2. _handle__functions_metadata_request"}, + {"id": "p312-m3", "source": "n312-disp", "target": "n312-load", "kind": "inproc-contract", "label": "3. load_function_metadata()"}, + {"id": "p312-m4", "source": "n312-load", "target": "n312-sdk", "kind": "exec-dispatch", "label": "4a. V2: import function_app & @app"}, + {"id": "p312-m5", "source": "n312-load", "target": "n312-ext", "kind": "pyo3-control", "label": "4b. inspect WebApp / SdkType", "dashed": True}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "disp", "label": "Monolith Dispatcher"}, + {"id": "load", "label": "loader.py"}, + {"id": "sdk", "label": "python-library / App"}, + ], + "steps": [ + {"from": "host", "to": "disp", "message": "FunctionsMetadataRequest(function_app_directory)", "payload": "Queries worker-indexed functions", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["p312-m1", "p312-m2"]}, + {"from": "disp", "to": "load", "message": "Check os.path.exists(function_app.py)", "payload": "PYTHON_SCRIPT_FILE_NAME (default function_app.py)", "activeNodes": ["n312-disp", "n312-load"], "activeEdges": ["p312-m3"]}, + {"from": "load", "to": "sdk", "message": "[V2] index_function_app() -> import function_app.py", "payload": "Extracts FunctionApp.get_functions() + Blueprints + triggers", "activeNodes": ["n312-load", "n312-sdk", "n312-ext"], "activeEdges": ["p312-m4", "p312-m5"]}, + {"from": "load", "to": "disp", "message": "Register FunctionInfo in _functions + build RawFunctionMetadata", "isReturn": True, "payload": "Or None if V1 app", "activeNodes": ["n312-load", "n312-disp"], "activeEdges": ["p312-m3"]}, + {"from": "disp", "to": "host", "message": "FunctionMetadataResponse(results, use_default_metadata_indexing)", "isReturn": True, "payload": "V2: False + metadata list | V1: True (Host scans function.json)", "activeNodes": ["n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["p312-m2", "p312-m1"]}, + ], + }, + # 4. Function Load (V1) + { + "id": "function_load", + "title": "4. Worker Functions Load Flow (V1 Primary)", + "proto": "FunctionLoadRequest → FunctionLoadResponse", + "scope": "V1 Programming Model (`function.json`); V2 Multi-Worker / Ack", + "summary": ( + "After the Host parses `function.json` files for a **V1** app, it sends one `FunctionLoadRequest` " + "per function with `script_file`, `entry_point`, and `bindings`. `loader.load_function()` imports " + "the target Python module (`__init__.py`), validates parameter type annotations against " + "`bindings/`, and stores `FunctionInfo` in `self._functions`. (For V2 single-worker, `_functions` " + "already holds the function from Phase 3, so it fast-acks; for 2nd+ workers in multi-proc mode, " + "it triggers lazy V2 indexing)." + ), + "bypassedNote": "For standard V2 apps, user scripts were already loaded in Phase 3 Metadata, making FunctionLoad a fast registry check.", + "activeNodes": ["n312-proto", "n312-grpc", "n312-disp", "n312-load", "n312-bind", "n312-sdk"], + "topologyEdges": [ + {"id": "p312-l1", "source": "n312-proto", "target": "n312-grpc", "kind": "grpc-wire", "label": "1. FunctionLoadRequest(metadata)"}, + {"id": "p312-l2", "source": "n312-grpc", "target": "n312-disp", "kind": "thread-queue", "label": "2. _handle__function_load_request"}, + {"id": "p312-l3", "source": "n312-disp", "target": "n312-load", "kind": "inproc-contract", "label": "3. loader.load_function(script_file)"}, + {"id": "p312-l4", "source": "n312-load", "target": "n312-sdk", "kind": "exec-dispatch", "label": "4. importlib import __init__.py"}, + {"id": "p312-l5", "source": "n312-disp", "target": "n312-bind", "kind": "inproc-contract", "label": "5. validate binding types & add_function"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "disp", "label": "Monolith Dispatcher"}, + {"id": "load", "label": "loader.py"}, + {"id": "app", "label": "V1 __init__.py"}, + ], + "steps": [ + {"from": "host", "to": "disp", "message": "FunctionLoadRequest(function_id, RpcFunctionMetadata)", "payload": "Carries script_file, entry_point, and function.json bindings", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["p312-l1", "p312-l2"]}, + {"from": "disp", "to": "load", "message": "loader.load_function(name, dir, script_file, entry_point)", "payload": "Only runs when function_id not yet in _functions (V1)", "activeNodes": ["n312-disp", "n312-load"], "activeEdges": ["p312-l3"]}, + {"from": "load", "to": "app", "message": "importlib.import_module() -> resolve main() callable", "payload": "Executes module top-level imports", "activeNodes": ["n312-load", "n312-sdk"], "activeEdges": ["p312-l4"]}, + {"from": "disp", "to": "disp", "message": "_functions.add_function(function_id, func, metadata)", "payload": "Validates Python type hints vsIn/Out binding converters", "activeNodes": ["n312-disp", "n312-bind"], "activeEdges": ["p312-l5"]}, + {"from": "disp", "to": "host", "message": "FunctionLoadResponse(function_id, Success)", "isReturn": True, "payload": "Function ready to invoke", "activeNodes": ["n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["p312-l2", "p312-l1"]}, + ], + }, + # 5. Invocation + { + "id": "invocation", + "title": "5. Worker Invocation Flow", + "proto": "InvocationRequest → InvocationResponse", + "scope": "Every Function Trigger Execution", + "summary": ( + "Host sends `InvocationRequest` with `input_data` (`TypedData` or POSIX/Win shared-memory map) " + "and `trigger_metadata`. `Dispatcher` decodes bindings via `python-library` (`Datum`) or " + "`python-extensions` (`SdkType` / `http_v2`), sets W3C OpenTelemetry context, executes `async def` " + "on the event loop or `def` on `_sync_call_tp`, and marshals `Out[T]` + `$return` back to protobuf." + ), + "bypassedNote": "`prodV4` boot and `loader.py` are idle during steady-state invocations.", + "activeNodes": ["n312-proto", "n312-grpc", "n312-disp", "n312-bind", "n312-sdk", "n312-ext"], + "topologyEdges": [ + {"id": "p312-v1", "source": "n312-proto", "target": "n312-grpc", "kind": "grpc-wire", "label": "1. InvocationRequest"}, + {"id": "p312-v2", "source": "n312-grpc", "target": "n312-disp", "kind": "thread-queue", "label": "2. call_soon_threadsafe"}, + {"id": "p312-v3", "source": "n312-disp", "target": "n312-bind", "kind": "inproc-contract", "label": "3. from_incoming_proto / ShMem"}, + {"id": "p312-v4", "source": "n312-bind", "target": "n312-ext", "kind": "pyo3-control", "label": "4a. http_v2 / SdkType", "dashed": True}, + {"id": "p312-v5", "source": "n312-bind", "target": "n312-sdk", "kind": "exec-dispatch", "label": "4b. await / _sync_call_tp"}, + {"id": "p312-v6", "source": "n312-disp", "target": "n312-grpc", "kind": "thread-queue", "label": "5. _grpc_resp_queue.put"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "disp", "label": "Monolith Dispatcher"}, + {"id": "bind", "label": "Bindings / SharedMem"}, + {"id": "app", "label": "SDK & User Handler"}, + ], + "steps": [ + {"from": "host", "to": "disp", "message": "InvocationRequest(invocation_id, function_id, input_data)", "payload": "google.protobuf StreamingMessage", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["p312-v1", "p312-v2"]}, + {"from": "disp", "to": "bind", "message": "from_incoming_proto() + get_context() + OTel traceparent", "payload": "Reads TypedData or SharedMemoryManager buffer", "activeNodes": ["n312-disp", "n312-bind"], "activeEdges": ["p312-v3"]}, + {"from": "bind", "to": "app", "message": "Execute async coroutine or run_in_executor(_sync_call_tp)", "payload": "Runs ExtensionManager pre/post invocation hooks", "activeNodes": ["n312-bind", "n312-sdk", "n312-ext"], "activeEdges": ["p312-v4", "p312-v5"]}, + {"from": "app", "to": "disp", "message": "to_outgoing_proto($return, Out) + sys.stdout.flush()", "isReturn": True, "payload": "Builds protos.InvocationResponse", "activeNodes": ["n312-sdk", "n312-bind", "n312-disp"], "activeEdges": ["p312-v5", "p312-v3"]}, + {"from": "disp", "to": "host", "message": "Enqueue StreamingMessage(invocation_response)", "isReturn": True, "payload": "Sent over grpcio EventStream", "activeNodes": ["n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["p312-v6", "p312-v1"]}, + ], + }, + # 6. Worker Status + { + "id": "worker_status", + "title": "6. Worker Status Flow (Host Health & Scale Probe)", + "proto": "WorkerStatusRequest → WorkerStatusResponse", + "scope": "Host Scale Controller & Worker Health Monitoring", + "summary": ( + "The Host periodically sends `WorkerStatusRequest` to measure worker round-trip responsiveness " + "for out-of-proc scaling decisions. In `≤ 3.12`, `_handle__worker_status_request` suppresses " + "all logging and returns an empty `WorkerStatusResponse` immediately—**however**, it still " + "hop-schedules onto the single Python `asyncio` event loop and acquires the GIL." + ), + "bypassedNote": "Loader, Bindings, SDK, Extensions, and User Code are completely untouched — pure transport/loop health probe.", + "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], + "topologyEdges": [ + {"id": "p312-st1", "source": "n312-proto", "target": "n312-grpc", "kind": "grpc-wire", "label": "1. WorkerStatusRequest"}, + {"id": "p312-st2", "source": "n312-grpc", "target": "n312-disp", "kind": "thread-queue", "label": "2. call_soon_threadsafe (GIL + loop)"}, + {"id": "p312-st3", "source": "n312-disp", "target": "n312-grpc", "kind": "thread-queue", "label": "3. WorkerStatusResponse (no logs)"}, + {"id": "p312-st4", "source": "n312-grpc", "target": "n312-proto", "kind": "grpc-wire", "label": "4. EventStream reply"}, + ], + "actors": [ + {"id": "host", "label": "Host Scale Controller"}, + {"id": "grpc", "label": "gRPC Thread"}, + {"id": "disp", "label": "Dispatcher (asyncio)"}, + ], + "steps": [ + {"from": "host", "to": "grpc", "message": "WorkerStatusRequest(request_id)", "payload": "Probes worker event-loop latency", "activeNodes": ["n312-proto", "n312-grpc"], "activeEdges": ["p312-st1"]}, + {"from": "grpc", "to": "disp", "message": "call_soon_threadsafe(_handle__worker_status_request)", "payload": "Warning: blocked user event loop delays status reply!", "activeNodes": ["n312-grpc", "n312-disp"], "activeEdges": ["p312-st2"]}, + {"from": "disp", "to": "host", "message": "WorkerStatusResponse() via _grpc_resp_queue", "isReturn": True, "payload": "Logging suppressed to avoid I/O overhead", "activeNodes": ["n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["p312-st3", "p312-st4"]}, + ], + }, + # 7. Worker Logs + { + "id": "worker_logs", + "title": "7. Worker Logs Flow (Worker → Host Reverse Stream)", + "proto": "Python logging.LogRecord → StreamingMessage(RpcLog) → Host", + "scope": "Reverse Telemetry Stream (System & Customer Logs)", + "summary": ( + "Before `EventStream` connects, logs print to stdout/stderr with prefix `LanguageWorkerConsoleLog`. " + "Once connected, `disable_console_logging()` detaches console output and attaches `AsyncLoggingHandler` " + "to the root logger. Every `logging.info/error` from customer code (`User` category) or worker " + "(`System` category) is stamped with `invocation_id` (from task/thread-local) and pushed **back " + "to the Host** on `_grpc_resp_queue` as `RpcLog` messages for Application Insights & Kusto." + ), + "bypassedNote": "Originates inside User Code or Worker internals and flows RIGHT → LEFT back to the Host without a Host request.", + "activeNodes": ["n312-sdk", "n312-disp", "n312-grpc", "n312-proto"], + "topologyEdges": [ + {"id": "p312-lg1", "source": "n312-sdk", "target": "n312-disp", "kind": "pyo3-control", "label": "1. logging.info() -> AsyncLoggingHandler"}, + {"id": "p312-lg2", "source": "n312-disp", "target": "n312-grpc", "kind": "thread-queue", "label": "2. stamp invocation_id -> _grpc_resp_queue"}, + {"id": "p312-lg3", "source": "n312-grpc", "target": "n312-proto", "kind": "grpc-wire", "label": "3. StreamingMessage(rpc_log) → Host"}, + ], + "actors": [ + {"id": "app", "label": "User Code / Worker"}, + {"id": "disp", "label": "AsyncLoggingHandler"}, + {"id": "grpc", "label": "_grpc_resp_queue"}, + {"id": "host", "label": "Host (AppInsights / Kusto)"}, + ], + "steps": [ + {"from": "app", "to": "disp", "message": "logging.info('Order processed') / logger.info(...)", "payload": "Captured by root logger AsyncLoggingHandler", "activeNodes": ["n312-sdk", "n312-disp"], "activeEdges": ["p312-lg1"]}, + {"from": "disp", "to": "disp", "message": "Classify System vs User + resolve invocation_id", "payload": "Checks current asyncio task / thread_local_storage.invocation_id", "activeNodes": ["n312-disp"], "activeEdges": []}, + {"from": "disp", "to": "grpc", "message": "put_nowait(StreamingMessage(rpc_log=RpcLog(...)))", "payload": "Non-blocking enqueue onto outbound gRPC queue", "activeNodes": ["n312-disp", "n312-grpc"], "activeEdges": ["p312-lg2"]}, + {"from": "grpc", "to": "host", "message": "Stream RpcLog frame over EventStream to Host", "isReturn": True, "payload": "User logs -> Customer AppInsights | System logs -> Kusto", "activeNodes": ["n312-grpc", "n312-proto"], "activeEdges": ["p312-lg3"]}, + ], + }, + ] + + +def get_py313_phases() -> list[dict[str, Any]]: + return [ + # 1. Worker Init + { + "id": "init", + "title": "1. Worker Initialization Flow (Runtime Selection + V2 Default Indexing)", + "proto": "StartStream → WorkerInitRequest → WorkerRequest(protos) → WorkerInitResponse", + "scope": "All SKUs (Cold Start)", + "summary": ( + "`proxyV4/worker.py` starts `proxy_worker` with `uvloop` and worker-first deps (`grpcio`/`protobuf`). " + "On `WorkerInitRequest`, `proxy_worker` promotes customer deps, runs **`reload_library_worker(dir)`** " + "(resolving `PYTHON_ENABLE_AGENT_RUNTIME` `entry_points` → `function_app.py` V2 → V1), starts the " + "runtime threadpool, and passes `WorkerRequest(properties={'protos': protos, 'host': host})`. " + "**Key change in 3.13+**: Runtime V2 now **indexes `function_app.py` and starts `http_v2` during `WorkerInit`**!" + ), + "bypassedNote": "Only ONE runtime engine (Agent, V2, or V1) is imported and bound to `_library_worker`.", + "activeNodes": ["n313-proto", "n313-grpc", "n313-router", "n313-rtv2", "n313-rtv1", "n313-ext", "n313-lib"], + "topologyEdges": [ + {"id": "p313-i1", "source": "n313-proto", "target": "n313-grpc", "kind": "grpc-wire", "label": "1. StartStream + WorkerInitRequest"}, + {"id": "p313-i2", "source": "n313-grpc", "target": "n313-router", "kind": "thread-queue", "label": "2. call_soon_threadsafe"}, + {"id": "p313-i3", "source": "n313-router", "target": "n313-ext", "kind": "syspath-mutation", "label": "3a. check Agent Runtime entry_points", "dashed": True}, + {"id": "p313-i4", "source": "n313-router", "target": "n313-rtv2", "kind": "inproc-contract", "label": "3b. V2: WorkerRequest(protos, host)"}, + {"id": "p313-i5", "source": "n313-router", "target": "n313-rtv1", "kind": "inproc-contract", "label": "3c. V1: WorkerRequest(protos, host)", "dashed": True}, + {"id": "p313-i6", "source": "n313-rtv2", "target": "n313-lib", "kind": "exec-dispatch", "label": "4. V2 indexes function_app.py in Init!"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "proxy", "label": "proxy_worker"}, + {"id": "ext", "label": "python-extensions"}, + {"id": "rt", "label": "Runtime (v2 / v1)"}, + {"id": "lib", "label": "python-library / App"}, + ], + "steps": [ + {"from": "host", "to": "proxy", "message": "Spawn proxyV4 -> start_worker(uvloop) + StartStream", "payload": "use_worker_dependencies() protects grpcio import", "activeNodes": ["n313-proto", "n313-grpc"], "activeEdges": ["p313-i1"]}, + {"from": "host", "to": "proxy", "message": "WorkerInitRequest(function_app_directory)", "payload": "prioritize_customer_dependencies() if Dedicated", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["p313-i1", "p313-i2"]}, + {"from": "proxy", "to": "ext", "message": "reload_library_worker(): check AGENT_RUNTIME entry_points", "payload": "RuntimeTrackerMeta -> else function_app.py (V2) -> else V1", "activeNodes": ["n313-router", "n313-ext"], "activeEdges": ["p313-i3"]}, + {"from": "proxy", "to": "rt", "message": "start_threadpool_executor() + worker_init_request(WorkerRequest)", "payload": "Injects properties={'protos': protos, 'host': host}", "activeNodes": ["n313-router", "n313-rtv2", "n313-rtv1"], "activeEdges": ["p313-i4", "p313-i5"]}, + {"from": "rt", "to": "lib", "message": "[V2] Cache protos, load_binding_registry, index_function_app()", "payload": "V2 indexes in Init & starts http_v2 server; V1 only runs install()", "activeNodes": ["n313-rtv2", "n313-lib"], "activeEdges": ["p313-i6"]}, + {"from": "rt", "to": "host", "message": "WorkerInitResponse -> wrapped in StreamingMessage", "isReturn": True, "payload": "Capabilities + PyPI runtime version metadata", "activeNodes": ["n313-rtv2", "n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["p313-i4", "p313-i2", "p313-i1"]}, + ], + }, + # 2. Worker Specialization + { + "id": "specialization", + "title": "2. Worker Specialization Flow (Consumption Runtime Swap)", + "proto": "FunctionEnvironmentReloadRequest → reload_library_worker() → ReloadResponse", + "scope": "Linux / Flex Consumption Only (Placeholder Mode)", + "summary": ( + "During placeholder boot, the worker defaulted to bundled `azure_functions_runtime`. On " + "`FunctionEnvironmentReloadRequest`, `proxy_worker` promotes the newly mounted customer `.python_packages`, " + "**re-runs `reload_library_worker(directory)`** (so a V1 app switches to `azure_functions_runtime_v1`, " + "an Agent app loads its custom runtime, or a customer-pinned PyPI `azure-functions-runtime` takes over!), " + "re-injects `protos`, reloads `os.environ` + `tzset()`, and indexes the specialized app." + ), + "bypassedNote": "Only occurs when specializing warm Consumption standby containers.", + "activeNodes": ["n313-proto", "n313-grpc", "n313-router", "n313-rtv2", "n313-rtv1", "n313-ext", "n313-lib"], + "topologyEdges": [ + {"id": "p313-s1", "source": "n313-proto", "target": "n313-grpc", "kind": "grpc-wire", "label": "1. FunctionEnvironmentReloadRequest"}, + {"id": "p313-s2", "source": "n313-grpc", "target": "n313-router", "kind": "thread-queue", "label": "2. prioritize_customer_deps(dir)"}, + {"id": "p313-s3", "source": "n313-router", "target": "n313-ext", "kind": "syspath-mutation", "label": "3a. re-evaluate Agent Runtime", "dashed": True}, + {"id": "p313-s4", "source": "n313-router", "target": "n313-rtv2", "kind": "inproc-contract", "label": "3b. reload V2 + inject protos"}, + {"id": "p313-s5", "source": "n313-router", "target": "n313-rtv1", "kind": "inproc-contract", "label": "3c. or switch to V1!", "dashed": True}, + {"id": "p313-s6", "source": "n313-rtv2", "target": "n313-lib", "kind": "exec-dispatch", "label": "4. reset_host_config + env + index"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "proxy", "label": "proxy_worker"}, + {"id": "rt", "label": "Selected Runtime"}, + {"id": "lib", "label": "Customer Env & App"}, + ], + "steps": [ + {"from": "host", "to": "proxy", "message": "FunctionEnvironmentReloadRequest(app_dir, env_vars)", "payload": "Specializes warm placeholder container", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["p313-s1", "p313-s2"]}, + {"from": "proxy", "to": "proxy", "message": "DependencyManager.prioritize_customer_dependencies(dir)", "payload": "Evicts worker runtime modules so customer PyPI pin can win", "activeNodes": ["n313-router"], "activeEdges": []}, + {"from": "proxy", "to": "rt", "message": "reload_library_worker(dir) + start_threadpool_executor()", "payload": "Dynamically switches _library_worker to Agent / V2 / V1", "activeNodes": ["n313-router", "n313-ext", "n313-rtv2", "n313-rtv1"], "activeEdges": ["p313-s3", "p313-s4", "p313-s5"]}, + {"from": "rt", "to": "lib", "message": "function_environment_reload_request(WorkerRequest(protos))", "payload": "Reloads os.environ, tzset(), OTel, http_v2, change_cwd, and indexes V2", "activeNodes": ["n313-rtv2", "n313-lib"], "activeEdges": ["p313-s6"]}, + {"from": "rt", "to": "host", "message": "FunctionEnvironmentReloadResponse(capabilities)", "isReturn": True, "payload": "Specialized container ready", "activeNodes": ["n313-rtv2", "n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["p313-s4", "p313-s2", "p313-s1"]}, + ], + }, + # 3. Metadata & Indexing + { + "id": "metadata", + "title": "3. Worker Metadata Flow (Cached V2 Result vs V1 Host Fallback)", + "proto": "FunctionsMetadataRequest → WorkerRequest → FunctionMetadataResponse", + "scope": "V2 Returns Cached Metadata; V1 Delegates Indexing to Host", + "summary": ( + "`proxy_worker` forwards `WorkerRequest(name='WorkerMetadataRequest')` (no `protos` needed — " + "already cached in runtime!). In **Runtime V2**, `functions_metadata_request` immediately returns " + "the `_metadata_result` indexed during Init/Specialization (`use_default_metadata_indexing=False`). " + "In **Runtime V1**, it returns `use_default_metadata_indexing=True, results=None` so the Host " + "reads `function.json` files." + ), + "bypassedNote": "V2 already indexed `function_app.py` during Init/Specialization, so Phase 3 is an O(1) cached return!", + "activeNodes": ["n313-proto", "n313-grpc", "n313-router", "n313-rtv2", "n313-rtv1"], + "topologyEdges": [ + {"id": "p313-m1", "source": "n313-proto", "target": "n313-grpc", "kind": "grpc-wire", "label": "1. FunctionsMetadataRequest"}, + {"id": "p313-m2", "source": "n313-grpc", "target": "n313-router", "kind": "thread-queue", "label": "2. _handle__functions_metadata_request"}, + {"id": "p313-m3", "source": "n313-router", "target": "n313-rtv2", "kind": "inproc-contract", "label": "3a. V2: return cached _metadata_result"}, + {"id": "p313-m4", "source": "n313-router", "target": "n313-rtv1", "kind": "inproc-contract", "label": "3b. V1: use_default_indexing=True", "dashed": True}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "proxy", "label": "proxy_worker"}, + {"id": "rtv2", "label": "Runtime V2"}, + {"id": "rtv1", "label": "Runtime V1"}, + ], + "steps": [ + {"from": "host", "to": "proxy", "message": "FunctionsMetadataRequest(function_app_directory)", "payload": "Requests function list", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["p313-m1", "p313-m2"]}, + {"from": "proxy", "to": "rtv2", "message": "[If V2] await rtv2.functions_metadata_request(req)", "payload": "Returns pre-indexed _metadata_result (use_default=False)", "activeNodes": ["n313-router", "n313-rtv2"], "activeEdges": ["p313-m3"]}, + {"from": "proxy", "to": "rtv1", "message": "[If V1] await rtv1.functions_metadata_request(req)", "payload": "Returns None + use_default_metadata_indexing=True", "activeNodes": ["n313-router", "n313-rtv1"], "activeEdges": ["p313-m4"]}, + {"from": "proxy", "to": "host", "message": "StreamingMessage(function_metadata_response)", "isReturn": True, "payload": "Host registers V2 functions or scans V1 function.json", "activeNodes": ["n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["p313-m2", "p313-m1"]}, + ], + }, + # 4. Function Load (V1) + { + "id": "function_load", + "title": "4. Worker Functions Load Flow (V1 Script Loading vs V2 No-Op)", + "proto": "FunctionLoadRequest → WorkerRequest → FunctionLoadResponse", + "scope": "Active Import for V1 (`runtimes/v1`); Immediate No-Op Ack for V2", + "summary": ( + "The split makes V1 vs V2 load behavior explicit: **`runtimes/v1/handle_event.py`** implements " + "real `load_function()` + `_functions.add_function(function_id, func, metadata, protos)` to import " + "each `function.json` script; **`runtimes/v2/handle_event.py`** (`L160`) is a 6-line immediate " + "`FunctionLoadResponse(Success)` because V2 functions were already loaded during indexing!" + ), + "bypassedNote": "In Runtime V2, `function_load_request` is a pure no-op acknowledgment.", + "activeNodes": ["n313-proto", "n313-grpc", "n313-router", "n313-rtv1", "n313-rtv2", "n313-lib"], + "topologyEdges": [ + {"id": "p313-l1", "source": "n313-proto", "target": "n313-grpc", "kind": "grpc-wire", "label": "1. FunctionLoadRequest"}, + {"id": "p313-l2", "source": "n313-grpc", "target": "n313-router", "kind": "thread-queue", "label": "2. WorkerRequest"}, + {"id": "p313-l3", "source": "n313-router", "target": "n313-rtv1", "kind": "inproc-contract", "label": "3a. V1: load_function + add_function"}, + {"id": "p313-l4", "source": "n313-rtv1", "target": "n313-lib", "kind": "exec-dispatch", "label": "4. V1 imports __init__.py & SDK hints"}, + {"id": "p313-l5", "source": "n313-router", "target": "n313-rtv2", "kind": "inproc-contract", "label": "3b. V2: immediate Success ack", "dashed": True}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "proxy", "label": "proxy_worker"}, + {"id": "rtv1", "label": "Runtime V1"}, + {"id": "app", "label": "V1 __init__.py"}, + ], + "steps": [ + {"from": "host", "to": "proxy", "message": "FunctionLoadRequest(function_id, metadata)", "payload": "Sent per function", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["p313-l1", "p313-l2"]}, + {"from": "proxy", "to": "rtv1", "message": "[V1] await rtv1.function_load_request(WorkerRequest)", "payload": "(In V2, rtv2 immediately returns Success no-op)", "activeNodes": ["n313-router", "n313-rtv1", "n313-rtv2"], "activeEdges": ["p313-l3", "p313-l5"]}, + {"from": "rtv1", "to": "app", "message": "loader.load_function() imports script_file & entry_point", "payload": "_functions.add_function(id, func, metadata, protos)", "activeNodes": ["n313-rtv1", "n313-lib"], "activeEdges": ["p313-l4"]}, + {"from": "rtv1", "to": "host", "message": "FunctionLoadResponse(function_id, Success)", "isReturn": True, "payload": "Wrapped in StreamingMessage by proxy_worker", "activeNodes": ["n313-rtv1", "n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["p313-l3", "p313-l2", "p313-l1"]}, + ], + }, + # 5. Invocation + { + "id": "invocation", + "title": "5. Worker Invocation Flow", + "proto": "InvocationRequest → WorkerRequest → Runtime v2/v1 → InvocationResponse", + "scope": "Every Function Trigger Execution", + "summary": ( + "`proxy_worker` stamps the `ContextEnabledTask` with `invocation_id` and awaits " + "`_library_worker.invocation_request(WorkerRequest)`. The runtime decodes inputs using its " + "cached `protos`, coordinates `http_v2` streaming or `SdkType` bindings (`python-extensions`), " + "executes async on `uvloop` or sync on the runtime's `ThreadPoolExecutor`, and returns `InvocationResponse`." + ), + "bypassedNote": "Runtime discovery and `protos` injection do NOT run per-invocation — cached from Phase 1/2.", + "activeNodes": ["n313-proto", "n313-grpc", "n313-router", "n313-rtv2", "n313-ext", "n313-lib"], + "topologyEdges": [ + {"id": "p313-v1", "source": "n313-proto", "target": "n313-grpc", "kind": "grpc-wire", "label": "1. InvocationRequest"}, + {"id": "p313-v2", "source": "n313-grpc", "target": "n313-router", "kind": "thread-queue", "label": "2. ContextEnabledTask"}, + {"id": "p313-v3", "source": "n313-router", "target": "n313-rtv2", "kind": "inproc-contract", "label": "3. WorkerRequest (no protos copy)"}, + {"id": "p313-v4", "source": "n313-rtv2", "target": "n313-ext", "kind": "pyo3-control", "label": "4a. http_v2 / SdkType / Settlement", "dashed": True}, + {"id": "p313-v5", "source": "n313-rtv2", "target": "n313-lib", "kind": "exec-dispatch", "label": "4b. execute_async / run_sync_func"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "proxy", "label": "proxy_worker"}, + {"id": "rt", "label": "Runtime Engine"}, + {"id": "app", "label": "SDK / Extensions / App"}, + ], + "steps": [ + {"from": "host", "to": "proxy", "message": "InvocationRequest(invocation_id, function_id)", "payload": "Dispatched onto ContextEnabledTask", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["p313-v1", "p313-v2"]}, + {"from": "proxy", "to": "rt", "message": "await _library_worker.invocation_request(WorkerRequest)", "payload": "Stamps global/task invocation_id before call", "activeNodes": ["n313-router", "n313-rtv2"], "activeEdges": ["p313-v3"]}, + {"from": "rt", "to": "app", "message": "from_incoming_proto -> http_v2/SdkType -> run handler", "payload": "invocation_id_cv set on contextvar + threadpool thread", "activeNodes": ["n313-rtv2", "n313-ext", "n313-lib"], "activeEdges": ["p313-v4", "p313-v5"]}, + {"from": "app", "to": "host", "message": "to_outgoing_proto -> InvocationResponse -> _grpc_resp_queue", "isReturn": True, "payload": "StreamingMessage sent over grpcio", "activeNodes": ["n313-lib", "n313-rtv2", "n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["p313-v5", "p313-v3", "p313-v2", "p313-v1"]}, + ], + }, + # 6. Worker Status + { + "id": "worker_status", + "title": "6. Worker Status Flow (Proxy Short-Circuit)", + "proto": "WorkerStatusRequest → proxy_worker → WorkerStatusResponse", + "scope": "Host Scale Controller Health Check (Runtime Bypassed!)", + "summary": ( + "In `3.13 – 3.14`, `_handle__worker_status_request` is the **only** message handler in " + "`proxy_worker/dispatcher.py` that **does NOT delegate to `_library_worker`**! It short-circuits " + "inside `proxy_worker` and returns `WorkerStatusResponse` immediately—though it still runs on " + "the shared CPython interpreter's `uvloop` and GIL." + ), + "bypassedNote": "Runtime V2, Runtime V1, SDK, Extensions, and Customer App are 100% bypassed!", + "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], + "topologyEdges": [ + {"id": "p313-st1", "source": "n313-proto", "target": "n313-grpc", "kind": "grpc-wire", "label": "1. WorkerStatusRequest"}, + {"id": "p313-st2", "source": "n313-grpc", "target": "n313-router", "kind": "thread-queue", "label": "2. proxy short-circuit (still GIL)"}, + {"id": "p313-st3", "source": "n313-router", "target": "n313-grpc", "kind": "thread-queue", "label": "3. WorkerStatusResponse"}, + {"id": "p313-st4", "source": "n313-grpc", "target": "n313-proto", "kind": "grpc-wire", "label": "4. EventStream reply"}, + ], + "actors": [ + {"id": "host", "label": "Host Scale Controller"}, + {"id": "grpc", "label": "Proxy gRPC Thread"}, + {"id": "proxy", "label": "Proxy Dispatcher"}, + ], + "steps": [ + {"from": "host", "to": "grpc", "message": "WorkerStatusRequest()", "payload": "Scale controller health check", "activeNodes": ["n313-proto", "n313-grpc"], "activeEdges": ["p313-st1"]}, + {"from": "grpc", "to": "proxy", "message": "_handle__worker_status_request (short-circuit)", "payload": "Never crosses WorkerRequest seam; still requires Python GIL/loop", "activeNodes": ["n313-grpc", "n313-router"], "activeEdges": ["p313-st2"]}, + {"from": "proxy", "to": "host", "message": "WorkerStatusResponse()", "isReturn": True, "payload": "Returned via _grpc_resp_queue", "activeNodes": ["n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["p313-st3", "p313-st4"]}, + ], + }, + # 7. Worker Logs + { + "id": "worker_logs", + "title": "7. Worker Logs Flow (4-Tier Invocation-ID Correlation → Host)", + "proto": "LogRecord → 4-Tier ID Resolver → StreamingMessage(RpcLog) → Host", + "scope": "Reverse Telemetry Stream (Worker / Runtime / Customer → Host)", + "summary": ( + "Because execution lives in a separate PyPI runtime package (or custom Agent Runtime), " + "`proxy_worker`'s `AsyncLoggingHandler` resolves `invocation_id` across **4 fallback tiers**: " + "1) global tracker (for custom runtimes without `cv`), 2) `ContextEnabledTask` attribute, " + "3) `_library_worker.invocation_id_cv` (`ContextVar`), and 4) `_thread_invocation_registry`—then " + "streams `RpcLog` frames back to the Host." + ), + "bypassedNote": "Flows RIGHT → LEFT from User App / Runtime loggers through proxy_worker's AsyncLoggingHandler to the Host.", + "activeNodes": ["n313-lib", "n313-rtv2", "n313-router", "n313-grpc", "n313-proto"], + "topologyEdges": [ + {"id": "p313-lg1", "source": "n313-lib", "target": "n313-rtv2", "kind": "exec-dispatch", "label": "1. user logging.info() (cv / TLS set)"}, + {"id": "p313-lg2", "source": "n313-rtv2", "target": "n313-router", "kind": "pyo3-control", "label": "2. root AsyncLoggingHandler + 4-tier ID"}, + {"id": "p313-lg3", "source": "n313-router", "target": "n313-grpc", "kind": "thread-queue", "label": "3. protos.RpcLog -> _grpc_resp_queue"}, + {"id": "p313-lg4", "source": "n313-grpc", "target": "n313-proto", "kind": "grpc-wire", "label": "4. StreamingMessage(rpc_log) → Host"}, + ], + "actors": [ + {"id": "app", "label": "User Code / Thread"}, + {"id": "rt", "label": "Runtime (invocation_id_cv)"}, + {"id": "proxy", "label": "Proxy AsyncLoggingHandler"}, + {"id": "host", "label": "Host (AppInsights / Kusto)"}, + ], + "steps": [ + {"from": "app", "to": "proxy", "message": "logging.info(...) from async task, threadpool, or user thread", "payload": "Triggers root logger AsyncLoggingHandler in proxy_worker", "activeNodes": ["n313-lib", "n313-rtv2", "n313-router"], "activeEdges": ["p313-lg1", "p313-lg2"]}, + {"from": "proxy", "to": "rt", "message": "get_current_invocation_id() 4-tier lookup", "payload": "Global -> ContextEnabledTask -> runtime.invocation_id_cv -> thread registry", "activeNodes": ["n313-router", "n313-rtv2"], "activeEdges": ["p313-lg2"]}, + {"from": "proxy", "to": "host", "message": "Tag System vs User -> protos.RpcLog -> EventStream", "isReturn": True, "payload": "Correlated by invocation_id in Application Insights", "activeNodes": ["n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["p313-lg3", "p313-lg4"]}, + ], + }, + ] + + +def get_py315_phases() -> list[dict[str, Any]]: + return [ + # 1. Worker Init + { + "id": "init", + "title": "1. Worker Initialization Flow (Embed CPython + Protobuf-Free Init)", + "proto": "bridge::configure → StartStream (prost) → WorkerInitRequest → protos_adapter", + "scope": "All SKUs (Cold Start)", + "summary": ( + "Host spawns compiled Rust binary `r2p2`. Before dialing gRPC, `main.rs` creates a single lock-free " + "`tokio::mpsc::unbounded_channel` and calls `bridge::configure()` via PyO3 (embedding CPython, " + "starting `bridge-asyncio` loop thread, and attaching `_RpcLogHandler` to Rust's `LogSink`). " + "Rust opens `tonic` `EventStream`, sends `prost`-encoded `StartStream`, and routes `WorkerInitRequest` " + "as a `PyDict` through `bridge.py` + pure-Python `protos_adapter.py` into Runtime V2/V1." + ), + "bypassedNote": "Hot FFI (`convert.rs` / `native_invocation.py`) is not used during control-plane initialization.", + "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi", "n315-bridge-py", "n315-rt", "n315-lib"], + "topologyEdges": [ + {"id": "p315-i1", "source": "n315-tonic", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "1. bridge::configure(LogSink)"}, + {"id": "p315-i2", "source": "n315-tonic", "target": "n315-proto", "kind": "grpc-wire", "label": "2. tonic connect + StartStream (prost)"}, + {"id": "p315-i3", "source": "n315-proto", "target": "n315-router", "kind": "grpc-wire", "label": "3. WorkerInitRequest bytes"}, + {"id": "p315-i4", "source": "n315-router", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "4. worker_init_req_to_py(PyDict)"}, + {"id": "p315-i5", "source": "n315-ctrl-ffi", "target": "n315-bridge-py", "kind": "pyo3-control", "label": "5. handle_control + select v2/v1"}, + {"id": "p315-i6", "source": "n315-bridge-py", "target": "n315-rt", "kind": "inproc-contract", "label": "6. WorkerRequest(protos=protos_adapter)"}, + {"id": "p315-i7", "source": "n315-rt", "target": "n315-lib", "kind": "exec-dispatch", "label": "7. V2 indexes @app decorators"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "rust", "label": "R2P2 (tonic+prost)"}, + {"id": "ctrl", "label": "bridge.rs (PyO3)"}, + {"id": "bpy", "label": "bridge.py + adapter"}, + {"id": "rt", "label": "Runtime v2 / v1"}, + ], + "steps": [ + {"from": "rust", "to": "bpy", "message": "Create mpsc + bridge::configure(LogSink)", "payload": "Embeds libpython3.15, starts bridge-asyncio thread, installs _RpcLogHandler", "activeNodes": ["n315-tonic", "n315-ctrl-ffi", "n315-bridge-py"], "activeEdges": ["p315-i1", "p315-i5"]}, + {"from": "rust", "to": "host", "message": "Dial tonic EventStream + send prost StartStream(worker_id)", "isReturn": True, "payload": "BytesCodec over HTTP/2 TCP_NODELAY", "activeNodes": ["n315-tonic", "n315-proto"], "activeEdges": ["p315-i2"]}, + {"from": "host", "to": "ctrl", "message": "WorkerInitRequest -> prost decode -> worker_init_req_to_py", "payload": "Zero google.protobuf in Python!", "activeNodes": ["n315-proto", "n315-router", "n315-ctrl-ffi"], "activeEdges": ["p315-i3", "p315-i4"]}, + {"from": "ctrl", "to": "bpy", "message": "bridge.handle_control('worker_init_request', dict)", "payload": "_prioritize_customer_dependencies + _select_runtime(dir)", "activeNodes": ["n315-ctrl-ffi", "n315-bridge-py"], "activeEdges": ["p315-i5"]}, + {"from": "bpy", "to": "rt", "message": "worker_init_request(_WorkerRequest(protos=protos_adapter))", "payload": "V2 indexes function_app.py & starts http_v2 if needed", "activeNodes": ["n315-bridge-py", "n315-rt", "n315-lib"], "activeEdges": ["p315-i6", "p315-i7"]}, + {"from": "rt", "to": "host", "message": "protos_adapter.to_dict() -> py_to_worker_init_response -> prost", "isReturn": True, "payload": "Sent via mpsc to Host", "activeNodes": ["n315-rt", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["p315-i6", "p315-i4", "p315-i2"]}, + ], + }, + # 2. Worker Specialization + { + "id": "specialization", + "title": "2. Worker Specialization Flow (Namespace-Safe Eviction + Reload)", + "proto": "FunctionEnvironmentReloadRequest → _clear_modules_from_path → ReloadResponse", + "scope": "Linux / Flex Consumption Only (Placeholder Mode)", + "summary": ( + "When specializing a warm R2P2 standby process, `bridge.py` runs **`_clear_modules_from_path(_workers_dir)`**: " + "file-backed worker modules & cached descendants are evicted and stale child attributes are detached " + "from retained parents, while **PEP 420 namespace parents (`azure`, `azurefunctions`) without `__file__` " + "are preserved** so customer SDK/extension children stay attached! Plus zero `grpcio`/`protobuf` C-extensions " + "ever existed in `sys.modules` to collide with customer pins." + ), + "bypassedNote": "Hot invocation seam is cleared (`_control_path_cache.clear()`) and re-bound to the newly selected runtime.", + "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi", "n315-bridge-py", "n315-rt", "n315-lib", "n315-ext"], + "topologyEdges": [ + {"id": "p315-s1", "source": "n315-proto", "target": "n315-router", "kind": "grpc-wire", "label": "1. FunctionEnvironmentReloadRequest"}, + {"id": "p315-s2", "source": "n315-router", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "2. env_reload_req_to_py(PyDict)"}, + {"id": "p315-s3", "source": "n315-ctrl-ffi", "target": "n315-bridge-py", "kind": "pyo3-control", "label": "3. _clear_modules_from_path (namespace-safe)"}, + {"id": "p315-s4", "source": "n315-bridge-py", "target": "n315-lib", "kind": "syspath-mutation", "label": "4a. promote customer .python_packages", "dashed": True}, + {"id": "p315-s5", "source": "n315-bridge-py", "target": "n315-ext", "kind": "syspath-mutation", "label": "4b. keep azurefunctions.* namespaces", "dashed": True}, + {"id": "p315-s6", "source": "n315-bridge-py", "target": "n315-rt", "kind": "inproc-contract", "label": "5. reload runtime + env_vars + index"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "ctrl", "label": "bridge.rs (prost/PyO3)"}, + {"id": "bpy", "label": "bridge.py Dep Engine"}, + {"id": "rt", "label": "Runtime + Customer Pkgs"}, + ], + "steps": [ + {"from": "host", "to": "ctrl", "message": "FunctionEnvironmentReloadRequest(app_dir, env_vars)", "payload": "prost decodes reload request into PyDict", "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi"], "activeEdges": ["p315-s1", "p315-s2"]}, + {"from": "ctrl", "to": "bpy", "message": "_clear_modules_from_path(_workers_dir) + invalidate_caches()", "payload": "Evicts file-backed worker deps; preserves namespace parents (`azure`, `azurefunctions`)", "activeNodes": ["n315-ctrl-ffi", "n315-bridge-py", "n315-ext"], "activeEdges": ["p315-s3", "p315-s5"]}, + {"from": "bpy", "to": "bpy", "message": "_prioritize_customer_dependencies(dir) + _select_runtime(dir)", "payload": "Clears _control_path_cache & re-caches _rt_tls for log correlation", "activeNodes": ["n315-bridge-py", "n315-lib"], "activeEdges": ["p315-s4"]}, + {"from": "bpy", "to": "rt", "message": "function_environment_reload_request(protos=protos_adapter)", "payload": "Reloads os.environ, tzset(), OTel, http_v2, and indexes customer app", "activeNodes": ["n315-bridge-py", "n315-rt"], "activeEdges": ["p315-s6"]}, + {"from": "rt", "to": "host", "message": "py_to_env_reload_response -> prost StreamingMessage", "isReturn": True, "payload": "Specialized without any protobuf descriptor-pool conflicts!", "activeNodes": ["n315-rt", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["p315-s6", "p315-s2", "p315-s1"]}, + ], + }, + # 3. Metadata & Indexing + { + "id": "metadata", + "title": "3. Worker Metadata & Indexing Flow", + "proto": "FunctionsMetadataRequest → control.rs → protos_adapter → prost", + "scope": "V2 Returns Indexed Decorator Metadata; V1 Signals Host Indexing", + "summary": ( + "Rust `bridge::handle` decodes `FunctionsMetadataRequest` and calls `bridge.handle_control('functions_metadata_request', {})`. " + "Runtime V2 returns its indexed `RawFunctionMetadata` list wrapped in `protos_adapter` objects (`.to_dict()`), " + "which `control::py_to_function_metadata_response` serializes into native `prost` wire bytes. " + "Runtime V1 returns `use_default_metadata_indexing=True`." + ), + "bypassedNote": "Hot native tuple FFI (`bridge::invoke`) is unused during metadata discovery.", + "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi", "n315-bridge-py", "n315-rt"], + "topologyEdges": [ + {"id": "p315-m1", "source": "n315-proto", "target": "n315-router", "kind": "grpc-wire", "label": "1. FunctionsMetadataRequest"}, + {"id": "p315-m2", "source": "n315-router", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "2. bridge::handle()"}, + {"id": "p315-m3", "source": "n315-ctrl-ffi", "target": "n315-bridge-py", "kind": "pyo3-control", "label": "3. handle_control(dict)"}, + {"id": "p315-m4", "source": "n315-bridge-py", "target": "n315-rt", "kind": "inproc-contract", "label": "4. functions_metadata_request()"}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "ctrl", "label": "bridge.rs + control.rs"}, + {"id": "bpy", "label": "bridge.py + adapter"}, + {"id": "rt", "label": "Runtime v2 / v1"}, + ], + "steps": [ + {"from": "host", "to": "ctrl", "message": "FunctionsMetadataRequest bytes", "payload": "prost-decoded in Rust", "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi"], "activeEdges": ["p315-m1", "p315-m2"]}, + {"from": "ctrl", "to": "bpy", "message": "handle_control('functions_metadata_request', {})", "payload": "Emits Kusto-compatible Received WorkerMetadataRequest system log", "activeNodes": ["n315-ctrl-ffi", "n315-bridge-py"], "activeEdges": ["p315-m3"]}, + {"from": "bpy", "to": "rt", "message": "_run_coro(_rt.functions_metadata_request(req))", "payload": "V2 returns _metadata_result | V1 returns use_default_metadata_indexing=True", "activeNodes": ["n315-bridge-py", "n315-rt"], "activeEdges": ["p315-m4"]}, + {"from": "rt", "to": "host", "message": "to_dict() -> py_to_function_metadata_response -> prost", "isReturn": True, "payload": "StreamingMessage(FunctionMetadataResponse) bytes", "activeNodes": ["n315-rt", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["p315-m4", "p315-m2", "p315-m1"]}, + ], + }, + # 4. Function Load (V1) + { + "id": "function_load", + "title": "4. Worker Functions Load Flow (V1 `_metadata_ns` Import vs V2 Ack)", + "proto": "FunctionLoadRequest → _metadata_ns(SimpleNamespace) → FunctionLoadResponse", + "scope": "V1 `function.json` Script Loading (`azure_functions_runtime_v1`); V2 Immediate Ack", + "summary": ( + "For **V1** apps, `control::function_load_req_to_py` decodes `RpcFunctionMetadata` & `BindingInfo` " + "into a `PyDict`, and `bridge._metadata_ns()` wraps them in `SimpleNamespace` objects matching " + "protobuf attribute access so `azure_functions_runtime_v1` imports `__init__.py` without `google.protobuf`! " + "For **V2** apps, `function_load_request` immediately returns `Success`." + ), + "bypassedNote": "Hot FFI stays idle; V2 apps return immediate Success while V1 apps import their `__init__.py` entrypoints here.", + "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi", "n315-bridge-py", "n315-rt", "n315-lib"], + "topologyEdges": [ + {"id": "p315-l1", "source": "n315-proto", "target": "n315-router", "kind": "grpc-wire", "label": "1. FunctionLoadRequest"}, + {"id": "p315-l2", "source": "n315-router", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "2. function_load_req_to_py(dict)"}, + {"id": "p315-l3", "source": "n315-ctrl-ffi", "target": "n315-bridge-py", "kind": "pyo3-control", "label": "3. _metadata_ns + _binding_ns"}, + {"id": "p315-l4", "source": "n315-bridge-py", "target": "n315-rt", "kind": "inproc-contract", "label": "4. V1 load_function() / V2 ack"}, + {"id": "p315-l5", "source": "n315-rt", "target": "n315-lib", "kind": "exec-dispatch", "label": "5. V1 imports __init__.py", "dashed": True}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "ctrl", "label": "bridge.rs + control.rs"}, + {"id": "bpy", "label": "bridge.py (_metadata_ns)"}, + {"id": "rt", "label": "Runtime V1 (or V2)"}, + {"id": "app", "label": "V1 __init__.py"}, + ], + "steps": [ + {"from": "host", "to": "ctrl", "message": "FunctionLoadRequest(function_id, metadata)", "payload": "prost decodes bindings & scriptFile", "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi"], "activeEdges": ["p315-l1", "p315-l2"]}, + {"from": "ctrl", "to": "bpy", "message": "handle_control('function_load_request', req_dict)", "payload": "_metadata_ns() constructs protobuf-free SimpleNamespace tree", "activeNodes": ["n315-ctrl-ffi", "n315-bridge-py"], "activeEdges": ["p315-l3"]}, + {"from": "bpy", "to": "rt", "message": "await _rt.function_load_request(_WorkerRequest)", "payload": "V2 returns immediate Success; V1 runs load_function()", "activeNodes": ["n315-bridge-py", "n315-rt"], "activeEdges": ["p315-l4"]}, + {"from": "rt", "to": "app", "message": "[V1 Only] Import script_file & register FunctionInfo", "payload": "Populates _functions[function_id]", "activeNodes": ["n315-rt", "n315-lib"], "activeEdges": ["p315-l5"]}, + {"from": "rt", "to": "host", "message": "py_to_function_load_response -> prost bytes", "isReturn": True, "payload": "FunctionLoadResponse(Success)", "activeNodes": ["n315-rt", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["p315-l4", "p315-l2", "p315-l1"]}, + ], + }, + # 5. Invocation + { + "id": "invocation", + "title": "5. Worker Invocation Flow (Dual Path: Native Datum Tuples vs Extensions)", + "proto": "InvocationRequest → [Hot: bridge::invoke | Ext: bridge::handle] → InvocationResponse", + "scope": "Every Trigger Execution (+122% Throughput on Hot Path)", + "summary": ( + "`main.rs` spawns a concurrent task per request and checks memoized `needs_control_path(function_id)`: " + "**(A) Hot Path** (standard V2/V1 `python-library` bindings) converts `prost` `TypedData` directly " + "into PyO3 `(tag, val)` datum tuples (`convert.rs`) and calls `native_invocation.py`—zero Python " + "protobuf! **(B) Control Path** (`python-extensions` `http_v2` FastAPI streaming or `SdkType` " + "deferred bindings) routes via `control.rs` `PyDict` + `protos_adapter.py` to `handle_event.py`." + ), + "bypassedNote": "Standard functions completely bypass `control.rs`, `protos_adapter.py`, and `handle_event.py`!", + "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-hot-ffi", "n315-ctrl-ffi", "n315-bridge-py", "n315-native-py", "n315-rt", "n315-lib", "n315-ext"], + "topologyEdges": [ + {"id": "p315-v1", "source": "n315-proto", "target": "n315-tonic", "kind": "grpc-wire", "label": "1. InvocationRequest (BytesCodec)"}, + {"id": "p315-v2", "source": "n315-tonic", "target": "n315-router", "kind": "thread-queue", "label": "2. tokio::spawn + control_path_cache"}, + {"id": "p315-v3", "source": "n315-router", "target": "n315-hot-ffi", "kind": "pyo3-hot", "label": "3a. Hot: bridge::invoke()"}, + {"id": "p315-v4", "source": "n315-hot-ffi", "target": "n315-native-py", "kind": "pyo3-hot", "label": "4a. invoke_native(datum tuples)"}, + {"id": "p315-v5", "source": "n315-native-py", "target": "n315-lib", "kind": "exec-dispatch", "label": "5a. Datum decode + sync/async fn"}, + {"id": "p315-v6", "source": "n315-router", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "3b. Ext: HTTP-v2 / SdkType", "dashed": True}, + {"id": "p315-v7", "source": "n315-ctrl-ffi", "target": "n315-bridge-py", "kind": "pyo3-control", "label": "4b. handle_invocation_control", "dashed": True}, + {"id": "p315-v8", "source": "n315-bridge-py", "target": "n315-rt", "kind": "inproc-contract", "label": "5b. handle_event + http_v2", "dashed": True}, + {"id": "p315-v9", "source": "n315-rt", "target": "n315-ext", "kind": "pyo3-control", "label": "6b. stream WebApp / SdkType", "dashed": True}, + ], + "actors": [ + {"id": "host", "label": "Functions Host"}, + {"id": "rust", "label": "Tokio + Router"}, + {"id": "hot", "label": "Hot FFI (convert.rs)"}, + {"id": "native", "label": "native_invocation.py"}, + {"id": "app", "label": "SDK / Extensions / App"}, + ], + "steps": [ + {"from": "host", "to": "rust", "message": "StreamingMessage(InvocationRequest) -> tokio::spawn", "payload": "Concurrent dispatch; checks control_path_cache(fn_id)", "activeNodes": ["n315-proto", "n315-tonic", "n315-router"], "activeEdges": ["p315-v1", "p315-v2"]}, + {"from": "rust", "to": "hot", "message": "[Hot Path] spawn_blocking -> bridge::invoke(&req)", "payload": "typed_data_to_tuple() converts inputs in Rust", "activeNodes": ["n315-router", "n315-hot-ffi"], "activeEdges": ["p315-v3"]}, + {"from": "hot", "to": "native", "message": "Python::attach -> invoke_native(fn_id, inputs, meta)", "payload": "_InvocShim + OTel context; zero Python protobuf", "activeNodes": ["n315-hot-ffi", "n315-native-py"], "activeEdges": ["p315-v4"]}, + {"from": "native", "to": "app", "message": "Decode python-library Datum + run sync/async handler", "payload": "(Or via handle_invocation_control -> http_v2 / SdkType if Extension)", "activeNodes": ["n315-native-py", "n315-lib", "n315-rt", "n315-ext"], "activeEdges": ["p315-v5", "p315-v8", "p315-v9"]}, + {"from": "app", "to": "host", "message": "tuple_to_typed_data -> prost encode -> mpsc", "isReturn": True, "payload": "754 req/s vs 339 req/s (+122% on GIL build)", "activeNodes": ["n315-lib", "n315-native-py", "n315-hot-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["p315-v5", "p315-v4", "p315-v3", "p315-v1"]}, + ], + }, + # 6. Worker Status + { + "id": "worker_status", + "title": "6. Worker Status Flow (Zero-GIL Native Rust Fast-Path)", + "proto": "WorkerStatusRequest → Rust prost → WorkerStatusResponse (0 Python / 0 GIL)", + "scope": "Host Scale Controller Health Check (Entire CPython Interpreter Bypassed!)", + "summary": ( + "Major architectural win in R2P2: when `bridge::handle` (`src/bridge.rs:L114`) matches " + "`Content::WorkerStatusRequest`, it builds `WorkerStatusResponse` in **pure Rust** and pushes " + "the bytes onto `mpsc` **without calling `Python::attach` and without acquiring the GIL**! " + "Even if customer Python threads are 100% CPU-saturated holding the GIL, scale probes answer in microseconds." + ), + "bypassedNote": "PyO3 GIL boundary, Bridge, Runtime Engine, SDK, Extensions, and User Code are 100% untouched!", + "activeNodes": ["n315-proto", "n315-tonic", "n315-router"], + "topologyEdges": [ + {"id": "p315-st1", "source": "n315-proto", "target": "n315-tonic", "kind": "grpc-wire", "label": "1. WorkerStatusRequest bytes"}, + {"id": "p315-st2", "source": "n315-tonic", "target": "n315-router", "kind": "thread-queue", "label": "2. prost match WorkerStatusRequest"}, + {"id": "p315-st3", "source": "n315-router", "target": "n315-tonic", "kind": "thread-queue", "label": "3. WorkerStatusResponse (ZERO GIL!)"}, + {"id": "p315-st4", "source": "n315-tonic", "target": "n315-proto", "kind": "grpc-wire", "label": "4. EventStream reply"}, + ], + "actors": [ + {"id": "host", "label": "Host Scale Controller"}, + {"id": "tonic", "label": "Tokio + Tonic"}, + {"id": "router", "label": "Rust Router (0 GIL)"}, + ], + "steps": [ + {"from": "host", "to": "tonic", "message": "StreamingMessage(WorkerStatusRequest)", "payload": "Host health & scale latency probe", "activeNodes": ["n315-proto", "n315-tonic"], "activeEdges": ["p315-st1"]}, + {"from": "tonic", "to": "router", "message": "Match Some(Content::WorkerStatusRequest(_)) in Rust", "payload": "Does NOT call Python::attach — immune to Python GIL contention!", "activeNodes": ["n315-tonic", "n315-router"], "activeEdges": ["p315-st2"]}, + {"from": "router", "to": "host", "message": "prost encode WorkerStatusResponse -> mpsc tx", "isReturn": True, "payload": "Sub-millisecond native reply", "activeNodes": ["n315-router", "n315-tonic", "n315-proto"], "activeEdges": ["p315-st3", "p315-st4"]}, + ], + }, + # 7. Worker Logs + { + "id": "worker_logs", + "title": "7. Worker Logs Flow (Non-Blocking PyO3 LogSink → Lock-Free Rust mpsc)", + "proto": "logging.LogRecord → _RpcLogHandler → #[pyclass] LogSink → prost → Host", + "scope": "Reverse Telemetry Stream (Customer & System Logs → Host)", + "summary": ( + "Startup logs emit `LanguageWorkerConsoleLog` via `src/log.rs`. Once `bridge.configure()` runs, " + "`_RpcLogHandler` captures all Python root logs, resolves `invocation_id` from **1) `_rt.invocation_id_cv`** " + "(async & threadpool handlers) or **2) `_rt_tls` (`context.thread_local_storage.invocation_id`)** " + "(user-spawned `threading.Thread`s!), and calls `LogSink.emit_log(dict)`. Rust `prost`-encodes `RpcLog` " + "and pushes onto the lock-free `UnboundedSender` without awaiting or deadlocking the GIL." + ), + "bypassedNote": "Flows RIGHT → LEFT from User Code / Runtime into PyO3 `LogSink` and straight onto the outbound tonic stream.", + "activeNodes": ["n315-lib", "n315-native-py", "n315-bridge-py", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], + "topologyEdges": [ + {"id": "p315-lg1", "source": "n315-lib", "target": "n315-native-py", "kind": "exec-dispatch", "label": "1. user logging.info() (cv / _rt_tls)"}, + {"id": "p315-lg2", "source": "n315-native-py", "target": "n315-bridge-py", "kind": "inproc-contract", "label": "2. _RpcLogHandler + _current_invocation_id()"}, + {"id": "p315-lg3", "source": "n315-bridge-py", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "3. LogSink.emit_log(flat dict)"}, + {"id": "p315-lg4", "source": "n315-ctrl-ffi", "target": "n315-tonic", "kind": "thread-queue", "label": "4. prost encode -> UnboundedSender"}, + {"id": "p315-lg5", "source": "n315-tonic", "target": "n315-proto", "kind": "grpc-wire", "label": "5. StreamingMessage(RpcLog) → Host"}, + ], + "actors": [ + {"id": "app", "label": "User Code / User Thread"}, + {"id": "bpy", "label": "bridge.py (_RpcLogHandler)"}, + {"id": "sink", "label": "Rust #[pyclass] LogSink"}, + {"id": "tonic", "label": "mpsc + Tonic"}, + {"id": "host", "label": "Host (AppInsights / Kusto)"}, + ], + "steps": [ + {"from": "app", "to": "bpy", "message": "logging.info(...) from handler or custom threading.Thread", "payload": "Intercepted by root _RpcLogHandler", "activeNodes": ["n315-lib", "n315-native-py", "n315-bridge-py"], "activeEdges": ["p315-lg1", "p315-lg2"]}, + {"from": "bpy", "to": "bpy", "message": "Resolve _current_invocation_id() + System vs User category", "payload": "1) _rt.invocation_id_cv -> 2) _rt_tls (context.thread_local_storage)", "activeNodes": ["n315-bridge-py"], "activeEdges": []}, + {"from": "bpy", "to": "sink", "message": "sink.emit_log({level, message, category, invocation_id})", "payload": "Passes primitive Python dict — zero Python protobuf!", "activeNodes": ["n315-bridge-py", "n315-ctrl-ffi"], "activeEdges": ["p315-lg3"]}, + {"from": "sink", "to": "tonic", "message": "py_log_to_streaming_message -> tx.send(Bytes)", "payload": "Non-blocking lock-free mpsc send; never deadlocks while holding GIL", "activeNodes": ["n315-ctrl-ffi", "n315-tonic"], "activeEdges": ["p315-lg4"]}, + {"from": "tonic", "to": "host", "message": "Drain UnboundedReceiverStream onto EventStream", "isReturn": True, "payload": "Delivered to Host AppInsights / Kusto", "activeNodes": ["n315-tonic", "n315-proto"], "activeEdges": ["p315-lg5"]}, + ], + }, + ] diff --git a/eng/docs/tests/test_extract_arch.py b/eng/docs/tests/test_extract_arch.py new file mode 100644 index 000000000..fc6c6b9b9 --- /dev/null +++ b/eng/docs/tests/test_extract_arch.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Unit tests for the multi-repo architecture graph extractor (`eng/docs/extract_arch.py`).""" + +import copy +import importlib.util +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +EXTRACTOR_PATH = REPO_ROOT / "eng" / "docs" / "extract_arch.py" + +SPEC = importlib.util.spec_from_file_location("extract_arch", EXTRACTOR_PATH) +extract_arch = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(extract_arch) + + +class TestExtractArch(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.graph = extract_arch.build_graph() + + def test_four_linked_repositories_present(self) -> None: + self.assertEqual( + set(self.graph["repos"].keys()), + { + "python-worker", + "python-library", + "python-extensions", + "language-worker-protobuf", + }, + ) + + def test_three_worker_eras_each_have_all_seven_phases(self) -> None: + expected_phases = [ + "init", + "specialization", + "metadata", + "function_load", + "invocation", + "worker_status", + "worker_logs", + ] + eras_by_id = {e["id"]: e for e in self.graph["eras"]} + for era_id in ("py312", "py313_314", "py315_r2p2"): + self.assertIn(era_id, eras_by_id) + era = eras_by_id[era_id] + self.assertEqual([p["id"] for p in era["phases"]], expected_phases) + for phase in era["phases"]: + self.assertTrue(phase["topologyEdges"]) + self.assertTrue(phase["steps"]) + self.assertIn("flowchart LR", phase["topologyMermaid"]) + self.assertIn("sequenceDiagram", phase["sequenceMermaid"]) + + def test_live_rust_and_proto_extraction(self) -> None: + self.assertGreaterEqual(len(self.graph["protoStreamingMessages"]), 20) + eras_by_id = {e["id"]: e for e in self.graph["eras"]} + r2p2_nodes = {n["id"]: n for n in eras_by_id["py315_r2p2"]["nodes"]} + router_syms = {s["name"] for s in r2p2_nodes["n315-router"]["extractedSymbols"]} + self.assertIn("process_message", router_syms) + + def test_structural_fingerprint_is_deterministic(self) -> None: + second = extract_arch.build_graph() + self.assertEqual( + self.graph["structuralFingerprint"], second["structuralFingerprint"] + ) + + def test_sibling_root_extracts_external_python_symbols(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + lib_dir = Path(tmp) / "python-library" / "azure" / "functions" + lib_dir.mkdir(parents=True) + (lib_dir / "meta.py").write_text( + "class Datum:\n def value(self):\n return None\n", + encoding="utf-8", + ) + graph = extract_arch.build_graph(sibling_root=Path(tmp)) + eco = next(e for e in graph["eras"] if e["id"] == "ecosystem") + datum_node = next(n for n in eco["nodes"] if n["id"] == "ne-lib-datum") + sym_names = [s["name"] for s in datum_node["extractedSymbols"]] + self.assertIn("Datum", sym_names) + + def test_validate_graph_rejects_dangling_phase_edge(self) -> None: + broken = copy.deepcopy(self.graph) + py315 = next(e for e in broken["eras"] if e["id"] == "py315_r2p2") + py315["phases"][0]["topologyEdges"][0]["target"] = "non-existent-node" + with self.assertRaises(ValueError): + extract_arch.validate_graph(broken) + + +if __name__ == "__main__": + unittest.main() From 7bd9b3cabbd0be6b1396f0903ccb1e31a15295ff Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Sun, 15 Nov 2026 19:02:01 -0600 Subject: [PATCH 2/7] ci: exclude docs and architecture Pages paths from worker test matrices Skip running the full worker build, emulator, and integration matrices when changes only touch docs/*, eng/docs/*, or .github/workflows/architecture-pages.yml. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/ci/emulator-tests.yml | 10 ++++++++++ eng/ci/integration-tests.yml | 10 ++++++++++ eng/ci/official-build.yml | 5 +++++ eng/ci/public-build.yml | 10 ++++++++++ 4 files changed, 35 insertions(+) diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml index c7792ea94..97a2b4784 100644 --- a/eng/ci/emulator-tests.yml +++ b/eng/ci/emulator-tests.yml @@ -4,12 +4,22 @@ trigger: # run for sdk and extension release branches include: - sdk/* - extensions/* + paths: + exclude: + - docs/* + - eng/docs/* + - .github/workflows/architecture-pages.yml pr: branches: include: - dev - release* + paths: + exclude: + - docs/* + - eng/docs/* + - .github/workflows/architecture-pages.yml schedules: - cron: "0 8 * * 1,2,3,4,5" diff --git a/eng/ci/integration-tests.yml b/eng/ci/integration-tests.yml index 656317ca2..c1d54d8e3 100644 --- a/eng/ci/integration-tests.yml +++ b/eng/ci/integration-tests.yml @@ -4,12 +4,22 @@ trigger: # run for sdk and extension release branches include: - sdk/* - extensions/* + paths: + exclude: + - docs/* + - eng/docs/* + - .github/workflows/architecture-pages.yml pr: branches: include: - dev - release* + paths: + exclude: + - docs/* + - eng/docs/* + - .github/workflows/architecture-pages.yml schedules: - cron: "0 8 * * 1,2,3,4,5" diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 9aa464710..3ed3ada52 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -6,6 +6,11 @@ trigger: - release/* # proxy worker releases - release-v1/* # V1 worker releases - release-v2/* # V2 worker releases + paths: + exclude: + - docs/* + - eng/docs/* + - .github/workflows/architecture-pages.yml # CI only, does not trigger on PRs. pr: none diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index e32028304..e2512e1d2 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -5,11 +5,21 @@ trigger: - dev - sdk/* - extensions/* + paths: + exclude: + - docs/* + - eng/docs/* + - .github/workflows/architecture-pages.yml pr: branches: include: - dev + paths: + exclude: + - docs/* + - eng/docs/* + - .github/workflows/architecture-pages.yml schedules: - cron: '0 0 * * MON' From 782878646d4bef79d39841c98ac5973e684b88e7 Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Sun, 15 Nov 2026 19:15:23 -0600 Subject: [PATCH 3/7] docs: replace header mark and favicon with official Azure Functions logo Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/site/assets/app.js | 2 +- docs/site/assets/function-apps.svg | 1 + docs/site/assets/styles.css | 15 +++++---------- docs/site/index.html | 1 + 4 files changed, 8 insertions(+), 11 deletions(-) create mode 100644 docs/site/assets/function-apps.svg diff --git a/docs/site/assets/app.js b/docs/site/assets/app.js index 5111cd16f..d00df2166 100644 --- a/docs/site/assets/app.js +++ b/docs/site/assets/app.js @@ -133,7 +133,7 @@
-
λ
+
Azure Functions Python Runtime Architecture Atlas diff --git a/docs/site/assets/function-apps.svg b/docs/site/assets/function-apps.svg new file mode 100644 index 000000000..00ec83860 --- /dev/null +++ b/docs/site/assets/function-apps.svg @@ -0,0 +1 @@ +Icon-compute-29 diff --git a/docs/site/assets/styles.css b/docs/site/assets/styles.css index 6030d81bb..6b4c3503a 100644 --- a/docs/site/assets/styles.css +++ b/docs/site/assets/styles.css @@ -67,16 +67,11 @@ body { gap: 10px; } -.brand-mark { - width: 28px; - height: 28px; - border-radius: 7px; - background: linear-gradient(135deg, #0ea5e9, #fb923c); - display: grid; - place-items: center; - font-weight: 800; - font-size: 13px; - color: #070b14; +.brand-logo { + width: 26px; + height: 26px; + display: block; + flex-shrink: 0; } .brand-title { diff --git a/docs/site/index.html b/docs/site/index.html index eed571757..0476f9340 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -4,6 +4,7 @@ Azure Functions Python Runtime — Architecture Atlas + From 708f8aec349dfdf9bee66c8d4c4c56f10cb40ee7 Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Sun, 15 Nov 2026 19:25:30 -0600 Subject: [PATCH 4/7] fix(docs): address PR review feedback on accessibility, zoom, listeners, layout, and AST method anchors - Add keyboard focus, Enter/Space activation, ARIA labels,
`; + `; }) .join("")} ${outbound .map((e) => { const tn = era.nodes.find((x) => x.id === e.target); - return `
+ return `
`; + `; }) .join("")}
@@ -988,7 +1002,38 @@ `; } + function captureFocusDescriptor() { + const a = document.activeElement; + if (!a || a === document.body) return null; + if (a.id) return { kind: "id", val: a.id }; + for (const attr of ["data-node", "data-edge", "data-seq-step", "data-phase", "data-mode", "data-nav"]) { + if (a.hasAttribute && a.hasAttribute(attr)) { + return { kind: "attr", attr, val: a.getAttribute(attr) }; + } + } + if (a.hasAttribute && a.hasAttribute("data-boundary-node")) { + return { kind: "attr", attr: "data-node", val: a.getAttribute("data-boundary-node") }; + } + return null; + } + + function restoreFocusDescriptor(desc) { + if (!desc) return; + const target = + desc.kind === "id" + ? document.getElementById(desc.val) + : document.querySelector(`[${desc.attr}="${CSS.escape(desc.val)}"]`); + if (target && typeof target.focus === "function") { + target.focus({ preventScroll: true }); + } + } + function render() { + if (stageListenersAbort) { + stageListenersAbort.abort(); + stageListenersAbort = null; + } + const focusDesc = captureFocusDescriptor(); const root = document.getElementById("root"); const era = currentEra(); let mainHtml = ""; @@ -1017,6 +1062,7 @@ fitDiagramToStage(era); } bindEvents(); + restoreFocusDescriptor(focusDesc); } function bindEvents() { @@ -1100,34 +1146,69 @@ const reset = document.getElementById("step-reset"); if (reset) reset.addEventListener("click", () => { state.stepIdx = -1; render(); }); + function onKeyActivate(el, fn) { + el.addEventListener("keydown", (ev) => { + if (ev.key === "Enter" || ev.key === " ") { + ev.preventDefault(); + ev.stopPropagation(); + fn(ev); + } + }); + } + document.querySelectorAll("[data-node]").forEach((el) => { - el.addEventListener("click", (ev) => { + const selectNode = (ev) => { ev.stopPropagation(); state.selectedNodeId = el.getAttribute("data-node"); state.selectedEdgeId = null; render(); + }; + el.addEventListener("click", selectNode); + onKeyActivate(el, selectNode); + }); + + document.querySelectorAll("[data-boundary-node]").forEach((el) => { + el.addEventListener("click", (ev) => { + ev.stopPropagation(); + state.selectedNodeId = el.getAttribute("data-boundary-node"); + state.selectedEdgeId = null; + render(); }); }); document.querySelectorAll("[data-edge]").forEach((el) => { - el.addEventListener("click", (ev) => { + const selectEdge = (ev) => { ev.stopPropagation(); state.selectedEdgeId = el.getAttribute("data-edge"); render(); - }); + }; + el.addEventListener("click", selectEdge); + onKeyActivate(el, selectEdge); }); document.querySelectorAll("[data-seq-step]").forEach((el) => { - el.addEventListener("click", () => { + const selectSeq = () => { state.stepIdx = Number(el.getAttribute("data-seq-step")); render(); - }); + }; + el.addEventListener("click", selectSeq); + onKeyActivate(el, selectSeq); }); const zin = document.getElementById("btn-zoom-in"); - if (zin) zin.addEventListener("click", () => { state.zoom = Math.min(1.6, +(state.zoom + 0.12).toFixed(2)); render(); }); + if (zin) { + zin.addEventListener("click", () => { + state.zoom = Math.min(ZOOM_MAX, Math.max(ZOOM_MIN, +(state.zoom + ZOOM_STEP).toFixed(2))); + render(); + }); + } const zout = document.getElementById("btn-zoom-out"); - if (zout) zout.addEventListener("click", () => { state.zoom = Math.max(0.35, +(state.zoom - 0.12).toFixed(2)); render(); }); + if (zout) { + zout.addEventListener("click", () => { + state.zoom = Math.max(ZOOM_MIN, Math.min(ZOOM_MAX, +(state.zoom - ZOOM_STEP).toFixed(2))); + render(); + }); + } const zfit = document.getElementById("btn-zoom-fit"); if (zfit) zfit.addEventListener("click", () => { fitDiagramToStage(currentEra()); render(); }); const bOrient = document.getElementById("btn-orient"); @@ -1154,10 +1235,12 @@ }); } - // Drag-to-pan + wheel-to-zoom on topology stage + // Drag-to-pan + wheel-to-zoom on topology stage (cleaned up via AbortController on rerender) const stage = document.getElementById("diagram-stage"); const vp = document.getElementById("viewport-g"); if (stage && vp) { + stageListenersAbort = new AbortController(); + const { signal } = stageListenersAbort; let dragging = false; let sx = 0, sy = 0, ox = state.panX, oy = state.panY; stage.addEventListener("mousedown", (ev) => { @@ -1168,19 +1251,23 @@ ox = state.panX; oy = state.panY; }); - window.addEventListener("mousemove", (ev) => { - if (!dragging) return; - state.panX = ox + (ev.clientX - sx); - state.panY = oy + (ev.clientY - sy); - vp.setAttribute("transform", `translate(${state.panX}, ${state.panY}) scale(${state.zoom})`); - }); - window.addEventListener("mouseup", () => { dragging = false; }); + window.addEventListener( + "mousemove", + (ev) => { + if (!dragging) return; + state.panX = ox + (ev.clientX - sx); + state.panY = oy + (ev.clientY - sy); + vp.setAttribute("transform", `translate(${state.panX}, ${state.panY}) scale(${state.zoom})`); + }, + { signal } + ); + window.addEventListener("mouseup", () => { dragging = false; }, { signal }); stage.addEventListener( "wheel", (ev) => { ev.preventDefault(); const factor = ev.deltaY < 0 ? 1.07 : 0.93; - state.zoom = Math.max(0.32, Math.min(1.8, +(state.zoom * factor).toFixed(3))); + state.zoom = Math.max(ZOOM_MIN, Math.min(ZOOM_MAX, +(state.zoom * factor).toFixed(3))); vp.setAttribute("transform", `translate(${state.panX}, ${state.panY}) scale(${state.zoom})`); }, { passive: false } diff --git a/docs/site/assets/styles.css b/docs/site/assets/styles.css index 6b4c3503a..d1afd3697 100644 --- a/docs/site/assets/styles.css +++ b/docs/site/assets/styles.css @@ -41,7 +41,7 @@ body { } .app-shell.sidebar-collapsed { - grid-template-columns: 0px 1fr; + grid-template-columns: 1fr; } .app-shell.sidebar-collapsed .sidebar { @@ -525,6 +525,9 @@ body { justify-content: space-between; align-items: center; gap: 6px; + width: 100%; + text-align: left; + cursor: pointer; padding: 5px 8px; border-radius: 6px; background: rgba(7, 11, 20, 0.65); @@ -539,6 +542,16 @@ body { border-color: var(--accent); } +.svg-node:focus-visible, +.svg-edge:focus-visible, +[data-seq-step]:focus-visible, +button:focus-visible, +a:focus-visible, +select:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} + .sym-line { color: var(--text-dim); flex-shrink: 0; diff --git a/eng/docs/extract_arch.py b/eng/docs/extract_arch.py index 354faeda5..01b3522c2 100644 --- a/eng/docs/extract_arch.py +++ b/eng/docs/extract_arch.py @@ -114,11 +114,19 @@ def run(args: list[str], default: str = "") -> str: def extract_python_file(rel_path: str, base_dir: Path = REPO_ROOT) -> dict[str, Any]: full = base_dir / rel_path if not full.exists(): - return {"path": rel_path, "exists": False, "lines": 0, "symbols": [], "routerCases": []} + return { + "path": rel_path, + "exists": False, + "lines": 0, + "symbols": [], + "symbolLines": {}, + "routerCases": [], + } src = full.read_text(encoding="utf-8", errors="replace") lines = src.count("\n") + 1 symbols: list[dict[str, Any]] = [] + symbol_lines: dict[str, int] = {} match_cases: list[str] = [] try: @@ -126,6 +134,7 @@ def extract_python_file(rel_path: str, base_dir: Path = REPO_ROOT) -> dict[str, doc = ast.get_docstring(tree) or "" for node in tree.body: if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbol_lines.setdefault(node.name, node.lineno) args = [a.arg for a in node.args.args if a.arg != "self"] symbols.append({ "name": node.name, @@ -134,10 +143,14 @@ def extract_python_file(rel_path: str, base_dir: Path = REPO_ROOT) -> dict[str, "signature": f"{node.name}({', '.join(args[:4])}{', ...' if len(args) > 4 else ''})", }) elif isinstance(node, ast.ClassDef): - methods = [ - n.name for n in node.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and not n.name.startswith("__") - ] + symbol_lines.setdefault(node.name, node.lineno) + methods: list[str] = [] + for n in node.body: + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbol_lines[f"{node.name}.{n.name}"] = n.lineno + symbol_lines.setdefault(n.name, n.lineno) + if not n.name.startswith("__"): + methods.append(n.name) symbols.append({ "name": node.name, "kind": "class", @@ -160,7 +173,8 @@ def extract_python_file(rel_path: str, base_dir: Path = REPO_ROOT) -> dict[str, "exists": True, "lines": lines, "docSummary": doc.strip().splitlines()[0] if doc.strip() else "", - "symbols": symbols[:14], + "symbols": symbols, + "symbolLines": symbol_lines, "routerCases": sorted(set(match_cases)), } @@ -168,11 +182,19 @@ def extract_python_file(rel_path: str, base_dir: Path = REPO_ROOT) -> dict[str, def extract_rust_file(rel_path: str) -> dict[str, Any]: full = REPO_ROOT / rel_path if not full.exists(): - return {"path": rel_path, "exists": False, "lines": 0, "symbols": [], "pyo3Calls": []} + return { + "path": rel_path, + "exists": False, + "lines": 0, + "symbols": [], + "symbolLines": {}, + "pyo3Calls": [], + } src = full.read_text(encoding="utf-8", errors="replace") src_lines = src.splitlines() symbols: list[dict[str, Any]] = [] + symbol_lines: dict[str, int] = {} pyo3_calls: list[str] = [] fn_re = re.compile(r"^\s*(pub(?:\(crate\))?\s+)?(async\s+)?fn\s+([A-Za-z0-9_]+)\s*\(([^)]*)\)") @@ -185,6 +207,7 @@ def extract_rust_file(rel_path: str) -> dict[str, Any]: is_pub, is_async, name, params = m_fn.groups() if name.startswith("test_"): continue + symbol_lines.setdefault(name, idx) kind = f"{'pub ' if is_pub else ''}{'async ' if is_async else ''}fn".strip() short_params = ", ".join( p.strip().split(":")[0].strip() for p in params.split(",") if p.strip() @@ -199,6 +222,7 @@ def extract_rust_file(rel_path: str) -> dict[str, Any]: m_st = struct_re.match(line) if m_st: _, kw, name = m_st.groups() + symbol_lines.setdefault(name, idx) symbols.append({"name": name, "kind": kw, "line": idx, "signature": name}) for m in call_method_re.finditer(src): @@ -208,7 +232,8 @@ def extract_rust_file(rel_path: str) -> dict[str, Any]: "path": rel_path, "exists": True, "lines": len(src_lines), - "symbols": symbols[:16], + "symbols": symbols, + "symbolLines": symbol_lines, "pyo3Calls": sorted(set(pyo3_calls)), } @@ -248,6 +273,9 @@ def scan_package_files(rel_dir: str, suffixes: tuple[str, ...]) -> list[dict[str def sym_line(meta: dict[str, Any], name: str, fallback: int = 1) -> int: + sym_map = meta.get("symbolLines", {}) + if name in sym_map: + return sym_map[name] for s in meta.get("symbols", []): if s["name"] == name: return s["line"] diff --git a/eng/docs/tests/test_extract_arch.py b/eng/docs/tests/test_extract_arch.py index fc6c6b9b9..8e706cd84 100644 --- a/eng/docs/tests/test_extract_arch.py +++ b/eng/docs/tests/test_extract_arch.py @@ -81,6 +81,19 @@ def test_sibling_root_extracts_external_python_symbols(self) -> None: sym_names = [s["name"] for s in datum_node["extractedSymbols"]] self.assertIn("Datum", sym_names) + def test_class_method_line_lookup_avoids_fallback(self) -> None: + proxy_meta = extract_arch.extract_python_file( + "workers/proxy_worker/dispatcher.py" + ) + mono_meta = extract_arch.extract_python_file( + "workers/azure_functions_worker/dispatcher.py" + ) + self.assertGreater(extract_arch.sym_line(proxy_meta, "__poll_grpc", -1), 1) + self.assertGreater( + extract_arch.sym_line(proxy_meta, "_dispatch_grpc_request", -1), 1 + ) + self.assertGreater(extract_arch.sym_line(mono_meta, "connect", -1), 1) + def test_validate_graph_rejects_dangling_phase_edge(self) -> None: broken = copy.deepcopy(self.graph) py315 = next(e for e in broken["eras"] if e["id"] == "py315_r2p2") From 145255c7eec3f659068c2230a792202f731031ad Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Sun, 15 Nov 2026 19:30:07 -0600 Subject: [PATCH 5/7] docs: separate Azure/azure-functions-host runtime from worker-protobuf contract Add Azure/azure-functions-host as the 5th linked repository across the ecosystem topology and all 3 worker generations so the .NET Script Host runtime (WorkerChannel / ServerDuplexChannel) is cleanly distinguished from the shared FunctionRpc.proto wire contract. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/architecture-pages.yml | 8 + docs/site/README.md | 11 +- docs/site/assets/app.js | 6 +- eng/docs/extract_arch.py | 248 +++++++++++++++++------ eng/docs/tests/test_extract_arch.py | 3 +- 5 files changed, 200 insertions(+), 76 deletions(-) diff --git a/.github/workflows/architecture-pages.yml b/.github/workflows/architecture-pages.yml index e2d273adc..2b5830038 100644 --- a/.github/workflows/architecture-pages.yml +++ b/.github/workflows/architecture-pages.yml @@ -39,6 +39,14 @@ jobs: - name: Checkout python-worker uses: actions/checkout@v4 + - name: Checkout azure-functions-host (gRPC channel metadata) + uses: actions/checkout@v4 + with: + repository: Azure/azure-functions-host + path: _sibling_repos/functions-host + depth: 1 + sparse-checkout: src/WebJobs.Script.Grpc + - name: Checkout azure-functions-python-library uses: actions/checkout@v4 with: diff --git a/docs/site/README.md b/docs/site/README.md index 45b123674..da02668ef 100644 --- a/docs/site/README.md +++ b/docs/site/README.md @@ -2,11 +2,12 @@ Interactive GitHub Pages site documenting the Azure Functions Python runtime architecture across: -- **4 Linked Repositories**: - - [`Azure/azure-functions-python-worker`](https://github.com/Azure/azure-functions-python-worker) - - [`Azure/azure-functions-python-library`](https://github.com/Azure/azure-functions-python-library) - - [`Azure/azure-functions-python-extensions`](https://github.com/Azure/azure-functions-python-extensions) - - [`Azure/azure-functions-language-worker-protobuf`](https://github.com/Azure/azure-functions-language-worker-protobuf) +- **5 Linked Repositories**: + - [`Azure/azure-functions-host`](https://github.com/Azure/azure-functions-host) (.NET Script Host runtime) + - [`Azure/azure-functions-language-worker-protobuf`](https://github.com/Azure/azure-functions-language-worker-protobuf) (Host ↔ Worker `.proto` contract) + - [`Azure/azure-functions-python-worker`](https://github.com/Azure/azure-functions-python-worker) (Worker shells + V1/V2 runtimes) + - [`Azure/azure-functions-python-library`](https://github.com/Azure/azure-functions-python-library) (`azure-functions` SDK) + - [`Azure/azure-functions-python-extensions`](https://github.com/Azure/azure-functions-python-extensions) (HTTP v2, `SdkType` bindings, Agent runtimes) - **3 Worker Generations**: Python `≤ 3.12` (`workers/azure_functions_worker`), Python `3.13–3.14` (`workers/proxy_worker` + `runtimes/`), and Python `3.15+` (`workers/r2p2` + `PyO3` + `runtimes/`). - **7 Lifecycle Phases per Generation**: Worker Init, Specialization (Consumption), Metadata & Indexing, Functions Load (V1), Invocation, Worker Status, and Worker Logs (`RpcLog`). diff --git a/docs/site/assets/app.js b/docs/site/assets/app.js index 91139153e..a435dfa7a 100644 --- a/docs/site/assets/app.js +++ b/docs/site/assets/app.js @@ -147,7 +147,7 @@