diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cdb60c..380ebc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,26 @@ jobs: - name: Test run: go test ./... -count=1 + # The release binary is built with the latest Go (release.yml), so test on + # it too; the job above tests the go.mod minimum. + test-stable: + name: Test (Go stable) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: stable + check-latest: true + + - name: Build + run: go build ./... + + - name: Test + run: go test ./... -count=1 + lint: name: Lint runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3f62f23..ff218c5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,10 +45,13 @@ jobs: echo "prerelease=$prerelease" >> "$GITHUB_OUTPUT" echo ">>> Releasing version $version (prerelease=$prerelease)" + # Build with the latest Go, not the go.mod minimum: the analyzer + # type-checks targets with the go/types compiled into it, so a binary + # built with an older Go degrades on modules with a newer go line. - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.25" + go-version: stable check-latest: true # ----- test gate: a broken build must not produce a release ----- diff --git a/.gitignore b/.gitignore index 39711d4..e0379be 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,7 @@ go.work.sum # Generated analysis output /output/ + +# Local planning artifacts (agent plans/specs scratch); not part of the repo +docs/superpowers/ +.superpowers/ diff --git a/CLAUDE.md b/CLAUDE.md index 40e2964..b918fbf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,9 +12,11 @@ v2 vocabulary (parity clause: add at the leaves, never rename shared names). The committed spec is the full transcript: `docs/design/specs/v2-l1-emission.md`. ### Identity & positioning (Group A — reused verbatim by L3/L4) -- **`application.id`** = `can://go/`, where `` is the **`--app-name`** value, - defaulting to the input directory's base name (per the CLI contract; the SDK's Neo4j - backend must use the same anchor). L3/L4 build ids on this unchanged. +- **`application.id`** = `can://` (app outermost, so a polyglot app shares one prefix), + where `` is the **`--app-name`** value, defaulting to the input directory's base name + (per the CLI contract; the SDK's Neo4j backend must use the same anchor). Module ids are + `can:///go/`; externals are language-neutral `can:///@external//`. + L3/L4 build ids on this unchanged. - **Callable signature** = existing `signatureOf()` output as the last `can://` path segment; receiver folded in for methods. One canonicalizer, unchanged from v1. - **`span.bytes`** = **UTF-8 byte offsets** into `module.source` (Go strings are @@ -43,7 +45,10 @@ committed spec is the full transcript: `docs/design/specs/v2-l1-emission.md`. the declaring file ≠ the nesting module, the callable carries `source_file` = that file's path (a `symbol_table` key). **Text recovery:** a node's text = `symbol_table[source_file].source[span.bytes]` if `source_file` present, else the - nesting module's `source`. Emitted only when it differs (absent = same file). This is + nesting module's `source`. Emitted only when it differs (absent = same file). It applies + to **every callable** whose declaring file differs from its containing module, so + **closures and nested closures** inside a cross-file method carry `source_file` too (no + implicit inheritance; text recovery holds as written for them and their body nodes). This is the leaf-level fix for the cross-file-method empty-span collision the real-app eval surfaced. Cross-language parity (C# partial classes, Rust `impl`, Ruby reopened classes, TS declaration merging) is **deferred** — promote `source_file` into the @@ -54,10 +59,46 @@ committed spec is the full transcript: `docs/design/specs/v2-l1-emission.md`. v1). `is_constructor_call` **dropped** — Go has no constructors. - `callee` is the sanctioned `null → id` refinement slot (null at L1, backfilled at L2). +### `external_symbols` (L2 library calls) +- **L2 keeps library call edges.** A call to a function outside the project (`fmt.Println`, + `(*strings.Builder).WriteString`) is no longer dropped: the call node's `callee` and the + `call_graph` edge `dst` are the callee's `@external` id. Builtins, type conversions + (no longer minted as `.__new__`) and calls through function values stay `null`. +- **Id** = `can:///@external//` where `` is the callee signature + minus the `.` prefix (`Println`, `Builder.WriteString`); it may also name a + library *interface* method (`@external/io/Writer.Write`). Methods on generic library types + record the declared (origin) signature, not the instance. `@external` is + language-neutral (not `go/`), so a polyglot app shares one external namespace. +- **Home:** `application.external_symbols{: {id, kind:"external", name, module, + signature, parameters, return_type}}`, omitempty; parameters come from the callee's + `*types.Signature` (receiver excluded, unnamed params `_`). +- Mirrors the Python/Java precedent (library callees are leaf nodes at application scope); + on by default at L2, no flag. + +### L3 body and edges (`-a 3`; spec `docs/design/specs/v2-l3-l4-dataflow.md`) +- **Body node `id`** = `@` (`v2.GlobalOrdinal`) on **every** body node, + **from `-a 1` on** (not L3+ only) — Python parity (codeanalyzer-python stamps body ids at + L1). **`parent`** (L3 only) = the local key of the statement a satellite call sits in. +- **Body kinds: shared vocabulary only, no Go-specific kinds** — `if` → `branch`; + `for`/`range` → `loop`; `switch`/type switch/`select` → `switch`; `return`; every other + statement and each case/comm-clause head → `statement`; L1 `call`; synthetic `entry`/`exit`. + `go`/`defer`/`select` facts ride `is_goroutine`/`is_deferred` and the `switch` kind. +- **`@entry` / `@exit` are span-less** (keystone: synthetic CFG endpoints have no span) and + carry no `callee` (a call-node field). +- **Defer two-node rule:** `defer f(x)` is a registration `statement` node (at the `defer` + keyword; evaluates `f`, `x` and weak-defines `:`) **plus** the L1 deferred + `call` node (`is_deferred`), placed only in the exit chain (reverse source order before + `@exit`; every defer assumed registered). `go f(x)` is NOT split — the statement is its call + node. +- **`cfg` kind `defer_resume`** — the one Go add to the shared `cfg` kinds: chain-internal + edges and the chain tail → `@exit`. The edge into the chain keeps its own kind. +- **`k_limit`** (envelope, L3+ only) = `--graph-field-depth` (default 3), the access-path + bound of the syntactic `ddg` (`x.f.g.h` → `x.f.g.*`); `ddg.prov` is `["ssa"]` (parity). + ### Deferred (not this train) -- L3 (CFG/CDG/DDG) and L4 (SDG param edges) — reuse Group A ids/spans unchanged. -- Go-specific `cfg` edge kinds (e.g. `defer_resume`, `select`) are recorded when L3 - is designed, not now. +- L4 (SDG param edges) — reuses Group A ids/spans and the L3 vocabulary unchanged. Designed in + `docs/design/specs/v2-l3-l4-dataflow.md`. A Go-specific `select` cfg kind was not added + (`select` is a `switch` with `switch_case` edges). ## Neo4j-projection decisions @@ -107,6 +148,13 @@ shared neutral vocabulary verbatim; never rename a sibling's shared name). Spec: - **`is_goroutine` / `is_deferred`** → boolean props on the call `:GoBodyNode` (pruned when false, matching JSON `omitempty`), not labels, not edge discriminants (the fact lives on the call site even when the callee is unresolved and no edge exists). +- **`:GoBodyNode.id`** = the JSON body node's `id` (`@`), which the JSON + carries **from `-a 1` on** (not L3+ only) — Python parity: codeanalyzer-python stamps body + ids at L1 (`stamp_body_ids`), so JSON and graph join on the same id at every level. +- **`call_node`** on a satellite call `:GoBodyNode` = the enclosing statement's **local** key + (JSON `parent` verbatim), resolved against the owning callable (the `GO_HAS_BODY_NODE` + source) — not a global id. Python parity: codeanalyzer-python projects `node.parent` + unchanged and python-sdk's `reconstruct` maps `call_node` back to `parent`. - **`source_file`** (cross-file method, § callable above) → scalar prop on `:GoCallable`; text recovery joins to the declaring `:GoModule` by that path. Present only when it differs from the nesting module. @@ -123,16 +171,25 @@ shared neutral vocabulary verbatim; never rename a sibling's shared name). Spec: own resolution claims). ### `_k` relationship discriminants (parallel edges survive MERGE) -- `GO_CFG_NEXT` per `kind`; `GO_DDG` per `(var, prov)`; `DECLARES_DEPENDENCY` per `kind` - — same discriminant mechanism as the siblings. - -### Reserved L3/L4 overlay (declared now, emits nothing until the levels land) -- `schema.go` declares the full overlay vocabulary **now** — `GO_HAS_BODY_NODE`, - `GO_CFG_NEXT`, `GO_CDG`, `GO_DDG`, `GO_PARAM_IN`, `GO_PARAM_OUT`, `GO_SUMMARY`, and the - body-node statement kinds — with parity-correct names and `_k` discriminants. The - analyzer caps at L2, so the projector walks these sections and emits **zero rows**; - when L3/L4 land they fill with **no schema-contract change**. (`--emit neo4j` is - "full-depth = max implemented" — L2 today; the graph grows additively.) +- `GO_CFG_NEXT` per `kind`; `GO_DDG` per `(var, prov)` as the composite + `_k = var + "|" + join(prov, ",")` (declared as `Discriminant: "var|prov"`); + `DECLARES_DEPENDENCY` per `kind` — same discriminant mechanism as the siblings. + +### L3 overlay (live at `-a 3`) +- `GO_CFG_NEXT {kind}`, `GO_CDG`, `GO_DDG {var, prov}` are no longer reserved; the projector + emits one row per JSON `cfg[]`/`cdg[]`/`ddg[]` entry (cross-projection count gate). + Endpoints are `:GoBodyNode` **global ids** (`v2.GlobalOrdinal(callable-id, local-key)`), + never recomposed by hand. `prov` is a string-array prop. +- `:GoBodyNode` covers every body node incl. synthetic `@entry`/`@exit` (id + `@entry`, no `span_json`), each with `GO_HAS_BODY_NODE` from its callable. + It carries `call_node` (JSON `parent` verbatim — the **local** key, see § Flattened + properties) and `var` (JSON `of`; L4 — absent at L3). `GO_HAS_BODY_NODE` may originate at `:GoExternal` (L4 stand-ins). + +### Reserved L4 overlay (declared now, emits nothing until L4 lands) +- `schema.go` still declares `GO_PARAM_IN`, `GO_PARAM_OUT`, `GO_SUMMARY` (`Reserved`) with + parity-correct names; the projector emits **zero rows** for them. When L4 lands they fill + with **no schema-contract change**. (`--emit neo4j` is "full-depth = max implemented" — + L3 today; the graph grows additively.) ### SDK facade (`GoAnalysisBackend`, on the `python-sdk` fork — frontend rung) - **Type-centric surface** (Go is the SDK's first class-less facade; no C precedent exists @@ -143,9 +200,12 @@ shared neutral vocabulary verbatim; never rename a sibling's shared name). Spec: take the callable, not class+method strings). - **Scope this train:** Tier-A lifecycle + L1/L2 leaf accessors + repository-artifact getters + `get_source`. Slicing / CFG / CDG / DDG / reachability are **omitted** (they - need L3/L4 the analyzer does not produce) per the ABC contract. + need L3/L4 the analyzer does not produce) per the ABC contract. (2026-10-24: L3 is now + emitted at `-a 3` — per-callable `cfg`/`cdg`/`ddg` and `GO_CFG_NEXT`/`GO_CDG`/`GO_DDG` — + so L3 accessors are supportable now; interprocedural slicing and taint still need L4: + `param_in`/`param_out`/`summary` and the points-to `ddg`.) ### Deferred (not this train) - SDK consumer slicing/taint surface and L3/L4 facade methods — arrive with the L3/L4 analyzer train, reusing this graph vocabulary unchanged. -- Struct-tag modeling; Go-specific `cfg` edge kinds (`defer_resume`, `select`) — with L3. +- Struct-tag modeling. (`defer_resume` landed with L3; see § L3 body and edges.) diff --git a/README.md b/README.md index 194c415..7050aa0 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ The binary is fully self-contained: a single static executable with no runtime d - [Install via Homebrew](#install-via-homebrew) - [Install via pip (PyPI)](#install-via-pip-pypi) - [Build from source](#build-from-source) + - [Toolchain](#toolchain) - [Usage](#usage) - [Command-line options](#command-line-options) - [Examples](#examples) @@ -46,6 +47,7 @@ The binary is fully self-contained: a single static executable with no runtime d - [Load and query](#load-and-query) - [Query reference](#query-reference) - [Python SDK (CLDK) integration](#python-sdk-cldk-integration) +- [Upgrading](#upgrading) - [Architecture & Tooling](#architecture--tooling) - [Development](#development) - [License](#license) @@ -55,7 +57,9 @@ The binary is fully self-contained: a single static executable with no runtime d - **Symbol table** — packages, structs, interfaces, fields, methods, package-level functions, imports, struct tags, and generic type parameters, with precise source spans. - **Call graph** — a resolver-based call graph via `go/types`: each call site is resolved to its - full import-path signature, with project-internal edges emitted (Level 2). + full import-path signature. Level 2 emits edges to project callees and to library callees + (homed in `application.external_symbols` under `can:///@external//`). + Level 3 (`-a 3`) adds per-callable `cfg`, `cdg` and `ddg`. - **Generics-aware** — Go 1.18+ generics (`Set[T]`, union-constraint interfaces, multi-type-param functions) are modeled in the unified type/callable schema. - **Self-contained binary** — a single static executable (`go build`, `CGO_ENABLED=0`); no runtime @@ -72,7 +76,8 @@ The binary is fully self-contained: a single static executable with no runtime d Running a prebuilt `cango` binary requires **nothing** — it is fully self-contained. To *analyze* a project, that project should be a normal Go module (contain a `go.mod`) so the type checker can -resolve imports. Building `cango` from source requires [Go 1.25+](https://go.dev/dl/). +resolve imports. Building `cango` from source requires [Go 1.26+](https://go.dev/dl/), and the +latest Go release is recommended — see [Toolchain](#toolchain). ### Install via shell script @@ -120,6 +125,25 @@ analyzer directly from source without compiling: go run ./cmd/codeanalyzer -i /path/to/project -a 2 ``` +### Toolchain + +`cango` type-checks the projects it analyzes with the `go/types` package compiled into the +binary, so **the Go that builds `cango` must be at least as new as the newest `go` line of the +projects it analyzes**. Build with the latest Go release (the release binaries are). `go.mod`'s +`go 1.26.0` is only the minimum that compiles the analyzer (set by `golang.org/x/tools`, whose +recent releases require Go 1.26). + +With a binary built by an older Go, every package that needs a newer Go type-checks only +partially, yet the run still exits 0: callees go unresolved, call edges and L3 defs are lost, +and nothing else looks wrong. `cango` prints one warning when this happens, naming the Go it was +built with, the highest version required and the number of affected packages: + +``` +[codeanalyzer-go WARN] 12 packages require a newer Go version (highest go1.27) than this analyzer was built with (go1.26.4); … +``` + +Rebuild `cango` with that Go version or newer. (`-vv` lists the affected packages.) + ## Usage ```bash @@ -138,12 +162,13 @@ Aliases: cango, codeanalyzer-go Flags: - -a, --analysis-level int Analysis level: 1=symbol table only, 2=+resolver call graph (default 1) + -a, --analysis-level int Analysis level: 1=symbol table, 2=+call graph, 3=+intraprocedural cfg/cdg/ddg (default 1) --app-name string Application anchor name for can:// ids and Neo4j :Application (default: input dir name) -c, --cache-dir string Cache directory (default: ~/.cldk/go-cache) --eager Force clean rebuild (ignore cache) --emit string Output projection: json|neo4j|schema (default "json") -f, --format string Output format: json|msgpack (default "json") + --graph-field-depth int Field-path depth bound for L3 dataflow nodes (k limit; applies at -a 3 only) (default 3) -h, --help help for cango -i, --input string Project root to analyze (required) -j, --jobs int Worker parallelism (default: CPU cores) @@ -221,7 +246,7 @@ cango -i /path/to/go/app -a 2 -o /path/to/output/ > of writing a file. **3. (Optional) name the application anchor.** `--app-name` sets the `` in every -`can://go//…` id and defaults to the input directory's base name. Set it when the +`can:///…` id and defaults to the input directory's base name. Set it when the directory name isn't the identity you want: ```bash @@ -249,10 +274,22 @@ are build artifacts, not project source), so only your own declarations are emit |-------|------|-----------|--------| | 1 | `-a 1` (default) | Symbol table only — types, functions, call sites | Implemented | | 2 | `-a 2` | Level 1 + resolver-based call graph via `go/types` | Implemented | +| 3 | `-a 3` | Level 2 + per-callable statement nodes and `cfg` / `cdg` / `ddg` | Implemented | +| 4 | `-a 4` | Level 3 + interprocedural parameter edges | Not implemented (rejected) | **Level 1** loads each package with `packages.NeedSyntax | NeedTypes | NeedTypesInfo` and walks the AST file by file. Call sites are recorded with `callee_signature = null` at this stage. -**Level 2** adds a resolver pass: for each call site, `go/types` resolves the callee to its full import-path signature (`pkgImportPath.TypeName.MethodName`). Only project-internal edges (both endpoints present in the symbol table) are emitted. `callee_signature` is backfilled on all successfully resolved sites. +**Level 3** completes every callable's `body` with its statement, control (`branch`, `loop`, +`switch`, `return`) and synthetic (`@entry`, `@exit`) nodes around the L1 call nodes, and adds +three intra-callable edge lists: `cfg` (control flow, including the `defer` chain and panic +`exception` edges), `cdg` (control dependence from post-dominators) and `ddg` (syntactic +reaching definitions over k-limited access paths, `prov: ["ssa"]`). `--graph-field-depth ` +(default 3, minimum 1) bounds the access paths (`x.f.g.h` → `x.f.g.*` at k = 3) and is +recorded in the envelope as `k_limit`. It is a flag error below `-a 3` and with +`--emit neo4j`. Callables are analyzed in parallel (`-j`), and the output is byte-identical +for every `-j`. L3 only adds to the L2 output: every L2 fact stays as it is. + +**Level 2** adds a resolver pass: for each call site, `go/types` resolves the callee to its full import-path signature (`pkgImportPath.TypeName.MethodName`). A project callee becomes an edge to its callable; a **library callee** (stdlib, a dependency, or a package vendored under `vendor/`) becomes an edge to an `@external` symbol, recorded once in `application.external_symbols`. A callee that is neither (an interface method of a project type, a conversion, a builtin, a call through a function value) gets no edge. Every resolved call node's `callee` is backfilled. ## Output schema @@ -262,7 +299,7 @@ and the [step-by-step section above](#generating-analysisjson-for-a-go-app) prod ### Canonical v2 shape The document is a manifest envelope wrapping one `application` containment tree. Every -node carries a `can://go//…` `id`, a `kind`, and a `span`: +node carries a `can:///…` `id`, a `kind`, and a `span`: ```json { @@ -271,11 +308,11 @@ node carries a `can://go//…` `id`, a `kind`, and a `span`: "max_level": 2, "analyzer": { "name": "codeanalyzer-go", "version": "0.1.0" }, "application": { - "id": "can://go/myapp", + "id": "can://myapp", "kind": "application", "symbol_table": { "pkg/greeter/greeter.go": { - "id": "can://go/myapp/pkg/greeter/greeter.go", + "id": "can://myapp/go/pkg/greeter/greeter.go", "kind": "module", "package": "greeter", "span": { "start": [1, 1], "end": [40, 2], "bytes": [0, 812] }, @@ -284,7 +321,7 @@ node carries a `can://go//…` `id`, a `kind`, and a `span`: "imports": [ { "name": "fmt", "path": "fmt", "span": {…} } ], "types": { "Greeter": { - "id": "can://go/myapp/pkg/greeter/greeter.go/example.com/pkg/greeter.Greeter", + "id": "can://myapp/go/pkg/greeter/greeter.go/example.com/pkg/greeter.Greeter", "kind": "struct", "span": { "start": [5, 1], "end": [7, 2], "bytes": [17, 63] }, "fields": { @@ -292,7 +329,7 @@ node carries a `can://go//…` `id`, a `kind`, and a `span`: }, "callables": { "example.com/pkg/greeter.Greeter.Greet": { - "id": "can://go/myapp/pkg/greeter/greeter.go/example.com/pkg/greeter.Greeter/example.com/pkg/greeter.Greeter.Greet", + "id": "can://myapp/go/pkg/greeter/greeter.go/example.com/pkg/greeter.Greeter/example.com/pkg/greeter.Greeter.Greet", "kind": "method", "signature": "example.com/pkg/greeter.Greeter.Greet", "span": { "start": [9, 1], "end": [11, 2], "bytes": [65, 140] }, @@ -303,8 +340,9 @@ node carries a `can://go//…` `id`, a `kind`, and a `span`: "body": { "10:2": { "kind": "call", + "id": "can://myapp/go/pkg/greeter/greeter.go/…/example.com/pkg/greeter.Greeter.Greet@10:2", "span": { "start": [10, 2], "end": [10, 30], "bytes": [90, 118] }, - "callee": "can://go/myapp/pkg/greeter/greeter.go/…/example.com/pkg/greeter.format" + "callee": "can://myapp/go/pkg/greeter/greeter.go/…/example.com/pkg/greeter.format" } } } @@ -318,8 +356,8 @@ node carries a `can://go//…` `id`, a `kind`, and a `span`: }, "call_graph": [ { - "src": "can://go/myapp/main.go/…/example.com/main.main", - "dst": "can://go/myapp/pkg/greeter/greeter.go/…/example.com/pkg/greeter.Greeter.Greet", + "src": "can://myapp/go/main.go/…/example.com/main.main", + "dst": "can://myapp/go/pkg/greeter/greeter.go/…/example.com/pkg/greeter.Greeter.Greet", "prov": ["go/types"], "weight": 1 } @@ -329,12 +367,14 @@ node carries a `can://go//…` `id`, a `kind`, and a `span`: ``` Key v2 schema properties: -- **Envelope** — `schema_version` (`"2.0.0"`), `language` (`"go"`), `max_level` (1 or 2), - and `analyzer{name,version}` wrap a single `application` node. +- **Envelope** — `schema_version` (`"2.0.0"`), `language` (`"go"`), `max_level` (1, 2 or 3), + `k_limit` (L3 only: the `--graph-field-depth` value) and `analyzer{name,version}` wrap a + single `application` node. - **One containment tree** — `application → module → type → callable → body`, keyed as `symbol_table` (modules, by **project-relative file path**), `types`, `callables`, `body`. -- **`id`** — every node carries a durable `can://go////` id; - `` is the `--app-name` anchor. Body call nodes use `` keyed by `line:col`. +- **`id`** — every node carries a durable `can:///go///` id; + `` is the `--app-name` anchor. A body node is keyed by `line:col` in its callable's `body` + map and carries `id` = `@` at every level, from `-a 1` on. - **`kind`** — `module` · `type` kinds `struct | interface | alias | defined` (collapses v1's `is_interface`) · `callable` kinds `function | method | lambda` · body `call`. - **`base_types: []`** (type, optional) — embedded type ids (struct/interface embedding — the @@ -342,20 +382,33 @@ Key v2 schema properties: - **`span`** — `{start:[line,col], end:[line,col], bytes:[from,to]}`; `bytes` are **UTF-8 byte offsets** into the owning `module.source` (source is stored once per module, every node slices it). - **`source_file`** (callable, optional) — set when a method is declared in a *different file* - than its receiver type; the method stays nested under the type, and its `span.bytes` index + than its receiver type, and on every closure (nested ones too) inside such a method; the + callable stays nested under the type, and its `span.bytes` (and its body nodes' spans) index `symbol_table[source_file].source` instead of the nesting module's. - **`error_channel: []`** — populated from `error`-typed returns (the returns also stay in `return_type`). **Closures** nest as `callables{}` on the enclosing callable. -- **`body` call nodes** — `callee` is the sanctioned `null → id` slot: `null` at L1 (and for - external/stdlib callees), a `can://` node id once resolved at L2. `is_goroutine` / `is_deferred` +- **`body` call nodes** — `callee` is the sanctioned `null → id` slot: `null` at L1; at L2 a + project callable's `can://` id, or for a library (stdlib or dependency) callee its + `can:///@external//` id, a key of `external_symbols`. It stays `null` + for conversions, builtins, calls through function values and unresolved sites. `is_goroutine` / `is_deferred` are boolean flags emitted **only when true** (a plain call omits them), for `go f()` / `defer f()`. -- **`call_graph` edges** — `{src, dst, prov, weight}`; `src`/`dst` are `can://` node ids that - exist in the tree (not raw signatures), `prov` is resolver provenance, e.g. `["go/types"]`. +- **`call_graph` edges** — `{src, dst, prov, weight}`; `src` is a callable id in the tree and + `dst` a callable id or an `external_symbols` key (never a raw signature); `prov` is resolver + provenance, e.g. `["go/types"]`. +- **`external_symbols`** (application, L2, optional) — one entry per library callee, keyed by + its `@external` id: `{id, kind: "external", name, module, signature, parameters[], return_type}` + (parameters carry no span). Absent when no library call was resolved. +- **L3 (`-a 3`)** — `body` gains `statement` / `branch` / `loop` / `switch` / `return` nodes + and the synthetic `@entry` / `@exit` (which have no `span`). Every body node carries `id` + (`@`). + A call nested in a statement carries `parent`, the statement's key. The callable gains + `cfg[] {src, dst, kind}`, `cdg[] {src, dst}` and `ddg[] {src, dst, var, prov}`, whose + endpoints are `body` keys. See `docs/design/specs/v2-schema-structure.md`. ## Neo4j projection (`--emit neo4j`) `--emit neo4j` projects the **same v2 envelope** into a Neo4j property graph. It is a *second -projection of the identical analysis* — JSON nodes and graph nodes carry the same `can://go//…` +projection of the identical analysis* — JSON nodes and graph nodes carry the same `can:///…` ids, so the two join on one string and are never recomposed. The graph vocabulary is `Go`-prefixed labels (`:GoModule`, `:GoType`, `:GoCallable`, `:GoField`, `:GoBodyNode`, `:GoExternal`, plus the shared merge label `:GoSymbol` and marker `:GoCanNode`) and `GO_`-prefixed relationships @@ -371,12 +424,25 @@ There are two output paths, chosen by whether `--neo4j-uri` is set: | **Use for** | Review, diffing, loading on your own schedule, CI | A DB you already have running | Both paths are **scoped to one application**: every destructive statement is anchored on -`:GoCanNode` and the `can://go//` id prefix, so re-emitting replaces exactly that app's subgraph -and never touches another app (or a sibling-language graph) in the same database. The projection is -**deterministic** — a second emit is byte-identical. - -> **`--emit neo4j` is always full depth.** The graph is projected at the max implemented level (L2 -> today — symbol table + call graph) and canonical schema v2. Do **not** pass `-a`/`--analysis-level` +`:GoCanNode` and the `can:///` id prefix, so it never touches another app (or a +sibling-language graph) in the same database. The projection is **deterministic** — a second emit +is byte-identical. + +What a re-emit replaces depends on the path: + +- **`graph.cypher` snapshot** and **`--eager` Bolt push**: the app's subgraph is purged and + rebuilt, so re-emitting replaces exactly that app's subgraph. +- **Default (lazy) Bolt push**: purely additive. Changed modules are MERGE-upserted and nothing is + deleted. After source edits, nodes and edges that the new analysis no longer emits stay in the + database. For L3 this includes `:GoBodyNode`s and their `GO_CFG_NEXT` / `GO_CDG` / `GO_DDG` + edges, whose ids are keyed on `line:col`: once lines shift, the old body nodes and their edges + remain beside the new ones. Pass `--eager` when you push an L3 graph after editing source. + The purge needs a full run: with `--target-files` it is skipped, because the push carries only + the targeted packages. + +> **`--emit neo4j` is always full depth.** The graph is projected at the max implemented level (L3 +> today — symbol table, call graph, and every callable's body nodes with `GO_CFG_NEXT` / `GO_CDG` / +> `GO_DDG`) and canonical schema v2. Do **not** pass `-a`/`--analysis-level` > with it — an explicit level is a flag error (`--analysis-level does not apply to --emit neo4j`), > because there is no shallower graph to ask for. @@ -451,7 +517,7 @@ MATCH (n:GoCanNode) RETURN labels(n)[0] AS label, count(*) AS n ORDER BY n DESC; MATCH ()-[r]->() RETURN type(r) AS rel, count(*) AS n ORDER BY n DESC; // Everything owned by one application (prefix seek). -MATCH (n:GoCanNode) WHERE n.id STARTS WITH 'can://go//' RETURN count(n) AS app_nodes; +MATCH (n:GoCanNode) WHERE n.id STARTS WITH 'can:///' RETURN count(n) AS app_nodes; // Types ranked by method count (the "god struct" finder). MATCH (t:GoType) OPTIONAL MATCH (t)-[:GO_HAS_METHOD]->(m) @@ -489,6 +555,35 @@ The SDK locates the analyzer via the `codeanalyzer-go` command on `PATH` (instal methods above, including `pip install codeanalyzer-go`). See [python-sdk](https://github.com/codellm-devkit/python-sdk) for full API documentation. +## Upgrading + +This release changes the output contract. If you consume `analysis.json` or the Neo4j graph, +check the following. + +- **Ids are app-outermost.** `can://go///…` is now `can:///go//…`. Library + callees are `can:///@external//`. +- **Body nodes carry `id` at every level.** Each body node has `id` = `@`, + from `-a 1` on, not only at `-a 3`. +- **L2 includes library calls.** The call graph now has edges to library (stdlib and dependency) + callees, and `application.external_symbols` describes them. A library call's `callee` is its + external id instead of `null`. A type conversion is still a call node, but its `callee` stays + `null`; it is no longer resolved to a `.__new__` pseudo-callee. +- **L1 records more.** + - A type's `kind` can now be `alias` or `defined`, not only `struct` or `interface`. + - Deferred calls carry `is_deferred: true`. + - Chained calls that start at the same position, such as `s.A().B()`, are now separate call + nodes keyed `l:c` and `l:c/2`. Previously the inner call was dropped. Do not parse body keys + as plain `line:col`. + - Calls inside `go` statement arguments, such as `g()` in `go f(g())`, are now call sites. + - Consumers that count call nodes will see more of them. +- **Neo4j:** + - `--emit neo4j` projects L3, adding `GO_CFG_NEXT`, `GO_CDG` and `GO_DDG`. + - `:GoBodyNode.call_node` is the enclosing statement's local key (the JSON `parent`), not a + global id. + - `--eager` now works on Neo4j 5.x. +- **Build.** Building from source needs Go 1.26 or newer. Release binaries are built with the + latest Go (see [Toolchain](#toolchain)). + ## Architecture & Tooling | Slot | Choice | Rationale | @@ -498,8 +593,10 @@ methods above, including `pip install codeanalyzer-go`). See | Type resolver | `golang.org/x/tools/go/packages` | Single API for both AST + full type resolution; handles modules natively | | Build/dep materialization | `go mod download` | Required before `packages.Load` so the module cache is warm; result cached by `go.sum` hash | | Packaging | Native binary (`go build`) | Zero-runtime-dep distribution; matches Rust/C++ analyzers | -| Analysis depth | Level 1 (rapid) | Symbol table + resolver call graph | +| Analysis depth | Levels 1–3 | Symbol table, resolver call graph, intraprocedural cfg/cdg/ddg | | Call-graph dispatch | Declared-type resolution via `go/types.Selections` | CHA-equivalent; sufficient for cross-package reachability at Level 1 | +| CFG (L3) | Hand-built from `go/ast` (`internal/dataflow`) | Adds what `go/cfg` does not model as edges: panic `exception` edges and the `defer` chain (`defer_resume`). A differential test checks reachability against `golang.org/x/tools/go/cfg` on every fixture body | +| Def-use / DDG (L3) | Syntactic reaching definitions over k-limited access paths | `go/types` objects as bases (shadowing-safe); weak defs through reference arguments; `prov: ["ssa"]` for parity with the Java and Python L3 | ### Package structure @@ -512,6 +609,7 @@ codeanalyzer-go/ │ ├── options/ # AnalysisOptions + AnalysisLevel constants │ ├── syntactic_analysis/ # SymbolTableBuilder (packages.Load → AST walk) │ ├── semantic_analysis/ # CallGraphBuilder (go/types resolver) +│ ├── dataflow/ # L3: body nodes, CFG, CDG, def-use, DDG; Run fans out per callable │ ├── analysis/ # Pluggable pass interface + registry (topo-ordered pipeline) │ ├── frameworks/ # BaseEntrypointFinder — extension seam for framework passes │ └── utils/ # DiscoverGoFiles, IsVendored, IsTestFile, logging diff --git a/cmd/codeanalyzer/main.go b/cmd/codeanalyzer/main.go index d41ae8c..296c5ba 100644 --- a/cmd/codeanalyzer/main.go +++ b/cmd/codeanalyzer/main.go @@ -39,6 +39,7 @@ func rootCmd() *cobra.Command { emit string appName string level int + fieldDepth int targetFiles []string skipTests bool eager bool @@ -75,15 +76,23 @@ via CLDK(language="go").analysis(project_path=...).`, emitTarget := options.EmitTarget(emit) switch emitTarget { case options.EmitJSON: - // valid — falls through to the JSON path below. + if err := validateAnalysisLevel(level); err != nil { + return err + } + if err := validateFieldDepth(cmd, level, fieldDepth); err != nil { + return err + } case options.EmitNeo4j: if cmd.Flags().Changed("analysis-level") { return fmt.Errorf("--analysis-level does not apply to --emit neo4j; the graph is always projected at full depth") } + if cmd.Flags().Changed("graph-field-depth") { + return fmt.Errorf("--graph-field-depth does not apply to --emit neo4j; the graph is always projected at full depth") + } // Force full depth: the graph carries every implemented // level's facts in both projections. (Output is always the // canonical v2 tree.) - level = int(options.LevelCallGraph) + level = int(options.LevelDataflow) case options.EmitSchema: path, err := neo4j.EmitSchema(outputDir) if err != nil { @@ -134,6 +143,7 @@ via CLDK(language="go").analysis(project_path=...).`, CacheDir: cacheDir, Jobs: jobs, Verbose: verbosity > 0, + FieldDepth: fieldDepth, Neo4jURI: firstNonEmpty(neo4jURI, os.Getenv("NEO4J_URI")), Neo4jUser: firstNonEmpty(neo4jUser, os.Getenv("NEO4J_USERNAME"), "neo4j"), Neo4jPassword: firstNonEmpty(neo4jPassword, os.Getenv("NEO4J_PASSWORD"), "neo4j"), @@ -150,7 +160,7 @@ via CLDK(language="go").analysis(project_path=...).`, // JSON path emits) and project it. --neo4j-uri set → live Bolt push; // absent → a graph.cypher snapshot in the output dir. if emitTarget == options.EmitNeo4j { - payload := v2emit.Emit(app, opts.AppName, opts.InputPath, int(opts.Level), opts.AnalyzerVersion) + payload := v2emit.Emit(app, opts.AppName, opts.InputPath, int(opts.Level), opts.AnalyzerVersion, core.KLimit(opts), analyzer.L3()) boltCfg := neo4j.BoltConfig{ URI: opts.Neo4jURI, User: opts.Neo4jUser, @@ -171,14 +181,14 @@ via CLDK(language="go").analysis(project_path=...).`, // writer so tests can capture it via cmd.SetOut. Both paths render // through core so the v1/v2 schema selection stays in one place. if outputDir == "" { - data, err := core.RenderJSON(app, opts) + data, err := core.RenderJSON(app, analyzer.L3(), opts) if err != nil { return err } _, err = cmd.OutOrStdout().Write(data) return err } - return core.WriteOutput(app, opts) + return core.WriteOutput(app, analyzer.L3(), opts) }, } @@ -190,7 +200,9 @@ via CLDK(language="go").analysis(project_path=...).`, f.StringVar(&appName, "app-name", "", "Application anchor name for can:// ids and Neo4j :Application (default: input dir name)") f.IntVarP(&level, "analysis-level", "a", 1, - "Analysis level: 1=symbol table only, 2=+resolver call graph") + "Analysis level: 1=symbol table, 2=+call graph, 3=+intraprocedural cfg/cdg/ddg") + f.IntVar(&fieldDepth, "graph-field-depth", 3, + "Field-path depth bound for L3 dataflow nodes (k limit; applies at -a 3 only)") f.StringSliceVarP(&targetFiles, "target-files", "t", nil, "Restrict analysis to specific files (incremental mode)") f.BoolVar(&skipTests, "skip-tests", true, "Skip *_test.go files") @@ -209,6 +221,31 @@ via CLDK(language="go").analysis(project_path=...).`, return cmd } +// validateAnalysisLevel accepts the implemented levels (1..3). Level 4 is +// reserved for a later release and gets its own message. +func validateAnalysisLevel(level int) error { + switch { + case level >= 1 && level <= int(options.LevelDataflow): + return nil + case level == 4: + return fmt.Errorf("analysis level 4 is not implemented yet") + default: + return fmt.Errorf("analysis level %d is not supported; use 1, 2 or 3", level) + } +} + +// validateFieldDepth checks --graph-field-depth. The flag only means something +// at L3+, and the k limit must be at least 1. +func validateFieldDepth(cmd *cobra.Command, level, fieldDepth int) error { + if cmd.Flags().Changed("graph-field-depth") && level < int(options.LevelDataflow) { + return fmt.Errorf("--graph-field-depth applies only at --analysis-level 3 or higher") + } + if fieldDepth < 1 { + return fmt.Errorf("--graph-field-depth must be >= 1; got %d", fieldDepth) + } + return nil +} + // firstNonEmpty returns the first non-empty string in vals, or "" if all are // empty. It encodes the contract's precedence: explicit flag > env var > default. func firstNonEmpty(vals ...string) string { diff --git a/cmd/codeanalyzer/main_test.go b/cmd/codeanalyzer/main_test.go index e81d0fe..cc35426 100644 --- a/cmd/codeanalyzer/main_test.go +++ b/cmd/codeanalyzer/main_test.go @@ -278,8 +278,8 @@ func TestRootCmd_EmitsCanonicalV2(t *testing.T) { if v.Language != "go" { t.Errorf("language = %q, want go", v.Language) } - if v.Application.ID != "can://go/greeter" { - t.Errorf("application.id = %q, want can://go/greeter", v.Application.ID) + if v.Application.ID != "can://greeter" { + t.Errorf("application.id = %q, want can://greeter", v.Application.ID) } } @@ -324,3 +324,182 @@ func TestRootCmd_TargetFilesRestrictsOutput(t *testing.T) { t.Error("--target-files: server/server.go should appear in output") } } + +// ── --analysis-level 3 and --graph-field-depth ─────────────────────────────── + +func TestRootCmd_Level3Accepted(t *testing.T) { + td := cliTestdataDir() + + out, _, err := runCmd( + "--input", filepath.Join(td, "greeter"), + "--analysis-level", "3", + "--cache-dir", t.TempDir(), + ) + if err != nil { + t.Fatalf("-a 3 should succeed; got %v", err) + } + var v struct { + MaxLevel int `json:"max_level"` + KLimit int `json:"k_limit"` + } + if jsonErr := json.Unmarshal([]byte(out), &v); jsonErr != nil { + t.Fatalf("stdout is not valid JSON: %v", jsonErr) + } + if v.MaxLevel != 3 { + t.Errorf("max_level = %d, want 3", v.MaxLevel) + } + if v.KLimit != 3 { + t.Errorf("k_limit = %d, want 3 (the --graph-field-depth default)", v.KLimit) + } +} + +func TestRootCmd_Level4RejectedInThisRelease(t *testing.T) { + td := cliTestdataDir() + + _, _, err := runCmd( + "--input", filepath.Join(td, "greeter"), + "--analysis-level", "4", + "--cache-dir", t.TempDir(), + ) + if err == nil { + t.Fatal("expected error for -a 4, got nil") + } + if err.Error() != "analysis level 4 is not implemented yet" { + t.Errorf("error = %q, want %q", err.Error(), "analysis level 4 is not implemented yet") + } +} + +func TestRootCmd_UnsupportedLevelRejected(t *testing.T) { + td := cliTestdataDir() + + for _, lvl := range []string{"0", "5", "-1"} { + _, _, err := runCmd( + "--input", filepath.Join(td, "greeter"), + "--analysis-level", lvl, + "--cache-dir", t.TempDir(), + ) + if err == nil { + t.Errorf("-a %s: expected error, got nil", lvl) + continue + } + if !strings.Contains(err.Error(), "analysis level "+lvl+" is not supported") { + t.Errorf("-a %s: error should name the unsupported level; got %q", lvl, err.Error()) + } + } +} + +func TestRootCmd_FieldDepthBelowLevel3Rejected(t *testing.T) { + td := cliTestdataDir() + + _, _, err := runCmd( + "--input", filepath.Join(td, "greeter"), + "--analysis-level", "2", + "--graph-field-depth", "2", + "--cache-dir", t.TempDir(), + ) + if err == nil { + t.Fatal("expected error for --graph-field-depth at -a 2, got nil") + } + if !strings.Contains(err.Error(), "--graph-field-depth applies only at --analysis-level 3 or higher") { + t.Errorf("error should explain the level gate; got %q", err.Error()) + } +} + +func TestRootCmd_FieldDepthBelowOneRejected(t *testing.T) { + td := cliTestdataDir() + + _, _, err := runCmd( + "--input", filepath.Join(td, "greeter"), + "--analysis-level", "3", + "--graph-field-depth", "0", + "--cache-dir", t.TempDir(), + ) + if err == nil { + t.Fatal("expected error for --graph-field-depth 0, got nil") + } + if !strings.Contains(err.Error(), "--graph-field-depth must be >= 1") { + t.Errorf("error should explain the minimum; got %q", err.Error()) + } +} + +func TestRootCmd_FieldDepthWithNeo4jRejected(t *testing.T) { + td := cliTestdataDir() + + _, _, err := runCmd( + "--input", filepath.Join(td, "greeter"), + "--emit", "neo4j", + "--graph-field-depth", "2", + ) + if err == nil { + t.Fatal("expected a flag error for --emit neo4j --graph-field-depth 2, got nil") + } + if !strings.Contains(err.Error(), "does not apply to --emit neo4j") { + t.Errorf("error should explain the neo4j flag gate; got %q", err.Error()) + } +} + +func TestRootCmd_EmitNeo4jRunsAtLevel3(t *testing.T) { + td := cliTestdataDir() + out := t.TempDir() + if _, _, err := runCmd("--input", filepath.Join(td, "multipackage"), "--emit", "neo4j", "--output", out); err != nil { + t.Fatalf("--emit neo4j should succeed; got %v", err) + } + data, err := os.ReadFile(filepath.Join(out, "graph.cypher")) + if err != nil { + t.Fatalf("graph.cypher not written: %v", err) + } + for _, rel := range []string{"GO_CFG_NEXT", "GO_CDG", "GO_DDG"} { + if !strings.Contains(string(data), rel) { + t.Errorf("graph.cypher should carry the L3 %s relationship", rel) + } + } +} + +// ── malformed source ───────────────────────────────────────────────────────── + +// TestRootCmd_MalformedSourceDoesNotPanic: testdata/malformed holds syntax +// errors (`if :=`, `range {`, `case 1, :`, `case <-:`), one per file, after +// which the parser recovers with a function node whose End is one past the +// end of the file: it has no token.File (the last file) or lies in the next +// one. Every level must still emit valid JSON, with the well-formed function +// that precedes each error present. +func TestRootCmd_MalformedSourceDoesNotPanic(t *testing.T) { + td := cliTestdataDir() + for _, level := range []string{"1", "2", "3"} { + t.Run("a"+level, func(t *testing.T) { + var out string + var err error + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("-a %s panicked: %v", level, r) + } + }() + out, _, err = runCmd("--input", filepath.Join(td, "malformed"), "-a", level, "--cache-dir", t.TempDir()) + }() + if err != nil { + t.Fatalf("-a %s failed: %v", level, err) + } + var v struct { + Application struct { + SymbolTable map[string]struct { + Functions map[string]struct { + Signature string `json:"signature"` + } `json:"functions"` + } `json:"symbol_table"` + } `json:"application"` + } + if jsonErr := json.Unmarshal([]byte(out), &v); jsonErr != nil { + t.Fatalf("-a %s stdout is not valid JSON: %v", level, jsonErr) + } + for file, want := range map[string]string{ + "if.go": "GoodIf", "range.go": "GoodRange", "case.go": "GoodCase", "recv.go": "GoodRecv", + } { + fns := v.Application.SymbolTable[file].Functions + if _, ok := fns["example.com/malformed."+want]; !ok { + t.Errorf("-a %s: %s: well-formed %s not emitted; have %v", level, file, want, fns) + } + } + }) + } +} diff --git a/cmd/codeanalyzer/toolchain_test.go b/cmd/codeanalyzer/toolchain_test.go new file mode 100644 index 0000000..ac0d1b9 --- /dev/null +++ b/cmd/codeanalyzer/toolchain_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// captureStderr runs fn with os.Stderr redirected (utils.Warn writes there, +// not to the cobra command's stderr) and returns what was written. +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + orig := os.Stderr + os.Stderr = w + done := make(chan string) + go func() { + b, _ := io.ReadAll(r) + done <- string(b) + }() + defer func() { os.Stderr = orig }() + fn() + w.Close() + os.Stderr = orig + return <-done +} + +// TestRootCmd_NewerGoVersionWarnsOnce (I4): packages that need a newer Go than +// the analyzer was built with type-check partially and the run still exits 0, +// so the degradation must be loud. Two such packages (go1.99, go1.98) give ONE +// consolidated warning naming the build Go, the highest version required and +// the affected-package count; stdout carries the analysis, untouched. +// +// The fixture's go.mod cannot itself say go 1.99: the go command refuses such +// a module before loading anything (no toolchain to switch to), so the run +// fails outright. Files gated by //go:build go1.99 / go1.98, with the tags set +// through GOFLAGS, reach go/types' same "requires newer Go version" check. +func TestRootCmd_NewerGoVersionWarnsOnce(t *testing.T) { + t.Setenv("GOFLAGS", "-tags=go1.99,go1.98") + t.Setenv("GOTOOLCHAIN", "local") + var out string + var runErr error + stderr := captureStderr(t, func() { + out, _, runErr = runCmd("--input", filepath.Join(cliTestdataDir(), "futurego"), "--cache-dir", t.TempDir()) + }) + if runErr != nil { + t.Fatalf("command failed: %v\nstderr:\n%s", runErr, stderr) + } + + var warns []string + for _, line := range strings.Split(stderr, "\n") { + if strings.Contains(line, "WARN") && strings.Contains(strings.ToLower(line), "rebuild") { + warns = append(warns, line) + } + } + if len(warns) != 1 { + t.Fatalf("want exactly one consolidated warning, got %d:\n%s", len(warns), stderr) + } + w := warns[0] + for _, want := range []string{runtime.Version(), "go1.99", "2 packages"} { + if !strings.Contains(w, want) { + t.Errorf("warning lacks %q: %s", want, w) + } + } + if strings.Contains(w, "go1.98") { + t.Errorf("warning names a lower version than the highest: %s", w) + } + + // (stdout embeds module sources, whose comments mention "newer Go + // version"; look for the log line's prefix instead.) + if strings.Contains(out, "[codeanalyzer-go") { + t.Errorf("a log line leaked onto stdout") + } + var v struct { + Application struct { + SymbolTable map[string]json.RawMessage `json:"symbol_table"` + } `json:"application"` + } + if err := json.Unmarshal([]byte(out), &v); err != nil { + t.Fatalf("stdout is not valid JSON: %v", err) + } + for _, rel := range []string{"a/a.go", "b/b.go", "c/c.go"} { + if _, ok := v.Application.SymbolTable[rel]; !ok { + t.Errorf("stdout symbol_table lacks %s", rel) + } + } +} diff --git a/docs/design/Neo4j_projection/specs/neo4j-projection.md b/docs/design/Neo4j_projection/specs/neo4j-projection.md index e5b1c8f..b8e840a 100644 --- a/docs/design/Neo4j_projection/specs/neo4j-projection.md +++ b/docs/design/Neo4j_projection/specs/neo4j-projection.md @@ -80,7 +80,7 @@ or a verbatim reuse of shared vocabulary. "Ref" names the sibling precedent. | **CG1** | `call_graph` → **`GO_CALLS`** (weight, prov); `error_channel[]` → **string-array prop** on `:GoCallable` | `GO_CALLS` parity with `PY_CALLS` (endpoints = callable id or `:GoExternal` ghost; no dangling). Go errors are returned values, not thrown types — a value-shaped fact rides the node, not a `GO_RAISES` edge. | | **I1** | `GO_IMPORTS` (aggregated per target) → `:GoModule` (in-project) or `:GoExternal` ghost (stdlib/external, keyed by import path) | Parity with aggregated `PY_IMPORTS` + edge-only-when-resolved. Go imports are overwhelmingly external → ghost targets, never dangling, never minted as `:Package` (packages come from dependencies, not imports). | | **A1** | Declare neutral `:Artifact`/`:Package`/`:ConfigKey` + `HAS_ARTIFACT`/`DECLARES_DEPENDENCY`(`_k=kind`)/`LOCKS`/`DEFINES_CONFIG` **verbatim**; project only facts the analyzer produces | These are the cross-language **merge targets** — reused unchanged so a sibling over the same repo converges. go.mod deps → `:Package` `pkg:golang/…`; absent facts → **zero rows** (declared contract, empty projection). `GO_PROVIDES`/`GO_UNRESOLVED_IMPORT` stay `GO_`-prefixed (this analyzer's claims). | -| **L1** | Declare the **full L3/L4 overlay vocabulary now**; projector emits **zero rows** until the levels land | `GO_HAS_BODY_NODE`, `GO_CFG_NEXT`(`_k=kind`), `GO_CDG`, `GO_DDG`(`_k=var,prov`), `GO_PARAM_IN`, `GO_PARAM_OUT`, `GO_SUMMARY` + body-node statement kinds. `--emit neo4j` = "full-depth = max implemented" (L2 today). The contract is declared once; L3/L4 fill it with **no schema change**. | +| **L1** | Declare the **full L3/L4 overlay vocabulary now**; projector emits **zero rows** until the levels land | `GO_HAS_BODY_NODE`, `GO_CFG_NEXT`(`_k=kind`), `GO_CDG`, `GO_DDG`(`_k=var,prov`), `GO_PARAM_IN`, `GO_PARAM_OUT`, `GO_SUMMARY` + body-node statement kinds. `--emit neo4j` = "full-depth = max implemented" (**L3 today**: `GO_CFG_NEXT`, `GO_CDG`, `GO_DDG` and the L3 body nodes are live; only `GO_PARAM_IN`, `GO_PARAM_OUT`, `GO_SUMMARY` remain reserved for L4). The contract is declared once; L3 filled it with **no schema change**, and L4 will too. | | **K1** | `_k` discriminants: `GO_CFG_NEXT` per `kind`; `GO_DDG` per `(var,prov)`; `DECLARES_DEPENDENCY` per `kind` | Same mechanism as the siblings — parallel edges survive MERGE instead of collapsing. | | **V1** | `SCHEMA_VERSION = "2.0.0"`, stamped on `:GoApplication`; additive-MINOR suspended for the 2.0.0 line | Parity with the sibling versioning ruling; consumers gate on the analyzer-version floor. | @@ -88,13 +88,13 @@ or a verbatim reuse of shared vocabulary. "Ref" names the sibling precedent. | Label | Merge label | Key | Notes | | --- | --- | --- | --- | -| `:GoApplication` | `GoApplication` | `id` | root `can://go/`; schema_version, analyzer name/version, max_level | +| `:GoApplication` | `GoApplication` | `id` | root `can://` (app-outermost; child ids are `can:///go/…`); schema_version, analyzer name/version, max_level | | `:GoModule` | `GoModule` | `id` | per file: package, **source** (whole file), content_hash, span | | `:GoType` | `GoSymbol` | `id` | `kind ∈ struct\|interface\|alias\|defined` (property), signature, span | | `:GoCallable` | `GoSymbol` | `id` | signature, return_type, `error_channel[]`, `source_file?`, metrics, span | | `:GoField` | `GoField` | `id` (`/`) | type, span | -| `:GoBodyNode` | `GoBodyNode` | `id` (global ordinal) | `kind` (`call` at L1), `is_goroutine?`, `is_deferred?`, span | -| `:GoExternal` | `GoSymbol` | `id` | library/stdlib ghost; name, import path | +| `:GoBodyNode` | `GoBodyNode` | `id` (global ordinal; the JSON body node's `id`, emitted from L1) | `kind` (`call` at L1; `statement\|branch\|loop\|switch\|return\|entry\|exit` at L3), `is_goroutine?`, `is_deferred?`, span (absent on `@entry`/`@exit`), `call_node?` (L3: the enclosing statement's **local** key, JSON `parent` verbatim — Python parity), `var?` (L4) | +| `:GoExternal` | `GoSymbol` | `id` | library/stdlib ghost (`can:///@external/…`); name, import path, signature, module | | `:Artifact` | `Artifact` | `id` | **neutral** | | `:Package` | `Package` | `id` (purl) | **neutral** | | `:ConfigKey` | `ConfigKey` | `id` | **neutral** | @@ -108,7 +108,8 @@ Every `can://`-keyed node also carries `:GoCanNode`. - **Inheritance:** `GO_EMBEDS` (deferred/resolved-only), `GO_SATISFIES` - **Imports:** `GO_IMPORTS` (aggregated) - **Repository (neutral):** `HAS_ARTIFACT`, `DECLARES_DEPENDENCY` (`_k`), `LOCKS`, `DEFINES_CONFIG`; (`GO_`) `GO_PROVIDES`, `GO_UNRESOLVED_IMPORT` -- **Reserved L3/L4 overlay (zero rows until the level lands):** `GO_CFG_NEXT` (`_k`), `GO_CDG`, `GO_DDG` (`_k`), `GO_PARAM_IN`, `GO_PARAM_OUT`, `GO_SUMMARY` +- **L3 overlay (live):** `GO_CFG_NEXT` (`kind`, `_k=kind`), `GO_CDG`, `GO_DDG` (`var`, `prov`, `_k=var|prov`) +- **Reserved L4 overlay (zero rows until L4 lands):** `GO_PARAM_IN`, `GO_PARAM_OUT`, `GO_SUMMARY` ### CLI surface (parity with TS/Python) @@ -133,7 +134,7 @@ yet; the live anchors are the class-centric Java + Python facades). Tier A (life | --- | --- | --- | | **SDK1** | **Type-centric** leaf surface: `get_types`/`get_type`, `get_functions` (package-level), `get_methods_of`/`get_method` (receiver methods), `get_fields` | Go has no class → **not** `get_classes`. Functions are kept distinct from methods (no "module-name-as-class" hack Python uses for module funcs). Establishes the SDK's first type-centric precedent for future languages. | | **SDK2** | `get_callers`/`get_callees` keyed on **callable id / `(type_id, sig)`** | Matches Go's identity model and the C-facade style (take the callable, not class+method strings). Clean for package-level funcs and methods alike. | -| **SDK3** | Scope: **Tier A + L1/L2 leaf accessors + repository-artifact getters + `get_source`**; omit slicing/CFG/CDG/DDG/reachability | Those need L3/L4 the analyzer does not produce. The generic `AnalysisBackend` ABC's abstractmethods are all implemented; L3/L4 methods are absent/`NotImplementedError` per the ABC contract — no misleading always-erroring surface. | +| **SDK3** | Scope: **Tier A + L1/L2 leaf accessors + repository-artifact getters + `get_source`**; omit slicing/CFG/CDG/DDG/reachability | Those need L3/L4 the analyzer does not produce. *(2026-10-24: L3 is now emitted at `-a 3` — per-callable `cfg`/`cdg`/`ddg` and the `GO_CFG_NEXT`/`GO_CDG`/`GO_DDG` edges — so L3 accessors (CFG/CDG/DDG) are supportable now; interprocedural slicing and taint still need L4 (`param_in`/`param_out`/`summary` and the points-to `ddg`), which is not produced.)* The generic `AnalysisBackend` ABC's abstractmethods are all implemented; L3/L4 methods are absent/`NotImplementedError` per the ABC contract — no misleading always-erroring surface. | | **SDK4** | `GoAnalysisBackend` sets `N="Go"`, `P="GO_"`; `GoBackend = Union[GoCodeAnalyzerConfig, Neo4jConnectionConfig]` | `Neo4jConnectionConfig` already exists in `commons/backend_config.py` and in each `*Backend` union → the consumer side is **additive**. The graph it reads is "populated out of band" by the analyzer's `--emit neo4j`. | One facade vocabulary, two encodings when the TS SDK catches up: `cldk/analysis/go/GoAnalysis` @@ -184,8 +185,8 @@ lockstep."** Concretely: - **Analyzer** ships the `internal/neo4j/` projection as part of the v2 line. `--emit neo4j` is independently landable behind the existing flag stubs; it produces a graph at - full-depth = L2 today and grows additively when L3/L4 arrive (no contract change — L1 - reserve). + full-depth = L3 today (L3 landed additively, with no contract change) and grows additively + again when L4 arrives (no contract change — L1 reserve). - **SDK** (`GoAnalysisBackend`) pins the analyzer version **only after** the analyzer v2 release is cut — same ordering constraint as the JSON wiring (the SDK's Go models can't parse v2 until then, and there is no Go wiring yet, so this is net-new, not a migration). diff --git a/docs/design/notes/l3-dataflow-metrics.md b/docs/design/notes/l3-dataflow-metrics.md new file mode 100644 index 0000000..eb619f2 --- /dev/null +++ b/docs/design/notes/l3-dataflow-metrics.md @@ -0,0 +1,811 @@ +# L3 dataflow metrics over the real-app corpus + +L3 is the intraprocedural dataflow level. At `-a 3`, every callable with a body gets +statement-granular body nodes and three edge lists: + +- `cfg`: hand-built from `go/ast`, with a defer chain and conservative panic edges. +- `cdg`: Cooper-Harvey-Kennedy post-dominators plus Ferrante-Ottenstein-Warren. +- `ddg`: syntactic may-reaching definitions over k-limited access paths, `prov:["ssa"]`. + +The semantics are in [`v2-l3-l4-dataflow.md`](../specs/v2-l3-l4-dataflow.md) §4.2–§4.5. + +This note records how L3 behaves on the corpus of [`l3-eval-2026-10.md`](l3-eval-2026-10.md), +which it accompanies. It covers volume, the shape of the graphs, cost, output size and internal +consistency at scale. That note holds the run protocol, the bugs the evaluation found, and the +degraded-input details. This note reuses its timings and extends its counts. + +- **When and where:** measured 2026-10-24 on an Apple M3 Max (16 cores, 64 GB, macOS), with the + analyzer built with go1.27.3 (`runtime.Version()`; no too-old-toolchain warning on any run). +- **Corpus.** Six apps, plus **Kubernetes v1.37.1 as the scale benchmark**, run as two modules: + `kubernetes` (the root module `k8s.io/kubernetes`) and `k8s-api` (`staging/src/k8s.io/api`, + almost all generated protobuf). cockroach is no longer a benchmark (see § Corpus); its pre-fix + figures are kept in one labelled subsection and never folded into totals. +- **Analyzer version (provenance).** Every number for the six apps and Kubernetes comes from the + `-a 3 -j 1` outputs of one `scripts/eval_l3.sh` pass with the binary it auto-builds from + `feat/l3-dataflow` at `9aeda43`. The analyzer code there is identical to `5508010` (the commit + after `9aeda43` touches only `scripts/`), so it includes every final-review fix: C1 (subslice + weak defs, `258b22f`), I2 (type-error defs, `6fdc2c5`), R15 (body-node `id` from L1, `9ae739e`) + and I7 (deferred calls read receiver and argument state, `3f31626`). The cockroach subsection is + the earlier `fd24bdf` binary, **pre-fix**. +- **"Before" figures** (§ What the final-review fixes changed) are the `fd24bdf` evaluation binary, + rerun on the same trees where the earlier outputs were not kept (dae linux, both Kubernetes + modules). + +One analysis engine, so there is no engine comparison here. Interprocedural (L4) results will be +added under [§ L4](#l4-to-be-added). + +## Results at a glance + +"6 apps" is cobra, listmonk, memos, nats-server, openbao and dae (`GOOS=linux`). "Kubernetes" is +the two Kubernetes modules together, `kubernetes` + `k8s-api`; they are disjoint (`go list ./...` of +the root does not descend into the `staging/` modules), so their counts add. All 8 outputs type-check +completely (0 packages with errors at prepare time, 0 type-error warnings at run time). + +| | 6 apps | Kubernetes (`kubernetes` + `k8s-api`) | +|---|---|---| +| callables with a body | 22,971 | 58,030 (42,603 + 15,427) | +| body nodes | 385,505 | 848,801 (492,627 + 356,174) | +| `cfg` / `cdg` / `ddg` edges | 468,353 / 261,656 / 777,658 | 923,459 / 545,150 / 1,464,541 | +| `cfg` / `cdg` / `ddg` per callable | 20.4 / 11.4 / 33.9 | 15.9 / 9.4 / 25.2 | +| `ddg` / `cdg` | 2.97 | 2.69 | +| `exception` share of `cfg` | 31.1% | 26.6% | +| L3 wall time over `-a 2`, `-j 1` | +2.35 s on 10.53 s (18% of `-a 3` wall) | +2.81 s on 9.62 s (23%) / +1.93 s on 3.13 s (38%) | +| L3 throughput, `-j 1` | 164k body nodes/s | 175k / 185k body nodes/s | +| peak RSS, `-a 3 -j 1` | 263 MB to 5.1 GB | 7.1 GB / 2.2 GB | +| `analysis.json`, `-a 2` to `-a 3` | 106.5 to 244.4 MB (2.3x) | 158.2 to 317.6 MB (2.0x) / 55.6 to 183.4 MB (3.3x) | +| consistency checks (§ below) | all PASS | all PASS | +| `-j 1` = `-j 16` = `-j 16`, byte-identical | yes (6/6) | yes (2/2) | + +**Key takeaways.** + +1. **L3 is cheap in time, not in bytes.** It adds 18% to `-a 3` wall time at `-j 1` on the 6 apps + and 23% on the Kubernetes root (2.8 s on 9.6 s for 493k body nodes), and at most 15% peak RSS. + But it doubles to triples `analysis.json`: L3 is 50–70% of an `-a 3` file. The global `id` + on every body node is about 30% of those L3 bytes, as large as the `ddg`. +2. **Kubernetes is the scale benchmark, and it is easy.** The root module (3,414 files, 44,514 + callables) runs `-a 3` in 12.4 s at 7.1 GB, against cockroach's 33 s at 13.5 GB, with full + type-checking where cockroach had 62,908 type errors. +3. **Panic edges shape the CFG and CDG.** The conservative `canPanic` gives 62% of non-synthetic + CFG nodes on the 6 apps an `exception` edge, so 31% of `cfg` edges are `exception`. 35% of `cdg` + edges hang off a controller whose only way to branch is that edge: "the next statement is + control-dependent on this call, because the call may panic." +4. **The DDG is long-tailed, and weak defs drive the tail.** The median callable has 4–13 `ddg` + edges, but the top 1% of callables hold 18–53% of each app's edges. In those callables, a + reference-typed base (a receiver `c`, a router `g`, a conversion `Scheme` `s`) is weak-defined + by every call and never killed, and every use of the bare base links to every reaching def. + The single largest callable in the corpus is Kubernetes' generated + `core/v1.RegisterConversions`: 130,304 edges, 16% of the root module's `ddg`. +5. **Generated protobuf marshallers (review finding I5): measured, and bounded.** `k8s-api` holds + 2,440 gogoproto `Marshal`/`Unmarshal` bodies over 60 `generated.pb.go` files. Its `ddg` is + 626,883 edges (93% in `.pb.go` code) and its `-a 3` file is 183 MB. The largest marshaller, + `PodSpec.MarshalToSizedBuffer` (43 fields), has 14,796 edges, and the per-message cost grows + about as fields^1.8. That is the same mechanism as the 600-field probe's 1.27M edges, but real + messages are small (median 3 fields, max 43), so the largest real marshaller is 86x below the + probe. § Protobuf-generated DDG volume has the detail. +6. **Internal consistency holds at full scale, with no sampling.** Every check passes on all + 81,001 bodies of the 8 outputs: one `@entry`/`@exit` each, every endpoint a body key, no + satellite endpoint, every node on an entry-to-exit path, no `cdg` into `@exit`, all + `prov == ["ssa"]`. +7. **The final-review fixes only add `ddg` edges.** Against the pre-fix binary, `body`, `cfg` and + `cdg` are identical on every app, no `ddg` edge is removed, and `ddg` grows by 0.6–1.9% on the + hand-written apps (mostly I7's deferred-call uses) and by 14.6% on `k8s-api` (C1's subslice + writes, `MarshalToSizedBuffer(dAtA[:i])`). + +## How to reproduce + +```bash +cd codeanalyzer-go +scripts/eval_l3.sh --prepare # clone at the pinned commits, prep, type-check (all apps) +scripts/eval_l3.sh cobra listmonk memos nats-server openbao dae kubernetes k8s-api + # builds $EVAL_OUT/cango with go1.27.3, then runs + +# L3 metrics and consistency checks, one JSON object per app: +python3 -I scripts/l3_metrics.py cobra=/tmp/eval-out/cobra-a3j1/analysis.json \ + k8s-api=/tmp/eval-out/k8s-api-a3j1/analysis.json ... > metrics.jsonl +``` + +- **What `--prepare` does.** For each app in the script's manifest: a shallow, blob-filtered fetch + of the pinned commit into `/tmp/eval/` (an existing clone is reused only at that commit), + a tool check, the app's prep (dae: the `dae_bpf_headers` submodule and `make ebpf`, which needs + `clang` and `llvm-strip` from Homebrew `llvm@18`), and a type-check with + `go list -e -export ./...`. It writes the result to `/tmp/eval-out/prepared/.env`. +- **What `eval_l3.sh` does.** It runs a warm-up, `-a 2 -j 1`, `-a 3 -j 1` and two `-a 3 -j N`, + all `--eager`, with each app's manifest env (dae: `GOOS=linux`). It records wall time and peak + RSS from `/usr/bin/time -l`, then compares the three `-a 3` outputs byte for byte. +- **What `scripts/l3_metrics.py` does.** It produces every count, ratio, distribution and check in + this note. It has no dependencies and loads one file at a time; the 318 MB Kubernetes root file + takes about 10 s. The per-family protobuf breakdown and the before/after edge diffs were + one-off scripts over the same outputs. + +## Corpus + +LOC for the six apps is from the analyzer plan's Task 13 table (2026-10-23) and excludes tests, +generated files and vendor. Kubernetes LOC is by `wc -l` over non-test `.go` files outside +`vendor/` and `staging/`, excluding `zz_generated*`, `generated.pb.go` and `*_generated.go` files +(for `k8s-api`, the module's own non-generated files). "Source analyzed" is the total +`module.source` in `analysis.json`, which includes generated files. + +| app | domain | pinned | LOC | source analyzed | why it is in the set | +|---|---|---|---|---|---| +| spf13/cobra | CLI framework (library) | `adbc881` | 7k | 0.2 MB | smoke test; no concurrency | +| knadh/listmonk | newsletter / mailing-list manager | `8216c4f` | 25k | 0.7 MB | small HTTP-to-SQL app: echo handlers into `sqlx` prepared statements | +| usememos/memos | note-taking service | `28719a9` | 96k | 3.5 MB | HTTP to SQL through an interface (`store.Driver`, three implementations) | +| nats-io/nats-server | messaging server | `f8f6db8` | 151k | 4.3 MB | CFG stress: cyclomatic p99 56, max 327; 140 `select`, 855 `defer` | +| openbao/openbao | secrets manager (Vault fork) | `1d58653` | 294k | 7.3 MB | large app; 1,944 `defer` | +| daeuniverse/dae | eBPF transparent proxy | `e3fee8f` | 65k | 1.9 MB | eBPF, cgo-adjacent, heavy `select`; Linux-only | +| kubernetes/kubernetes (root module) | container orchestration | `f78e722` (v1.37.1) | 748k | 34.6 MB | **scale benchmark**: Apache-2.0, fully type-checked from a plain checkout | +| kubernetes/kubernetes `k8s.io/api` | Kubernetes API types | `f78e722` (v1.37.1) | 50k (+316k `generated.pb.go`) | 12.4 MB | generated gogoproto marshallers (`dAtA[...]`): the I5 DDG-volume question | + +Why Kubernetes and not cockroach: + +- cockroach is now under the proprietary CockroachDB Software License, so it cannot serve as a + published benchmark. +- Without Bazel (`./dev generate go`), cockroach is only partly type-checked: 62,908 type-error + warnings, and at prepare time 973 of 1,380 packages with errors. +- Kubernetes commits its generated code, including + `staging/src/k8s.io/api/**/generated.pb.go`, so a plain checkout type-checks fully (1,468 and 61 + packages, 0 with errors) and needs no prep. + +The plan's counts were taken with other tools and do not always match what cango analyzes: + +- **Module scope.** cango loads one Go module per run. + - openbao's 1,944 `defer` include the `api/` and `sdk/` submodules (533 by grep). The root + module's are about 1,413, against 1,411 deferred call nodes emitted. + - nats-server's 855 include 10 in non-host platform files. That leaves 845, against 844 + emitted. + - Kubernetes is a workspace of 34 modules (`go.work`); the corpus runs the root and `k8s.io/api`. +- **Cyclomatic.** The plan's "max 327" for nats-server is from a different counter. cango's + `metrics.cyclomatic` for `server.client.parse` is 453. + +## Per-app counts + +The columns fall into three groups. + +**Structure.** + +- `modules`: `symbol_table` entries. +- `types`: type declarations of every `kind` (struct, interface, alias, defined). +- `callables`: every callable node, closures included at any nesting depth. +- `w/ body`: callables that carry a `cfg`. The difference from `callables` is exactly the + interface methods (signature only). + +**Nodes.** + +- `body nodes`: every body-map entry of a callable with a body. +- `satellite`: `call` nodes with a `parent`. They are nested inside a statement and are never CFG + steps. +- `deferred` / `goroutine`: `call` nodes with `is_deferred` / `is_goroutine`. + +**Edges.** Each overlay is an edge list. + +- `cfg`: control-flow edges, deduplicated by `(src, dst, kind)`. +- `cdg`: control-dependence edges `{src, dst}`. +- `ddg`: def-to-use pairs, one per `(def node, use node, use spelling)`. So this column is the + du-pair count. + +| app | modules | types | callables | functions / methods / closures | w/ body | body nodes | satellite | deferred | goroutine | cfg | cdg | ddg | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| cobra | 18 | 17 | 320 | 106 / 164 / 50 | 319 | 3,671 | 1,106 | 14 | 0 | 4,166 | 2,239 | 6,319 | +| listmonk | 101 | 191 | 820 | 200 / 533 / 87 | 800 | 16,252 | 5,863 | 51 | 17 | 17,502 | 9,815 | 39,391 | +| memos | 320 | 770 | 6,076 | 1,350 / 4,193 / 533 | 5,536 | 68,823 | 20,986 | 424 | 14 | 75,289 | 39,551 | 84,100 | +| nats-server | 97 | 749 | 5,279 | 1,114 / 3,412 / 753 | 5,041 | 105,409 | 24,890 | 844 | 81 | 141,829 | 81,598 | 345,318 | +| openbao | 702 | 1,136 | 8,712 | 1,898 / 5,664 / 1,150 | 8,447 | 147,951 | 41,526 | 1,411 | 125 | 177,091 | 98,844 | 234,460 | +| dae (linux) | 226 | 482 | 2,891 | 884 / 1,535 / 472 | 2,828 | 43,399 | 11,173 | 451 | 74 | 52,476 | 29,609 | 68,070 | +| **6 apps** | **1,464** | **3,345** | **24,098** | **5,552 / 15,501 / 3,045** | **22,971** | **385,505** | **105,544** | **3,195** | **311** | **468,353** | **261,656** | **777,658** | +| kubernetes | 3,414 | 4,890 | 44,514 | 18,788 / 16,132 / 9,594 | 42,603 | 492,627 | 158,009 | 2,346 | 388 | 515,474 | 280,473 | 837,658 | +| k8s-api | 494 | 1,462 | 15,427 | 504 / 14,500 / 423 | 15,427 | 356,174 | 82,880 | 0 | 0 | 407,985 | 264,677 | 626,883 | +| **8 incl. Kubernetes** | **5,372** | **9,697** | **84,039** | **24,844 / 46,133 / 13,062** | **81,001** | **1,234,306** | **346,433** | **5,541** | **699** | **1,391,812** | **806,806** | **2,242,199** | + +`call_graph` edges, for reference, are L2 and outside this note. All are from the same HEAD runs: + +| app | edges | in-project / external | +|---|---|---| +| cobra | 803 | 505 / 298 | +| listmonk | 3,703 | 1,439 / 2,264 | +| memos | 12,009 | 4,248 / 7,761 | +| nats-server | 19,384 | 11,630 / 7,754 | +| openbao | 23,914 | 10,431 / 13,483 | +| dae | 7,985 | 3,230 / 4,755 | +| kubernetes | 58,735 | 33,157 / 25,578 | +| k8s-api | 24,127 | 16,500 / 7,627 | + +## Edge and node kinds + +### Body nodes by kind + +| app | `call` | `statement` | `branch` | `loop` | `switch` | `return` | `entry` | `exit` | `call` share | satellite share of `call` | +|---|---|---|---|---|---|---|---|---|---|---| +| cobra | 1,474 | 633 | 448 | 88 | 7 | 383 | 319 | 319 | 40.2% | 75% | +| listmonk | 7,054 | 3,017 | 2,174 | 254 | 59 | 2,094 | 800 | 800 | 43.4% | 83% | +| memos | 23,781 | 12,481 | 8,914 | 690 | 216 | 11,669 | 5,536 | 5,536 | 34.6% | 88% | +| nats-server | 38,324 | 27,310 | 16,222 | 1,905 | 598 | 10,968 | 5,041 | 5,041 | 36.4% | 65% | +| openbao | 51,914 | 33,202 | 21,007 | 1,829 | 1,099 | 22,006 | 8,447 | 8,447 | 35.1% | 80% | +| dae | 15,353 | 9,412 | 6,015 | 753 | 355 | 5,855 | 2,828 | 2,828 | 35.4% | 73% | +| **6 apps** | **137,900** | **86,055** | **54,780** | **5,519** | **2,334** | **52,975** | **22,971** | **22,971** | **35.8%** | **77%** | +| kubernetes | 193,415 | 90,334 | 51,468 | 8,208 | 1,247 | 62,749 | 42,603 | 42,603 | 39.3% | 82% | +| k8s-api | 87,808 | 110,567 | 57,208 | 10,180 | 1,282 | 58,275 | 15,427 | 15,427 | 24.7% | 94% | + +`switch` covers expression `switch`, type `switch` and `select`, one kind per spec §4.2. Each +`case`/`default`/comm-clause head is a `statement`. The split, recovered from the first keyword of +each `switch` node's span, follows. Cases are `switch_case` out-edges per header (mean / max). + +| app | expression `switch` | type `switch` | `select` | `select` per 1k body nodes | `for {}` headers | callables with a defer chain | max defers in one callable | `if err != nil` share of `branch` | +|---|---|---|---|---|---|---|---|---| +| cobra | 7 (4.4 / 5) | 0 | 0 | 0.00 | 0 | 14 | 1 | 12% | +| listmonk | 44 (3.9 / 8) | 5 (3.2 / 4) | 10 (2.0 / 2) | 0.62 | 9 | 46 | 4 | 41% | +| memos | 161 (4.4 / 35) | 43 (4.9 / 23) | 12 (2.2 / 4) | 0.17 | 12 | 409 | 3 | 39% | +| nats-server | 404 (4.7 / **77**) | 56 (3.2 / 12) | 138 (2.8 / **12**) | 1.31 | 70 | 730 | 12 | 12% | +| openbao | 833 (3.4 / 24) | 87 (2.9 / 6) | 179 (2.2 / 6) | 1.21 | 106 | 1,078 | **20** | 38% | +| dae | 197 (3.7 / 22) | 28 (3.4 / 6) | 130 (2.1 / 5) | **3.00** | 94 | 369 | 5 | 17% | +| kubernetes | 969 (3.3 / 25) | 91 (4.2 / 18) | 187 (2.1 / 8) | 0.38 | 137 | 1,992 | 10 | 21% | +| k8s-api | 1,282 (4.5 / 44) | 0 | 0 | 0.00 | **5,748** | 0 | 0 | 11% | + +How the last three columns are counted: + +- `for {}` headers are `loop` nodes whose only non-`exception` out-edge is `fallthrough`. Those are + infinite loops, which get the synthetic `exception` edge to `@exit`. In `k8s-api` they are the + generated varint decoders, `for shift := uint(0); ; shift += 7 { … }`, which have no condition + (5,748, one per field decode loop in `Unmarshal`). +- `if err != nil` counts `branch` nodes whose source text matches `if [init;] err != nil {`. +- "Max defers" is the most `is_deferred` call nodes in one body, i.e. the longest defer chain. + +### CFG edges by kind + +| app | `fallthrough` | `true` | `false` | `switch_case` | `loop_back` | `exception` | `return` | `break` | `continue` | `defer_resume` | +|---|---|---|---|---|---|---|---|---|---|---| +| cobra | 1,115 | 536 | 492 | 31 | 54 | 1,369 | 522 | 10 | 23 | 14 | +| listmonk | 4,606 | 2,419 | 2,391 | 207 | 210 | 5,347 | 2,190 | 20 | 61 | 51 | +| memos | 18,407 | 9,592 | 9,495 | 951 | 554 | 22,668 | 13,014 | 27 | 157 | 424 | +| nats-server | 40,195 | 18,056 | 17,398 | 2,476 | 2,019 | 46,427 | 13,516 | 236 | 662 | 844 | +| openbao | 47,283 | 22,727 | 22,356 | 3,440 | 1,447 | 54,066 | 23,670 | 157 | 534 | 1,411 | +| dae | 14,273 | 6,670 | 6,423 | 1,106 | 632 | 15,679 | 6,932 | 70 | 240 | 451 | +| **6 apps** | **125,879** | **60,000** | **58,555** | **8,211** | **4,916** | **145,556** | **59,844** | **520** | **1,677** | **3,195** | +| kubernetes | 150,551 | 59,476 | 57,341 | 3,993 | 6,464 | 161,502 | 71,770 | 414 | 1,617 | 2,346 | +| k8s-api | 121,312 | 61,640 | 61,640 | 5,703 | 8,389 | 84,057 | 59,496 | 5,748 | 0 | 0 | + +### Ratios + +Per-callable figures divide by callables with a body. + +| app | cfg / callable | cdg / callable | ddg / callable | ddg / cdg | `call` share of body | `exception` share of cfg | `defer_resume` share of cfg | `fallthrough` share of cfg | +|---|---|---|---|---|---|---|---|---| +| cobra | 13.1 | 7.0 | 19.8 | 2.82 | 40.2% | 32.9% | 0.34% | 26.8% | +| listmonk | 21.9 | 12.3 | 49.2 | 4.01 | 43.4% | 30.6% | 0.29% | 26.3% | +| memos | 13.6 | 7.1 | 15.2 | 2.13 | 34.6% | 30.1% | 0.56% | 24.4% | +| nats-server | 28.1 | 16.2 | 68.5 | 4.23 | 36.4% | 32.7% | 0.60% | 28.3% | +| openbao | 21.0 | 11.7 | 27.8 | 2.37 | 35.1% | 30.5% | 0.80% | 26.7% | +| dae | 18.6 | 10.5 | 24.1 | 2.30 | 35.4% | 29.9% | 0.86% | 27.2% | +| **6 apps** | **20.4** | **11.4** | **33.9** | **2.97** | **35.8%** | **31.1%** | **0.68%** | **26.9%** | +| kubernetes | 12.1 | 6.6 | 19.7 | 2.99 | 39.3% | 31.3% | 0.46% | 29.2% | +| k8s-api | 26.4 | 17.2 | 40.6 | 2.37 | 24.7% | 20.6% | 0.00% | 29.7% | + +`k8s-api` is the outlier on the control side: generated `Unmarshal` bodies are long byte loops +with a wide field `switch`, so the `cfg` per callable is the highest in the corpus, while the +`exception` share is the lowest (most statements are index arithmetic on locals, which cannot +panic, rather than calls). + +### DDG detail + +**Edges per callable.** The distribution is over callables with a body. "Top 1%" is the share of +the app's `ddg` held by its 1% of callables with the most `ddg` edges. + +| app | median | p90 | p99 | max | callables with 0 `ddg` | top 1% hold | callables > 1,000 edges (their share) | +|---|---|---|---|---|---|---|---| +| cobra | 5 | 34 | 322 | 722 | 7 | 29% | 0 | +| listmonk | 13 | 57 | 283 | 14,752 | 9 | 53% | 4 (48%) | +| memos | 4 | 41 | 138 | 1,259 | 864 | 18% | 1 (1%) | +| nats-server | 9 | 89 | 935 | 63,747 | 102 | 47% | 46 (46%) | +| openbao | 7 | 58 | 326 | 5,819 | 595 | 28% | 18 (14%) | +| dae | 8 | 47 | 219 | 3,810 | 62 | 30% | 4 (17%) | +| **6 apps** | | | | | | | **73 (29%)** | +| kubernetes | 4 | 30 | 160 | **130,304** | 3,003 | 45% | 50 (30%) | +| k8s-api | 5 | 101 | 539 | 14,796 | 1,969 | 29% | 45 (17%) | + +**Top-3 callables by `ddg` count.** The last column is the single `var` with the most edges. It +gives the number of distinct def nodes and distinct use nodes that var's edges touch, and the +density (edges / (defs x uses)). + +| app | callable | `ddg` | body nodes | cyclomatic | dominant `var` | +|---|---|---|---|---|---| +| cobra | `github.com/spf13/cobra.Command.execute` | 722 | 111 | 37 | `c`: 342 edges, 25 defs x 32 uses (43%) | +| cobra | `github.com/spf13/cobra.Command.getCompletions` | 625 | 154 | 39 | `finalCmd`: 351, 22 x 30 (53%) | +| cobra | `github.com/spf13/cobra.defaultUsageFunc` | 461 | 90 | 21 | `w`: 229, 21 x 21 (52%) | +| listmonk | `github.com/knadh/listmonk/cmd.initHTTPHandlers` | 14,752 | 510 | 9 | `a`: 7,175, 60 x 181 (66%) | +| listmonk | `…/internal/messenger/postback.easyjsonDf11841fDecode…Postback2` | 1,682 | 102 | 23 | `in`: 1,457, 41 x 42 (85%) | +| listmonk | `…/internal/messenger/postback.easyjsonDf11841fDecode…Postback` | 1,506 | 95 | 25 | `in`: 1,169, 37 x 38 (83%) | +| memos | `…/proto/gen/api/apiconnect.NewUserServiceHandler` | 1,259 | 178 | 37 | `userServiceMethods`: 595, 34 x 34 (51%) | +| memos | `…/proto/gen/api.RegisterUserServiceHandlerClient` | 663 | 37 | 69 | `mux`: 595, 34 x 34 (51%) | +| memos | `…/proto/gen/api.RegisterUserServiceHandlerServer` | 663 | 37 | 69 | `mux`: 595, 34 x 34 (51%) | +| nats-server | `github.com/nats-io/nats-server/v2/server.client.parse` | 63,747 | 917 | 453 | `c`: 50,840, 237 x 232 (**92%**) | +| nats-server | `…/server.raft.processAppendEntry` | 9,065 | 358 | 89 | `n`: 5,404, 132 x 146 (28%) | +| nats-server | `…/server.Server.updateAccountClaimsWithRefresh` | 4,231 | 410 | 120 | `a`: 1,999, 78 x 53 (48%) | +| openbao | `github.com/openbao/openbao/v2/internal/command.ServerCommand.Run` | 5,819 | 548 | 120 | `c`: 1,513, 120 x 43 (29%) | +| openbao | `…/internal/command/server.Config.Merge` | 4,133 | 177 | 50 | `result`: 3,918, 93 x 87 (48%) | +| openbao | `…/internal/vault.NewTestCluster` | 2,649 | 344 | 70 | `coreConfig`: 1,172, 53 x 48 (46%) | +| dae | `github.com/daeuniverse/dae/cmd.reloadWorker.run` | 3,810 | 346 | 53 | `w.log`: 1,560, 40 x 39 (**100%**) | +| dae | `…/cmd.Runner.Run` | 3,790 | 419 | 99 | `reloadManager`: 1,127, 38 x 38 (78%) | +| dae | `…/control.NewControlPlaneWithContextOptions` | 2,496 | 480 | 106 | `log`: 955, 41 x 46 (51%) | +| kubernetes | `k8s.io/kubernetes/pkg/apis/core/v1.RegisterConversions` | **130,304** | 2,548 | 510 | `s`: 129,795, 509 x 509 (50%) | +| kubernetes | `…/pkg/printers/internalversion.AddHandlers` | 9,247 | 616 | 1 | `h`: 9,045, 134 x 134 (50%) | +| kubernetes | `…/pkg/kubelet/apis/config/v1beta1.autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration` | 6,247 | 258 | 38 | `out`: 5,344, 126 x 89 (48%) | +| k8s-api | `k8s.io/api/core/v1.PodSpec.MarshalToSizedBuffer` | 14,796 | 558 | 75 | `dAtA`: 13,877, 165 x 166 (51%) | +| k8s-api | `k8s.io/api/core/v1.VolumeSource.MarshalToSizedBuffer` | 6,935 | 369 | 61 | `dAtA`: 5,672, 106 x 107 (50%) | +| k8s-api | `k8s.io/api/core/v1.PodSpec.Unmarshal` | 6,640 | 1,380 | 443 | `m`: 2,752, 64 x 43 (100%) | + +**`var` classes.** Classes are by the spelling of the edge's `var`: + +- plain: no wildcard; +- `[*]`: has an element step; +- trailing `.*`: a dereference, k-limit truncation, or callee-may-write wildcard; +- both: `[*]` and a trailing `.*`; +- `:` pseudo: the defer registration to deferred-call path. + +"Bare base" means the `var` is a single identifier (`c`, not `c.f`), which matches every reaching +def under that base. "Def at/after use" means the def node's position is not before the use's. +It is an approximate count of loop-carried edges. + +| app | plain | `[*]` | trailing `.*` | both | `:` | wildcard or pseudo | bare base | def at/after use | +|---|---|---|---|---|---|---|---|---| +| cobra | 96.74% | 2.07% | 0.97% | 0.00% | 0.22% | 3.26% | 79% | 5% | +| listmonk | 98.39% | 0.99% | 0.49% | 0.00% | 0.13% | 1.61% | 83% | 8% | +| memos | 96.43% | 1.71% | 1.36% | 0.00% | 0.50% | 3.57% | 79% | 4% | +| nats-server | 96.13% | 1.42% | 2.20% | 0.01% | 0.24% | 3.87% | 71% | 18% | +| openbao | 95.96% | 1.78% | 1.65% | 0.01% | 0.60% | 4.04% | 69% | 6% | +| dae | 95.73% | 2.13% | 1.45% | 0.02% | 0.66% | 4.27% | 67% | 12% | +| **6 apps** | **96.20%** | **1.61%** | **1.78%** | **0.01%** | **0.41%** | **3.80%** | **72%** | **12%** | +| kubernetes | 95.62% | 1.56% | 2.43% | 0.11% | 0.28% | 4.38% | 80% | 4% | +| k8s-api | 93.39% | **5.21%** | 1.40% | 0.00% | 0.00% | 6.61% | 85% | **20%** | + +No pseudo path other than `:` appears in any output. The trailing-`.*` share rose from +0.64% (pre-fix) to 1.78% on the 6 apps: I7 makes every deferred call *use* the `recv.*` / `a.*` +paths it weak-defines, so `defer mu.Unlock()` now reads `mu.*` (§ What the final-review fixes +changed). + +**`ddg` by file class.** The class is assigned by the file the callable is declared in: +`*.pb.go` is protobuf, `*.eg.go` is cockroach `execgen`, and easyjson / `_gen.go` / `/gen/` / +`zz_generated*` files are other generated code. + +| app | hand-written | protobuf | other generated | +|---|---|---|---| +| listmonk | 84.1% | — | 15.9% | +| memos | 60.8% | 35.8% | 3.4% | +| openbao | 99.0% | 1.0% | — | +| dae | 99.5% | 0.5% | — | +| cobra, nats-server | 100% | — | — | +| **6 apps** | **94.6%** | **4.2%** | **1.2%** | +| kubernetes | 61.4% | 0.0% | **38.6%** (`zz_generated.conversion.go` etc.) | +| k8s-api | 0.3% | **93.1%** | 6.6% (`zz_generated.deepcopy.go`) | + +## Protobuf-generated DDG volume (`k8s-api`, review finding I5) + +The question: how much `ddg` do gogoproto-style generated marshallers produce at real scale? A +reviewer probe of one synthetic 600-field marshaller produced **1.27M `ddg` edges and 77 MB of +JSON**, and no tree in the earlier corpus contained a `dAtA[` marshaller to check it against. +`k8s.io/api` does: 60 committed `generated.pb.go` files (316k lines, 7.3 MB) holding 1,220 +messages, each with `Marshal`, `MarshalTo`, `MarshalToSizedBuffer`, `Size`, `Unmarshal`, `String` +and `Reset`, all type-checked. + +**Totals.** `ddg` is 626,883 edges for 15,427 callables, 93.1% of them in `.pb.go` code. The +`-a 3` file is 183.4 MB, 3.3x the `-a 2` file (55.6 MB), the largest growth in the corpus. +The `ddg` section alone is 37 MB of it (about 62 bytes per edge). Wall time is 5.1 s at +`-j 1` (L3: +1.9 s), 2.2 GB RSS. + +**By generated method family** (each family has 1,220 bodies; "L3 bytes per node" is the +re-serialized `body`+`cfg`+`cdg`+`ddg` of the family divided by its body nodes): + +| family | `ddg` | share | median | p90 | p99 | max | body nodes | `ddg` per node | L3 MiB | L3 bytes per node | +|---|---|---|---|---|---|---|---|---|---|---| +| `Unmarshal` | 311,110 | 49.6% | 173 | 443 | 1,359 | 6,640 | 175,274 | 1.77 | 77.0 | 461 | +| `MarshalToSizedBuffer` | 208,977 | 33.3% | 73 | 296 | 1,477 | 14,796 | 54,150 | 3.86 | 30.0 | 581 | +| `Size` | 23,984 | 3.8% | 13 | 35 | 119 | 588 | 31,723 | 0.76 | 11.6 | 383 | +| `Marshal` | 13,420 | 2.1% | 11 | 11 | 11 | 11 | 13,420 | 1.00 | 5.1 | 396 | +| `String` (1,209) | 11,056 | 1.8% | 6 | 18 | 43 | 149 | 18,828 | 0.59 | — | — | +| other `.pb.go` (`MarshalTo`, `Reset`, `skipGenerated`, varint helpers, …) | 14,940 | 2.4% | | | | | | | | | +| `zz_generated.deepcopy.go` and closures | 41,620 | 6.6% | | | | | | | | | +| hand-written (`types.go` helpers) | 1,776 | 0.3% | | | | | | | | | + +Over the whole module, the `-a 3` file is 540 bytes per body node (the corpus's lowest; generated +nodes have short spans), and 14.9 bytes per byte of source. + +**Top callables.** All ten largest are `core/v1` marshallers: `PodSpec.MarshalToSizedBuffer` +14,796; `VolumeSource.MarshalToSizedBuffer` 6,935; `PodSpec.Unmarshal` 6,640; +`VolumeSource.Unmarshal` 5,183; `Container.` and `EphemeralContainerCommon.MarshalToSizedBuffer` +4,181 each; `PodStatus.MarshalToSizedBuffer` 3,567; `PersistentVolumeSource.MarshalToSizedBuffer` +3,519 and `.Unmarshal` 3,111; `ServiceSpec.MarshalToSizedBuffer` 3,006. + +**The mechanism is the probe's.** `MarshalToSizedBuffer` writes the buffer backwards: each field is +`i -= len(x); copy(dAtA[i:], x); i = encodeVarint(dAtA, i, …); i--; dAtA[i] = 0x…`, or for a nested +message `m.X.MarshalToSizedBuffer(dAtA[:i])`. Every one of those statements weak-defines +`dAtA[*]` (an index store, `copy` into a subslice, or a call with the subslice argument, C1), and +none kills, so every later use of `dAtA` meets all of them. In `PodSpec.MarshalToSizedBuffer`, +`dAtA` has 13,877 edges between 165 defs and 166 uses, 51% of the full matrix: the exact triangle +of a straight-line sequence. `i` adds only 624, because `i -= …` is a strong def. `Unmarshal` is +different: its volume is spread over the field `switch` (`m` and `iNdEx` in a loop), with a lower +`ddg` per node (1.77) but many more nodes. + +**Scaling, and the comparison with the probe.** Over the 226 messages with at least 5 fields, a +log-log fit of `MarshalToSizedBuffer` `ddg` against the message's field count gives about +**10 x fields^1.8**, i.e. slightly sub-quadratic. Extrapolated to 600 fields that is about 1.07M +edges, the same order as the probe's 1.27M (the probe's exact code is not recorded, so only the +order is comparable). So the probe was right about the mechanism and the growth rate. What it did not tell is the distribution: real +Kubernetes messages have a median of 3 fields and a maximum of 43 (`PodSpec`), so the largest +real marshaller is 14,796 edges, 86x below the probe, and the whole module's 2,440 marshal and +unmarshal bodies together hold 520k edges, 0.4x one probe. + +**Verdict on I5.** On real gogoproto output the volume is large but bounded. It is the biggest +L3 file per source byte in the corpus (3.3x growth), but no single callable is an outlier on the +scale of `RegisterConversions` (130k) or `client.parse` (64k). A guard is not needed for +Kubernetes-sized messages. It would still matter for a schema with hundred-field messages, since +the cost per message grows as fields^1.8. A kill or summarization rule for sequential indexed +writes to one buffer (`dAtA[i] = …` after `i--`) would collapse the triangle; it is not +implemented and not needed for this corpus. + +## Cost: time, memory, output size + +**Time and memory.** All runs are `--eager`, one timed run each, with wall time and peak RSS from +`/usr/bin/time -l` (MB = 2^20 bytes). All rows are the one HEAD pass (`9aeda43` binary). + +- **L3 delta and throughput.** The L3 delta is `-a 3 -j 1` minus `-a 2 -j 1`. Throughput is body + nodes divided by that delta. +- **Noise.** Repeated `-a 2 -j 1` runs of one binary on openbao spread over 3.75–4.37 s, so a + single wall time is good to roughly ±0.2 s on the mid-size apps and ±0.05 s on the small ones. + cobra's negative delta (-0.05 s) is noise. + +| app | `-a 2 -j 1` (s) | `-a 3 -j 1` (s) | L3 delta | L3 share of `-a 3` | `-a 3 -j 16` (s) | `-j 16` speedup | 2nd `-j 16` (s) | RSS `-a 2` | RSS `-a 3 -j 1` | RSS `-a 3 -j 16` | L3 throughput (nodes/s) | +|---|---|---|---|---|---|---|---|---|---|---|---| +| cobra | 0.47 | 0.42 | -0.05 | n/a | 0.41 | 1.02x | 0.43 | 243 | 263 | 252 | n/a | +| listmonk | 0.80 | 0.97 | +0.17 | 18% | 0.86 | 1.13x | 0.90 | 915 | 936 | 885 | 95,600 | +| memos | 2.26 | 2.52 | +0.26 | 10% | 2.32 | 1.09x | 2.25 | 2,308 | 2,613 | 2,364 | 264,703 | +| nats-server | 1.44 | 2.24 | +0.80 | 36% | 1.74 | 1.29x | 1.77 | 1,034 | 1,078 | 1,155 | 131,761 | +| openbao | 4.13 | 4.98 | +0.85 | 17% | 4.39 | 1.13x | 4.41 | 4,983 | 5,082 | 4,966 | 174,059 | +| dae (linux) | 1.43 | 1.75 | +0.32 | 18% | 1.63 | 1.07x | 1.58 | 1,332 | 1,534 | 1,461 | 135,621 | +| **6 apps** | **10.53** | **12.88** | **+2.35** | **18%** | **11.35** | **1.13x** | | | | | **164,044** | +| kubernetes | 9.62 | 12.43 | +2.81 | 23% | 11.76 | 1.06x | 10.92 | 6,961 | 7,135 | 7,592 | 175,312 | +| k8s-api | 3.13 | 5.06 | +1.93 | 38% | 3.97 | 1.27x | 4.02 | 2,043 | 2,198 | 2,203 | 184,546 | + +**Output size.** `analysis.json` sizes are on disk. The per-section bytes ("L3 composition") were +measured by re-serializing each callable's `body`, `cfg`, `cdg` and `ddg` compactly, as cango +writes them, and are given as a share of the L3 delta in bytes: + +- body-node `id`s: the `"id":"@"` string on every body node L3 adds (the + non-`call` nodes). Since R15 (`9ae739e`), `call` nodes carry their `id` from L1, so those ids + are in the `-a 2` file and not in the delta; +- other: the new statement nodes' spans, kinds and `parent` fields. + +| app | `-a 2` MB | `-a 3` MB | growth | L3 share of file | bytes / body node | L3 composition: body-node `id`s / `ddg` / `cfg` / `cdg` / other | `-a 3` bytes per source byte | +|---|---|---|---|---|---|---|---| +| cobra | 1.0 | 2.0 | 2.1x | 51% | 567 | 22 / 36 / 19 / 6 / 17% | 9.0 | +| listmonk | 4.7 | 10.0 | 2.1x | 53% | 643 | 23 / 41 / 15 / 5 / 15% | 13.9 | +| memos | 19.5 | 39.8 | 2.0x | 51% | 606 | 35 / 25 / 17 / 6 / 18% | 11.5 | +| nats-server | 27.3 | 72.8 | 2.7x | 63% | 724 | 23 / 44 / 15 / 5 / 13% | 16.9 | +| openbao | 42.6 | 94.5 | 2.2x | 55% | 670 | 36 / 27 / 16 / 6 / 15% | 13.0 | +| dae | 11.5 | 25.3 | 2.2x | 55% | 612 | 31 / 29 / 17 / 6 / 16% | 13.3 | +| **6 apps** | **106.5** | **244.4** | **2.3x** | **56%** | **665** | **31 / 33 / 16 / 6 / 15%** | **13.7** | +| kubernetes | 158.2 | 317.6 | 2.0x | 50% | 676 | 34 / 31 / 15 / 5 / 15% | 9.2 | +| k8s-api | 55.6 | 183.4 | **3.3x** | **70%** | 540 | 31 / 29 / 15 / 6 / 19% | 14.9 | + +The `-a 2` files are about 28% larger than in the pre-fix evaluation (6 apps: 82.9 to 106.5 MB) +because of R15's call-node `id`; the `-a 3` files barely moved (243.7 to 244.4 MB), because +those ids were already there at L3. Growth is accordingly lower than before (2.9x to 2.3x). + +## Internal-consistency checks + +Every check ran on **every** callable with a body (22,971 on the 6 apps, 58,030 on Kubernetes, +81,001 in all), with no sampling, using `scripts/l3_metrics.py`. + +| # | check | 6 apps | Kubernetes (both modules) | result | +|---|---|---|---|---| +| 1 | `entry` count == `exit` count == callables with a body, and each body has exactly one `@entry` and one `@exit` key | 22,971 = 22,971 = 22,971; 0 bodies off | 58,030 each; 0 off | **PASS** | +| 2 | every `cfg` / `cdg` / `ddg` endpoint is a body key of its own callable | 0 bad of 1,507,667 edges | 0 bad of 2,933,150 | **PASS** | +| 3 | no satellite `call` node is a `cfg`, `cdg` or `ddg` endpoint | 0 | 0 | **PASS** | +| 4 | every non-satellite node is reachable from `@entry` and reaches `@exit` (forward and backward BFS per body; full, not sampled) | 279,961 nodes checked; 0 unreachable, 0 not reaching exit | 607,912 checked; 0 / 0 | **PASS** | +| 5 | each `return` node has exactly one `return` edge; `return` edges = `return` nodes + fall-off-end edges | 0 off; 59,844 = 52,975 + 6,869 | 0 off; 131,266 = 121,024 + 10,242 | **PASS** | +| 6 | no `cdg` edge into `@exit`; no `cdg` self-edge | 0 / 0 | 0 / 0 | **PASS** | +| 7 | every `ddg` edge has `prov == ["ssa"]` | 0 of 777,658 differ | 0 of 1,464,541 | **PASS** | +| 8 | no duplicate `cfg` edge `(src, dst, kind)` | 0 | 0 | **PASS** | +| 9 | one `defer_resume` edge per deferred call node, and one `:` `ddg` edge per deferred call | 3,195 = 3,195 = 3,195 | 2,346 each | **PASS** | +| 10 | at most one `exception` edge per node (edges == nodes carrying one) | 145,556 = 145,556 | 245,559 = 245,559 | **PASS** | +| 11 | `-j 1` == `-j 16` == 2nd `-j 16`, byte-identical | 6 / 6 | 2 / 2 | **PASS** | +| 12 | no closure whose spans index another file lacks `source_file` (R19, `b87c636`) | 0 | 0 | **PASS** | + +Notes on checks 5 and 9: + +- **Check 5.** `return` is also the kind of a fall-off-end edge, so `return` edges exceed `return` + nodes by exactly the fall-off count. Fall-off edges leave the last straight-line node, branch or + loop of a body. An empty body has a single `@entry -> @exit` `return` edge: there are 478 of + those on the 6 apps (memos has 345, e.g. no-op marker methods such as + `filter.ComparisonCondition.isCondition`). 11,962 of the 6 apps' 59,844 `return` edges go to the + first member of a defer chain, not to `@exit`. +- **Check 9.** It confirms the §4.3.1 split end to end, by totals. Each `defer` gives a + registration `statement`, a deferred `call`, one `defer_resume` edge (chain-internal or tail to + `@exit`), and one registration-to-call `:` du-pair, and the three totals agree. + +`true` exceeds `false` by 1,445 on the 6 apps. That matches, to within 5, the 1,450 `branch`/`loop` +headers whose not-taken arm falls off the end of the function, which is therefore kind `return`, +not `false`. In `k8s-api`, `true` and `false` are exactly equal (61,640): no generated body ends +in a branch. + +### What the final-review fixes changed + +The previous version of this note measured the `fd24bdf` binary. HEAD adds four fixes that change +L3 or the file: C1 (`258b22f`, a subslice or array-slice argument weak-defines `base[*]`), I2 +(`6fdc2c5`, defs kept in packages with type errors), R15 (`9ae739e`, body-node `id` from L1) and +I7 (`3f31626`, a deferred call uses the paths it may write: `defer mu.Unlock()` uses `mu.*`). +Each pre-fix output was diffed against HEAD's per callable, edge by edge: + +| app | `ddg` before | after | added | removed | added into a deferred call (I7) | added elsewhere (C1) | +|---|---|---|---|---|---|---| +| cobra | 6,261 | 6,319 | +58 (+0.9%) | 0 | 58 | 0 | +| listmonk | 39,287 | 39,391 | +104 (+0.3%) | 0 | 99 | 5 | +| memos | 83,413 | 84,100 | +687 (+0.8%) | 0 | 503 | 184 | +| nats-server | 340,338 | 345,318 | +4,980 (+1.5%) | 0 | 3,900 | 1,080 | +| openbao | 230,816 | 234,460 | +3,644 (+1.6%) | 0 | 3,515 | 129 | +| dae (linux) | 66,825 | 68,070 | +1,245 (+1.9%) | 0 | 879 | 366 | +| **6 apps** | **766,940** | **777,658** | **+10,718 (+1.4%)** | **0** | **8,954** | **1,764** | +| kubernetes | 833,012 | 837,658 | +4,646 (+0.6%) | 0 | 4,518 | 128 | +| k8s-api | 547,122 | 626,883 | **+79,761 (+14.6%)** | 0 | 0 | 79,761 | + +- **Only `ddg` changed.** Per callable, `body`, `cfg` and `cdg` hash identically before and after + on all 8 apps (the body-node `id` strings were already present at L3 before R15), and no `ddg` + edge disappeared. Every fix widens the over-approximation; none narrows it. +- **The non-deferred additions are almost all C1.** In nats-server, 1,079 of the 1,080 are defs at a + statement that slices (`a.w.Write(p[:wanted])`, `binary.PutUvarint(tmp[:], v)`, + `tokenizeSubjectIntoSlice(rsa[:0], r)`); in openbao, `copy(result.Audits[len(c.Audits):], …)` + in `Config.Merge` (4,034 to 4,133 edges). +- **`k8s-api` is the C1 showcase.** All 79,761 added edges are on `dAtA` (59,077) or `dAtA[*]` + (20,684), in `MarshalToSizedBuffer` (+39,575, via `m.X.MarshalToSizedBuffer(dAtA[:i])` and + `copy(dAtA[i:], …)`) and `Unmarshal` (+38,966). Before C1 those calls were invisible as writes + to the buffer, a false refutation of exactly the protobuf flow; `PodSpec.MarshalToSizedBuffer` + went from 12,019 to 14,796. +- **I2 changed nothing measurable here.** All 8 trees type-check fully, so no def depended on it. + It matters for degraded inputs such as cockroach. +- **Bytes.** R15 moved about 23 MB of call-node ids from the L3 delta into the `-a 2` file on the + 6 apps; the `-a 3` totals grew 0.3% (243.7 to 244.4 MB, from the extra `ddg` edges). +- **Time and memory** moved within noise (6-app `-a 3 -j 1` 12.17 to 12.88 s; repeated runs of + one binary spread ±0.2 s), and L3 throughput is unchanged (165k to 164k body nodes/s). + +### Determinism + +All 8 outputs are byte-identical across `-j 1`, `-j 16` and a second `-j 16`. Neither Kubernetes +module emitted any stderr line (no type error, no toolchain warning, no `L3 skipped`). + +## Observations + +### Where L3 cost goes + +- **Loading and type-checking dominate.** L3 adds 2.35 s to 10.5 s of `-a 2` across the 6 apps + (18% of `-a 3` wall at `-j 1`), and 2.8 s to 9.6 s on the Kubernetes root. +- **`-j` only parallelizes L3.** `-j` feeds only the per-callable L3 worker pool + (`dataflow.Run`), so the `-j 16` speedup of the whole run (1.02–1.29x) is Amdahl-bound by the + serial front. It is largest where L3 is the largest share: nats-server (36%) and `k8s-api` (38%). +- **Edges, not nodes, seem to set L3's cost.** No per-phase profile was taken; this is an + inference from the throughput column. nats-server has 29% fewer body nodes than openbao but 47% + more `ddg` edges, and almost the same L3 delta (0.80 vs 0.85 s), so its throughput is lower + (132k vs 174k nodes/s). +- **The known worst case was not measured here.** A reviewer probe of one 6,000-case function took + 15 s in `BuildDDG`'s kill computation, which is O(nodes x defs) per body (§ Limitations). No + corpus function triggers it: the largest real body is `RegisterConversions` (2,548 nodes, 509 + defs of `s`), which the whole 2.8 s Kubernetes L3 delta absorbs. +- **Memory.** L3 adds 2–15% peak RSS at `-j 1` (most on dae and memos). `go/packages` type + information dominates RSS: the Kubernetes root needs 7.0 GB at `-a 2`, 7.1 GB at `-a 3`. + +### How Go idioms show up in the graph + +- **The error idiom is visible in the node mix.** `if err != nil` is 28% of all `branch` nodes on + the 6 apps (38–41% in the HTTP/SQL services listmonk, memos, openbao; 12% in cobra and + nats-server; 21% in the Kubernetes root). It accounts for much of Go's 2.4 `branch` and 2.3 + `return` nodes per callable. +- **Defer chains are short and common.** 2,646 of 22,971 bodies (12%) on the 6 apps have a defer + chain, and the median chain has one member. The longest is 20 defers (openbao) or 12 + (nats-server). + - Every exit funnels through the chain: 39,567 `exception` and 11,962 `return` edges enter a + chain rather than `@exit`. + - Deferred calls that may themselves panic add 549 chain-internal `exception` edges. + - `defer_resume` is under 1% of `cfg` edges everywhere. The defer model costs little volume. +- **`select` is where the concurrency-heavy apps differ.** dae has 3.0 `select` headers per 1,000 + body nodes, against 1.2–1.3 for nats-server and openbao, 0.4 for the Kubernetes root and at most + 0.6 for the rest. + - `select` fan-out is narrow: a mean of 2.1–2.8 comm clauses, max 12 in nats-server. + - Infinite `for {}` loops cluster in the same apps: 94 in dae, 70 in nats-server, 106 in openbao, + against 0–12 in the rest of the 6. +- **Wide switches are where cyclomatic complexity lives.** nats-server has the widest expression + `switch` (77 cases). `server.client.parse`, nats-server's protocol state machine, is cyclomatic + 453 in one body: a byte loop around a state `switch`. Kubernetes' largest cyclomatic is the + generated `RegisterConversions` (510), a straight sequence of `if err := s.AddGeneratedConversionFunc(…); err != nil { return err }`. + +### Over-approximation, measured + +- **Panic edges.** 62% of non-synthetic CFG nodes on the 6 apps (65% on the Kubernetes root) carry + an `exception` edge, so `exception` is the most common kind, ahead of `fallthrough` (31.1% vs + 26.9%). The cause is §4.3.2's rule: any call, index, dereference, division, send or unchecked + assertion may panic. This is the price of having the defer chain and `@formal_out` see + panicking paths. It is sound for taint, but it blurs the CFG for readers who want "normal" + control flow. Filtering on `kind != "exception"` recovers that view exactly, since each node has + at most one such edge. +- **Panic-induced control dependence.** 34.9% of `cdg` edges on the 6 apps (33.6% on Kubernetes) + come from a controller with no `true`/`false`/`switch_case` out-edge. Its only second successor + is its `exception` edge. So in `a(); b()`, `b` is control-dependent on `a`. A backward slice over + `cdg` therefore pulls in every earlier panic-capable statement on the path. A further 10.6% of + `cdg` edges are the `@entry` anchors for uncontrolled nodes (spec §4.4). +- **Weak defs, and defs of distinct fields, never kill each other.** A call weak-defines `a.*` + for every reference-typed argument and receiver, and `x.f = …` kills only `x.f…`. So along any + path the defs on a long-lived receiver (`c`, `s`, `mset`) only accumulate. 72% of `ddg` edges + are uses of a bare base (80–85% on Kubernetes), which by the interference rule (`u.Covers(d)`) + matches every reaching def under that base. +- **Wildcards themselves are a minor contributor in hand-written code.** They are 3.8% of 6-app + edges, mostly `[*]` element writes and (since I7) the `recv.*` uses of deferred calls. The + inflation comes from bare-base uses meeting accumulated weak defs, not from the wildcard + spelling. `k8s-api` is the exception: 5.2% `[*]`, from `dAtA[*]`. + +### The DDG long tail + +The long tail is one mechanism repeated: an inner loop or a long straight-line sequence of calls +on one reference-typed base. + +- **Kubernetes `core/v1.RegisterConversions`.** It holds 130,304 edges, 16% of the root module's + `ddg`, in one generated function (`zz_generated.conversion.go`). + - It is 509 consecutive `if err := s.AddGeneratedConversionFunc(…); err != nil { return err }`. + Each call weak-defines `s.*` (the `*runtime.Scheme`) and uses `s`, so every registration + links to every earlier one: 509 defs x 509 uses, 50% filled, the exact triangle + (509 x 510 / 2 = 129,795, which is the `s` count). + - The 100 `RegisterConversions` functions of the root module hold 191,433 edges, 23% of its + `ddg`; `zz_generated*` files hold 38.6%. + - `printers/internalversion.AddHandlers` (9,247; 134 `h.TableHandler(…)` calls) is the same + pattern in hand-written code. +- **nats-server `client.parse`.** It holds 63,747 edges, 18% of the app's `ddg`. + - 237 nodes define a path under `c`: field writes such as `c.state = …`, plus every call with + `c` as pointer receiver, which weak-defines `c.*`. 232 nodes use bare `c`. + - They sit inside a single loop, so every def reaches every use: 92% of the 237 x 232 matrix is + filled. + - 18% of nats-server's edges are def-at-or-after-use (loop-carried), the highest of the 6 apps. +- **listmonk `initHTTPHandlers`.** It holds 14,752 edges, 37% of the app's `ddg`. + - It is straight-line route registration: 497 call nodes, cyclomatic 9. + - Each `g.GET(...)` on the `*echo.Group` weak-defines `g.*` and uses `g`, so every registration + links to every earlier one: 185 defs x 185 uses, 21% filled, cut into blocks by the two + `g := e.Group(…)` and one `g = e.Group(…)` strong defs. The `a` (`*App`) edges are 66% + filled. + - Complexity metrics do not predict this tail. Call density on one base does. +- **openbao `Config.Merge`.** Each of 87 `result.X = …` statements strong-defines only + `result.X`, and also uses bare `result` to write through it. A def of `result.X` kills only + `result.X…`, so the defs of distinct fields accumulate, and each statement's `result` use + matches every earlier one: 3,918 `result` edges for 177 body nodes (93 defs since C1 adds the + `copy(result.Audits[…:], …)` writes). +- **Generated marshallers** (`k8s-api`): the same triangle on `dAtA`, § Protobuf-generated DDG + volume. + +### Output size + +**Output size is the main caution.** + +- **Growth.** `-a 3` is 2.3x `-a 2` on the 6 apps, 2.0x on the Kubernetes root and 3.3x on + `k8s-api`, at 540 to 724 bytes per body node and 9 to 17 bytes of JSON per byte of analyzed + source. The 318 MB Kubernetes root file loads whole in Python in about 10 s. +- **What the bytes are.** Body-node `id` strings are 27% of an `-a 3` file on the 6 apps (28–29% + on Kubernetes). They average about 165–225 bytes per node by app, because each repeats the full + callable id. Of the L3 delta alone, the ids L3 adds are 31% and `ddg` 33% (44% on nats-server, + the long-tail app). +- **The `id` is derivable.** It is `@`. It is present for SDK parity with Python + (#176, R15; spec §4.2), so dropping or compressing it is a schema-contract decision, not a free + win. The measurement says that decision is worth about a quarter of an `-a 3` file. + +### Context: Java's AST-engine figures + +The Java sibling's AST engine is the nearest syntactic L3, measured on a different corpus (ten +Java apps; see `codeanalyzer-java/docs/design/notes/l3-dataflow-metrics.md`). The ratios below +are context for consumers moving between languages, not a quality comparison. The two engines +make different, documented modelling choices, and the corpora differ in domain and size. + +| ratio | Go, 6 apps | Go, Kubernetes root | Java AST engine, 10 apps | +|---|---|---|---| +| body nodes / callable | 16.8 | 11.6 | 11.3 | +| `cfg` / callable | 20.4 | 12.1 | 6.6 | +| `cdg` / callable | 11.4 | 6.6 | 1.5 | +| `ddg` / callable | 33.9 | 19.7 | 3.2 | +| `ddg` / `cdg` | 2.97 | 2.99 | 2.12 | +| `call` share of body | 35.8% | 39.3% | 62.3% | +| `exception` share of `cfg` | 31.1% | 31.3% | 3.1% | +| `fallthrough` share of `cfg` | 26.9% | 29.2% | 75.5% | +| bytes / body node | 665 | 676 | about 1,700 (commons-lang only; derived from "~222 MB" / 128,961 nodes) | + +The large differences have identifiable causes. + +- **Exception edges.** Java's are syntactic throwers inside a `try` only. Go's `canPanic` applies + everywhere, because a panic always reaches the defer chain. That alone explains the `exception` + share, and through panic-induced control dependence, about a third of Go's `cdg`. +- **Go's error idiom adds explicit branches and returns.** Go has 2.4 branches and 2.3 returns per + callable, against Java's 0.28 and 0.55. That lowers `call`'s share of the body and adds `cdg`. +- **Go's weak-def rule for reference-typed arguments and receivers** has no counterpart that + would generate comparable `ddg` density in an object-insensitive Java AST pass. The ddg/cdg + ratio, which normalizes away the control-side differences, is the closest of the figures + (3.0 vs 2.1). + +## cockroach: very large, partly type-checked (pre-fix L3) + +cockroach `d30c905f` is kept for the record as a very large tree, not as a benchmark: it is under +the proprietary CockroachDB Software License, and without Bazel (`./dev generate go`) it has no +generated protobuf code (183 `.proto`, 0 `.pb.go`). `scripts/eval_l3.sh --prepare cockroach` +reports it `degraded: missing bazel or bazelisk; 973 packages with errors` (22 packages that fail +to compile, 951 more broken through them; 1,380 in all). It was **not** rerun with the HEAD +binary: every figure below is the `fd24bdf` binary (pre-fix: no C1, I2, R15 or I7), and none is +in any total above. + +| metric | cockroach (pre-fix L3, degraded) | +|---|---| +| modules / types / callables (functions / methods / closures) | 5,953 / 15,002 / 88,765 (21,223 / 53,058 / 14,484) | +| callables with a body / body nodes / satellite | 82,752 / 1,407,838 / 406,091 | +| `cfg` / `cdg` / `ddg` | 1,618,928 / 959,530 / 4,600,814 | +| per callable `cfg` / `cdg` / `ddg`; `ddg` / `cdg` | 19.6 / 11.6 / 55.6; 4.79 | +| `exception` share of `cfg`; `fallthrough` share | 29.7%; 30.5% | +| `ddg` per callable median / p90 / p99 / max; top 1% hold | 6 / 54 / 303 / 73,013; 65% | +| wildcard or pseudo `var`s; trailing `.*` | 15.31%; 13.82% (k-limit truncations in `.eg.go`) | +| `ddg` by file class | hand-written 38.7%, `execgen` 60.8%, other generated 0.6% | +| wall `-a 2` / `-a 3 -j 1` / `-a 3 -j 16`; RSS `-a 3 -j 1` | 23.08 / 33.36 / 26.96 s; 13,473 MB | +| `analysis.json` `-a 2` to `-a 3` | 335.5 to 1,112.4 MB (3.3x) | +| type-error warnings at run time | 62,908 across 633 packages | +| consistency checks; `-j` byte-identical (after `679f995`) | all PASS; yes | + +- **The `execgen` tail.** The top callables are three generated merge-joiner closures in + `colexecjoin` (73,013 edges each, 1,058 nodes, 100% dense on `o`). Promoted embedded fields are + spelled out (§4.5), so `o.builderState.left.groupsIdx` becomes + `o.mergeJoinBase.builderState.left.groupsIdx`, which k=3 truncates to + `o.mergeJoinBase.builderState.*`. With `--graph-field-depth 4`, total `ddg` falls 7.5% and the + trailing-`.*` class falls from 635,616 to 10,769. +- **Type-error defs.** The pre-fix binary dropped L3 defs whose receiver failed to type-check. I2 + fixes that, so a HEAD rerun would have more `ddg` edges than shown; these figures are a lower + bound even for this degraded tree. + +## Limitations and known gaps + +### Degraded inputs + +- **cockroach** is the only degraded tree, and is reported only in its own subsection above. +- **dae** is Linux-only. It is run with `GOOS=linux` (from the manifest), where it type-checks + fully. On darwin it would have 90 type errors in 4 packages. + +### Model limits (by design at L3) + +- **Syntactic, alias-free L3.** Two paths with different bases never interact: `p := &x; p.f = 1; + use(x.f)` has no edge. Points-to and alias flow arrive at L4 (§4.10) as additive + `prov:["points-to"]` edges. +- **Not refutation-complete** (spec §4.5 "L3 precision"): no edge does not mean no flow. + Writes to captured variables inside closures, value structs whose fields are slices, maps or + pointers, and `append` into a destination's backing array are not modelled. Deferred calls now + read their receiver and argument paths at exit (I7), keyed on the variable, not the value + bound at registration. +- **Every defer is assumed registered on every path,** with one chain member per `defer` + statement. `recover()` does not resume normal flow, and `os.Exit`-style calls are ordinary calls + (§4.3.1–§4.3.2). + +### Open follow-ups + +- **Sequential buffer writes.** Generated marshallers produce a def x use triangle on their output + buffer (§ Protobuf-generated DDG volume). Fine for Kubernetes-sized messages, quadratic-ish + (fields^1.8) for very wide ones. A kill rule for `dAtA[i] = …` after `i--`, or a summarization, + is not implemented. +- **Quadratic kill computation.** `BuildDDG` computes KILL by testing every strong def of every + node against every def in the body. A reviewer probe of one 6,000-case switch-heavy function + took 15 s. Nothing in this corpus comes close. + +## L4 (to be added) + +L4 (interprocedural SDG) results will be added here, over the same corpus and pinned commits, with +the same reproduction harness (`scripts/eval_l3.sh --prepare`, then run). Planned measurements: + +- **The points-to delta.** `ddg` edges with `prov:["points-to"]` added on top of the L3 `ssa` + edges, which must be strictly additive (§4.10): count, share, per-app, and which access-path + shapes the type-based oracle links. +- **Parameter wiring.** `@formal_in` / `@formal_out` vertices and the per-call-site actual + vertices (§4.6), and `GO_PARAM_IN` / `GO_PARAM_OUT` edges (§4.7). These are counted per callable + and per resolved call site, against L2's `call_graph` edges. +- **Summary edges** (§4.9): count, SCC sizes, fixpoint iterations on recursive SCCs, and the + cost of the bottom-up pass. +- **Library stand-ins** (§6): the number of external stand-ins with modelled bodies, and how many + `formal_in`/`formal_out` vertices they contribute. +- **VTA dispatch** (§4.8): call edges added over the `go/types` call graph. +- **Cost and size.** Wall time, RSS and `analysis.json` growth of `-a 4` over `-a 3`, plus the + `-j` determinism check, with Kubernetes as the scale benchmark. +- **Consistency at scale.** Every param edge joins a real call site to a real formal, and every + summary edge spans a call site's actual-in to actual-out. +- **Effect on the L3 caveats above.** Whether points-to edges make the weak-def tail denser, and + whether the protobuf-marshaller triangle compounds across `MarshalToSizedBuffer(dAtA[:i])` + calls once they are wired interprocedurally. diff --git a/docs/design/notes/l3-eval-2026-10.md b/docs/design/notes/l3-eval-2026-10.md new file mode 100644 index 0000000..4bddc41 --- /dev/null +++ b/docs/design/notes/l3-eval-2026-10.md @@ -0,0 +1,250 @@ +# L3 real-app evaluation (2026-10-24) + +Open-source Go apps at pinned commits, analyzed with `cango -a 3` via `scripts/eval_l3.sh`. +Host: Apple M3 Max, 16 cores, 64 GB, macOS. Analyzer built with go1.27.3 (see Finding 2). Every +run is `--eager` (no cache). Each app is first run once and discarded to warm the Go build/module +cache, then timed: `-a 2 -j 1`, `-a 3 -j 1`, `-a 3 -j 16`, and a second `-a 3 -j 16`. Wall time +and peak RSS come from `/usr/bin/time -l`. Counts are from the `-a 3 -j 1` output. "In-project" +and "external" split `call_graph` edges by whether `dst` is an `@external` id. + +L3 volume, edge/node-kind breakdowns, cost, output size, the protobuf-marshaller measurement and +consistency checks over these outputs: [`l3-dataflow-metrics.md`](l3-dataflow-metrics.md). + +## Corpus change: Kubernetes replaces cockroach as the scale benchmark + +- **Why.** cockroach is now under the proprietary CockroachDB Software License, so it cannot be a + published benchmark. Without Bazel it is also only partly type-checked (62,908 type-error + warnings; § Degraded measurements). Kubernetes is Apache-2.0 and commits its generated code + (`staging/src/k8s.io/api/**/generated.pb.go`), so a plain checkout type-checks fully. +- **What runs.** Kubernetes v1.37.1 (`f78e722310e50bcaca9276be22276d9e91d91308`) as two modules: + `kubernetes`, the root module `k8s.io/kubernetes` (the scale benchmark), and `k8s-api`, + `staging/src/k8s.io/api` (almost all generated gogoproto marshallers: the measurement of DDG + volume on generated protobuf, review finding I5). +- **cockroach** stays in the manifest as "very large, partly type-checked without Bazel; not a + published benchmark". Its figures below are the first evaluation's (pre-fix binary) and are not + rerun. + +## Results after the final-review fixes (HEAD) + +One `scripts/eval_l3.sh` pass over the six apps and both Kubernetes modules, with the binary the +script builds from `feat/l3-dataflow` at `9aeda43` (`runtime.Version()` go1.27.3; analyzer code +identical to `5508010`, so it includes C1, I2, R15 and I7). The type-check column is the +`--prepare` status (`go list -e -export ./...` with the app's env): `complete` means no package +has an `Error` or `DepsErrors`. No run printed a type-error warning, the too-old-toolchain +warning, or an `L3 skipped` line. + +| app | status | type-check | wall -a 2 -j 1 (s) | wall -a 3 -j 1 (s) | wall -a 3 -j 16 (s) | RSS -a 3 -j 1 (MB) | -j identical | L3 skips | +|---|---|---|---|---|---|---|---|---| +| cobra `adbc881` | ok | complete | 0.47 | 0.42 | 0.41 | 263 | yes | 0 | +| listmonk `8216c4f` | ok | complete | 0.80 | 0.97 | 0.86 | 936 | yes | 0 | +| memos `28719a9` | ok | complete | 2.26 | 2.52 | 2.32 | 2,613 | yes | 0 | +| nats-server `f8f6db8` | ok | complete | 1.44 | 2.24 | 1.74 | 1,078 | yes | 0 | +| openbao `1d58653` | ok | complete | 4.13 | 4.98 | 4.39 | 5,082 | yes | 0 | +| dae `e3fee8f` (GOOS=linux, ebpf generated) | ok | complete | 1.43 | 1.75 | 1.63 | 1,534 | yes | 0 | +| **kubernetes** `f78e722` (root) | ok | complete (1,468 pkgs) | 9.62 | 12.43 | 11.76 | 7,135 | yes | 0 | +| **k8s-api** `f78e722` (`k8s.io/api`) | ok | complete (61 pkgs) | 3.13 | 5.06 | 3.97 | 2,198 | yes | 0 | + +| app | modules | callables | body nodes | cfg edges | cdg edges | ddg edges | external_symbols | call_graph in-project | call_graph external | analysis.json -a 3 (MB) | +|---|---|---|---|---|---|---|---|---|---|---| +| cobra | 18 | 320 | 3,671 | 4,166 | 2,239 | 6,319 | 80 | 505 | 298 | 2.0 | +| listmonk | 101 | 820 | 16,252 | 17,502 | 9,815 | 39,391 | 471 | 1,439 | 2,264 | 10.0 | +| memos | 320 | 6,076 | 68,823 | 75,289 | 39,551 | 84,100 | 736 | 4,248 | 7,761 | 39.8 | +| nats-server | 97 | 5,279 | 105,409 | 141,829 | 81,598 | 345,318 | 627 | 11,630 | 7,754 | 72.8 | +| openbao | 702 | 8,712 | 147,951 | 177,091 | 98,844 | 234,460 | 1,481 | 10,431 | 13,483 | 94.5 | +| dae (linux) | 226 | 2,891 | 43,399 | 52,476 | 29,609 | 68,070 | 777 | 3,230 | 4,755 | 25.3 | +| kubernetes | 3,414 | 44,514 | 492,627 | 515,474 | 280,473 | 837,658 | 1,550 | 33,157 | 25,578 | 317.6 | +| k8s-api | 494 | 15,427 | 356,174 | 407,985 | 264,677 | 626,883 | 97 | 16,500 | 7,627 | 183.4 | + +- **Kubernetes is easier than expected.** The root module runs `-a 3 -j 1` in 12.4 s at 7.1 GB, + against cockroach's 33 s at 13.5 GB, with full type-checking. L3 adds 2.8 s to 9.6 s of + `-a 2`. The `-a 3` file is 318 MB. +- **k8s-api answers I5.** 626,883 `ddg` edges, 93% in `generated.pb.go`; max per callable 14,796 + (`core/v1.PodSpec.MarshalToSizedBuffer`); 183 MB of JSON, 3.3x the `-a 2` file. The detail and + the comparison with the 600-field probe are in `l3-dataflow-metrics.md` § Protobuf-generated + DDG volume. +- **What changed against the first evaluation** (the tables under § First evaluation): + - `body`, `cfg` and `cdg` are identical on every app, edge for edge. `ddg` only grows, by + 0.3–1.9% on the six apps (I7: a deferred call uses `recv.*` / `a.*`; C1: subslice arguments + weak-define `base[*]`) and by 14.6% on `k8s-api` (C1 on `MarshalToSizedBuffer(dAtA[:i])` + and `copy(dAtA[i:], …)`). No edge was removed. + - `-a 2` files are 28% larger (R15: call body nodes carry `id` from L1); `-a 3` files barely + change (+0.3%), since L3 already carried those ids. + - `call_graph` is unchanged for the five fully-built apps, and dae has the post-`679f995` + 7,985 edges. + - Wall times and RSS move within run-to-run noise (about ±0.2 s on the mid-size apps). + - dae now passes the byte-identical `-j` check (Finding 1, fixed in `679f995`). + +## First evaluation (`fd24bdf`, pre-fix) + +The tables in this section are the first pass, with the `fd24bdf` binary, before the `.init` +fix (Finding 1) and the final-review fixes. They are kept as recorded; cockroach's row is the only +one not superseded above. + +| app | status | wall -a 2 -j 1 (s) | wall -a 3 -j 1 (s) | wall -a 3 -j 16 (s) | RSS -a 3 -j 1 (MB) | -j identical | degraded | L3 skips | +|---|---|---|---|---|---|---|---|---| +| cobra `adbc881` | ok | 0.30 | 0.32 | 0.31 | 262 | yes | no | 0 | +| listmonk `8216c4f` | ok | 0.69 | 0.83 | 0.74 | 890 | yes | no | 0 | +| memos `28719a9` | ok | 1.96 | 2.34 | 2.13 | 2,380 | yes | no | 0 | +| nats-server `f8f6db8` | ok | 1.47 | 2.19 | 1.77 | 1,069 | yes | no | 0 | +| openbao `1d58653` | ok | 3.96 | 4.77 | 4.20 | 4,547 | yes | no | 0 | +| dae `e3fee8f` (GOOS=linux, ebpf generated) | **FAIL: -j differs** | 1.39 | 1.74 | 1.44 | 1,423 | **NO** | no (0 type errors) | 0 | +| dae `e3fee8f` (GOOS=darwin, ebpf generated) | **FAIL: -j differs** | 1.34 | 1.73 | 1.58 | 1,311 | **NO** | yes (90 type errors) | 0 | +| cockroach `d30c905f` | **FAIL: -j differs** | 23.08 | 33.36 | 26.96 | 13,473 | **NO** | yes (62,908 type errors) | 0 | + +No app crashed and no `L3 skipped` line (a caught per-callable panic) appeared anywhere. All +outputs are valid; "FAIL" means only the byte-identical `-j` check failed (see Finding 1, since +fixed; the dae and cockroach reruns after the fix pass). + +| app | modules | callables | body nodes | cfg edges | cdg edges | ddg edges | external_symbols | call_graph in-project | call_graph external | analysis.json (MB) | +|---|---|---|---|---|---|---|---|---|---|---| +| cobra | 18 | 320 | 3,671 | 4,166 | 2,239 | 6,261 | 80 | 505 | 298 | 2 | +| listmonk | 101 | 820 | 16,252 | 17,502 | 9,815 | 39,287 | 471 | 1,439 | 2,264 | 10 | +| memos | 320 | 6,076 | 68,823 | 75,289 | 39,551 | 83,413 | 736 | 4,248 | 7,761 | 40 | +| nats-server | 97 | 5,279 | 105,409 | 141,829 | 81,598 | 340,338 | 627 | 11,630 | 7,754 | 72 | +| openbao | 702 | 8,712 | 147,951 | 177,091 | 98,844 | 230,816 | 1,481 | 10,431 | 13,483 | 94 | +| dae (linux) | 226 | 2,891 | 43,399 | 52,476 | 29,609 | 66,825 | 777 | 3,230 | 4,745 | 26 | +| dae (darwin) | 223 | 2,813 | 41,732 | 50,605 | 28,564 | 64,746 | 727 | 3,139 | 4,541 | 24 | +| cockroach | 5,953 | 88,765 | 1,407,838 | 1,618,928 | 959,530 | 4,600,814 | 3,862 | 156,068 | 55,624 | 1,112 | + +### Observations (first evaluation) + +- **Scale:** L3 adds little wall time over L2: +0.0 to +0.8 s on the six smaller apps and + +10 s on cockroach (23 s to 33 s). Throughput is dominated by loading and type-checking. + `-j 16` buys up to about 20 percent on `-a 3` (nothing on cobra), not more, because L3 is a small part of total time. +- **Memory:** peak RSS is about 13 GB on cockroach (1.1 GB of JSON) and 4.5 GB on openbao. + It is dominated by `go/packages` type information, not by L3. +- **-a 3 over -a 2:** the delta is single-digit seconds even at 1.4M body nodes. No app came + near the 60-minute stop condition; cockroach took 33 s per run. The Kubernetes root, at HEAD, + took 12.4 s. + +## Findings + +### 1. Non-deterministic `call_graph` for repeated `.init` (fixed in `679f995`) + +dae and cockroach failed the byte-identical `-j` check. Fixed in `679f995` (Task 12b); the +rerun results are below. + +- Not a `-j` problem: three `-j 1` runs of dae also differ from each other, and so do three + `-a 2` runs. It is run-to-run nondeterminism. +- Scope: everything except `call_graph` is byte-identical across all three runs (checked by + hashing the JSON with `.application.call_graph` deleted). The `call_graph` edge counts are + equal; the differing edges are all `src = ....init` edges (dae 4 differing lines, + cockroach 118 differing lines out of 211,692 edges), where the same `(init, callee)` edge is attributed to a + different file's `init`. +- Cause (confirmed): `internal/semantic_analysis/call_graph.go` `Build` + iterates the `symbolTable` map, and dedups edges with `seen[[2]string{callable.Signature, + calleeSig}]`. `.init` is non-unique (one per file; Go allows several `init` per + package), so the first file visited in random map order wins and the edge's `SourcePath` + varies. +- The dedup also dropped edges, not only moved them: when two files' inits called the same + callee, only the first-visited init kept the edge, so the edge counts were equal across runs + but short. +- Fix (`679f995`): `seen` is keyed on `(callable.Path, callable.Signature, calleeSig)`, so every + init keeps its own edges, and `Build` walks files, types and callables in sorted key order. + Regression tests: `internal/syntactic_analysis/v2emit/gate_l2_inits_test.go` over the + `testdata/dataflow/inits` fixture (two files, each `init()` calling `setup()` and + `os.Getenv`), asserting each init's edges and 10 byte-identical fresh builds. +- Rerun after the fix (same pinned commits, `--eager`, `-j 16`), with `-a 3 -j 1`, `-a 3 -j 16` + and a second `-a 3 -j 16`: + + | app | -j identical | call_graph edges before | after | in-project / external after | + |---|---|---|---|---| + | dae (GOOS=linux) | yes | 7,975 | 7,985 | 3,230 / 4,755 | + | cockroach | yes | 211,692 | 212,605 | 156,876 / 55,729 | + + Against the pre-fix output (dae: a fresh `GOOS=linux` run of `fd24bdf`), no edge is removed, + and every added edge (dae 10, cockroach 913) has an `init` source: the edges the old dedup + dropped. Everything outside `call_graph` is byte-identical. +- Latent elsewhere: listmonk (25), memos (36), nats-server (8) and openbao (49) have `init` + edges too and happened to be stable in three runs, so the unit and gate tests would not + catch this on small fixtures. + +### 2. Analyzer must be built with a Go toolchain at least as new as the target's `go` line + +Built with go1.26.4, the analyzer type-checked listmonk, memos and openbao (`go 1.27.x`) +with one "package requires newer Go version go1.27 (application built with go1.26)" warning +per package, producing a degraded result with no error exit. The eval binary is built with +`GOTOOLCHAIN=go1.27.3 go build -o /tmp/cango-eval ./cmd/codeanalyzer`. Release packaging +should build with the newest Go toolchain, and the warning is worth surfacing in the docs. + +Since addressed (`123fe18`): releases build with the latest Go, and the analyzer prints one +consolidated warning naming its build Go and the highest version required. `eval_l3.sh` now builds +its binary with `EVAL_GO` (default go1.27.3) and counts that warning ("toolchain warn" column); it +fired on no HEAD run. + +### 3. Neo4j `--eager` push failed on Neo4j 5.x (fixed in `6f250ea`) + +Found by the Task 12 live run against Neo4j 5.26. The `--eager` purge +(`internal/neo4j/bolt.go`, `eagerPurgeQuery`) uses `CALL (n) { DETACH DELETE n } IN +TRANSACTIONS OF 5000 ROWS` but ran inside `session.ExecuteWrite`, an explicit transaction. +Neo4j rejects that (`'CALL { ... } IN TRANSACTIONS' can only be executed in an implicit +transaction`), so every `--eager` push onto a database that already held the app failed. Both +`TestIntegration_*` tests (build tag `neo4j_it`) failed on it. Fix (`6f250ea`): the purge runs +as an auto-commit transaction (`session.Run` plus `Consume`). It is the only `IN TRANSACTIONS` +statement; every other write still goes through `ExecuteWrite`. Verified live on Neo4j 5.26: +the full `neo4j_it` suite passes, and two CLI `--eager` pushes of `testdata/dataflow` give the +same `GO_CFG_NEXT` / `GO_CDG` / `GO_DDG` counts, 364 / 224 / 201, equal to the JSON (352 / 219 / +201 before the `inits` fixture was added to it), and a planted stale node under the app prefix is +purged. + +## Degraded measurements + +### dae: eBPF generated; darwin type-checks partially + +- `make ebpf` needed the `dae_bpf_headers` submodule (`git submodule update --init --depth 1`), + then succeeded with Homebrew `llvm@18` on PATH (clang, llvm-strip). The `bpf2go` outputs + (`control/bpf_bpfel.go`, `trace/bpf_arm64_bpfel.go`) are present. +- dae is Linux-only. Run on darwin (the default `GOOS`), four packages (`cmd`, `component`, + `component/outbound/dialer`, `control`) fail to type-check with 90 warnings, e.g. + `undefined: unix.RTN_PROHIBIT`, `undefined: netlink.FAMILY_ALL`. That row is **degraded**. +- Run with `GOOS=linux`, dae type-checks fully (0 type errors, 3 more modules, 78 more + callables). The linux row is the representative one, and the manifest now sets `GOOS=linux` + for dae, so the HEAD pass runs only that. At prepare time, `go list -e -export` with darwin + shows 2 packages failing to compile and 3 more broken through them; with linux, 0 of 34. + +### cockroach: no protobuf generation; heavily degraded (first evaluation only) + +- `scripts/eval_l3.sh --prepare cockroach` now reports this itself: `missing tool: bazel or + bazelisk`, prep skipped, and `degraded: missing bazel or bazelisk; 973 packages with errors` + (22 that fail to compile, 951 more through dependencies, of 1,380). +- Bazel is not installed (`bazel`, `bazelisk` absent) and was not installed, so + `./dev generate go` was not run. A plain checkout has 183 `.proto` and 0 `.pb.go`. +- Result: 62,908 type-error warnings across 633 packages (e.g. `undefined: + tracingpb.StructuredRecord`, `undefined: build.Info`), plus cgo failures (`jemalloc/jemalloc.h`, + `proj_api.h` not found). The analyzer still completed with all 5,953 modules and 88,765 + callables, but references into generated types are unresolved, so the call graph + (156,068 in-project edges) is a lower bound. Counts and timings are real but not + comparable to a fully generated tree. + +### memos: no generation needed + +`go build ./...` succeeds on the plain checkout; the single `go:generate` directive +(`markdown/parser/tag.go`, unicode tables) already has its output committed. `go generate` +was **not** run. 0 type errors. + +## Reproducing + +``` +scripts/eval_l3.sh --prepare # all apps; or name some: --prepare kubernetes k8s-api +scripts/eval_l3.sh cobra listmonk memos nats-server openbao dae kubernetes k8s-api +``` + +- **`--prepare [APP…]`** reads the manifest in `scripts/eval_l3.sh` (name, repo, full pinned SHA, + module subdir, env, prep, tools, role). For each app it fetches the pinned commit into + `/tmp/eval/` (`git init`, `git fetch --depth 1 --filter=blob:limit=2m origin `, + `git checkout FETCH_HEAD`), or reuses a clone already at that commit and refuses one at any + other commit. It checks the prep tools, runs the prep (dae: `git submodule update --init --depth 1` + and `make ebpf`, with Homebrew `llvm@18` on PATH), type-checks with `go list -e -export ./...` + under `GOTOOLCHAIN=go1.27.3`, and writes `/tmp/eval-out/prepared/.env`. A missing tool is + named with how to get it; nothing is installed. +- **Prepare status on this host:** `complete` for cobra, listmonk, memos, nats-server, openbao, + dae, kubernetes and k8s-api; cockroach `degraded: missing bazel or bazelisk; 973 packages with + errors`. +- **Run.** Without `CANGO`, the run phase builds `/tmp/eval-out/cango` with + `GOTOOLCHAIN=$EVAL_GO go build` (`EVAL_GO` defaults to go1.27.3, at least the newest `go` line in + the corpus). Each app runs with its manifest env (`GOOS=linux` for dae). `--all` runs every + prepared app; plain directories still work, as before; `--clean` removes run outputs and keeps + the clones; `EVAL_PRUNE=1` deletes the duplicate `-j N` outputs after the comparison. +- Raw outputs and stderr logs are in `/tmp/eval-out`. The "type errors" column in the script's own + note counts `package ...` warnings on stderr, and "toolchain warn" counts the analyzer's + too-old-Go warning. diff --git a/docs/design/roadmap.md b/docs/design/roadmap.md index 2f306b3..aa00b94 100644 --- a/docs/design/roadmap.md +++ b/docs/design/roadmap.md @@ -10,7 +10,7 @@ A schema **major**: move the Go analyzer off the v1 keystone booleans, `signature` as id) onto the v2 canonical schema (additive tree, `can://` ids, `span` with byte offsets, `source` per module, typed edge lists). The SDK moves in lockstep as a coordinated major release. **Reach for this train: L1 + L2 to parity.** -L3/L4 deferred (see Not now). +L3 shipped in release A (below); L4 remains deferred (see Not now). Golden rule carried from `schema-migration.md`: keep everything that *computes* facts (parser, resolver, call-graph builder); replace only what *serializes* them. @@ -51,7 +51,7 @@ C (edge shape) ─┴─► L2 emission (call_graph rename @ application sc │ A/B/C ─► D (Neo4j relabel) ─► SDK v2 model remap ─► pin SDK→analyzer (once BOTH cut) │ - (later train) L3 ─► L4 ──────┘ reuse A & C unchanged + L3 (release A) ─► L4 (deferred) ┘ reuse A & C unchanged ``` ## Release trains @@ -62,14 +62,22 @@ A/B/C ─► D (Neo4j relabel) ─► SDK v2 model remap ─► pin SDK→analyz analyzer version in the SDK *only once both are released* — until then the SDK's old models won't parse v2 output. Superset gate: v2 output must contain every fact v1 did, modulo the sanctioned drops (`code`, `is_*` booleans folded into `kind`). -- **Train 2+ (deferred).** L3 (CFG/CDG/DDG, AST-only, per-callable parallel — net-new for - Go) then L4 (SDG `param_in`/`param_out`/`summary`, needs a points-to oracle). Both reuse - group A & C vocabulary; no new schema-major coordination beyond what Train 1 establishes. +- **Release A — v2 with L3 (`-a 3`).** L3 is no longer deferred. Ships the new `can://` id + grammar, L2 library calls (`external_symbols`, `@external` call-graph endpoints), and + per-callable L3 CFG/CDG/DDG (AST-only, intraprocedural, `--graph-field-depth` k-limit), + with the Neo4j `GO_CFG_NEXT`/`GO_CDG`/`GO_DDG` projection. + Real-app evaluation: `docs/design/notes/l3-eval-2026-10.md`. It and the live Neo4j run + found two issues, both fixed before release: L2 call-graph dedup on repeated `.init` + signatures (nondeterministic, dropped edges) and the Neo4j `--eager` purge failing on + Neo4j 5.x. The release tag is the user's decision. +- **Later (deferred).** L4 (SDG `param_in`/`param_out`/`summary`, needs a points-to oracle). + Reuses group A & C vocabulary; no new schema-major coordination beyond what Train 1 + establishes. ## Not now (considered, deliberately excluded) -- **L3 / L4 dataflow** — deferred by decision this session; net-new heavy construction, not - required for v2 parity. Revisit as Train 2. +- **L4 dataflow (SDG)** — deferred; needs a points-to oracle and interprocedural + construction. (L3 is in release A.) - **Points-to oracle** — L4 prerequisite; parked with L4. - **`--materialize-expressions` / `--materialize-basic-blocks`** — optional body node kinds; out of scope for the parity migration. diff --git a/docs/design/specs/v2-l3-l4-dataflow.md b/docs/design/specs/v2-l3-l4-dataflow.md new file mode 100644 index 0000000..a36eebc --- /dev/null +++ b/docs/design/specs/v2-l3-l4-dataflow.md @@ -0,0 +1,771 @@ +# Spec: Go L3/L4 dataflow (CFG/CDG/DDG + SDG) with SDK taint (codeanalyzer-go) + +**Status:** design, awaiting approval · **Date:** 2026-10-23 · **Repos:** `codeanalyzer-go`, +`python-sdk` · **Supersedes:** the "L3/L4 deferred" entries in `docs/design/roadmap.md`. + +## 1. Intent + +`python-sdk` ships one shared, language-agnostic `taint()` / slicing / reachability surface that +already works on Python, Java and TypeScript analyses. It walks five edge families — +`DDG`, `CDG`, `PARAM_IN`, `PARAM_OUT`, `SUMMARY` (`cldk/analysis/commons/graphs.py:sdg_rels`) — +and nothing else. Go stops at L2, so none of it works on Go code. + +This effort brings `codeanalyzer-go` to **L3 (intraprocedural CFG/CDG/DDG) and L4 (SDG)** in parity +with the sibling analyzers, shaped specifically so the SDK's existing `taint()`, `slice_backward`/ +`slice_forward`, `paths_between`, `reaches`, `flows_to_call`, `flows_to_argument` answer correctly +on a Go application through **both** the JSON (local) and Neo4j backends — and builds the Go side of +the SDK that exposes them. + +**Why now:** there is a concrete downstream need to run taint analysis on Go code; parity with the +siblings is required too; and the id grammar has to change for polyglot apps anyway (§4.1). + +### Goals + +1. `can:///go/…` id grammar (polyglot parity), shipped first as its own breaking release. +2. L3: per-callable `body{}` completed with statement nodes; `cfg[]`, `cdg[]`, `ddg[]` per callable. +3. L4: synthetic `formal_in/out`, `actual_in/out` vertices; `param_in[]`/`param_out[]` at + application scope; `summary[]` per callable; alias-aware `ddg` delta (`prov:["points-to"]`). +4. Taint-relevant Go flows are **not missed** (user-ranked): + - **Must:** (1) through function calls; (3) through interface (dynamic) dispatch. + - **Should:** (2) through struct fields and pointers (k-limited access paths); closure captures; + goroutine launches; channels (coarse); library calls (pass-through). +5. Sinks and sources can name **library functions directly**, e.g. `("query", "database/sql.DB.Query")`, + with the shared SDK taint code unchanged (§6). +6. Neo4j projection fills the already-reserved `GO_CFG_NEXT/GO_CDG/GO_DDG/GO_PARAM_IN/GO_PARAM_OUT/ + GO_SUMMARY` overlay. +7. python-sdk: a v2 Go model, a `GoAnalysis` facade, a local `GoCodeanalyzer` backend and a + `GoNeo4jBackend`, answering the same L1/L2/L3/L4 surface the Python/Java/TS facades do. + +### Non-goals + +- No taint engine, sources/sinks catalogue, or `taint_flows` section in the analyzer — it is a graph + provider; the SDK owns queries (keystone provider/client boundary). +- No goroutine interleaving / happens-before, no `sync` modelling, no `select` fairness. +- No analysis of the standard library's or dependencies' bodies (stand-ins only, §6). +- No `cgo`, `unsafe`, `reflect`-driven flows beyond what pass-through over-approximation gives. +- No context-sensitive summaries, no hammock-region decomposition (neither sibling built it). +- No change to the shared `commons/` taint/slice code in python-sdk (that is a later, + cross-language revisit — §11). + +### Anti-goals (would be failure even if it "works") + +- **A false refutation.** Any missing edge that lets `taint()` put a real flow in `exhausted` + ("searched to exhaustion, no flow") closes a live alert. Every precision choice here leans to + **over-approximation** (extra edges ⇒ false positives are acceptable; dropped edges are not). +- Breaking the level-monotonicity superset `json(-a1) ⊆ … ⊆ json(-a4)`. +- Non-deterministic output (`-j 1` vs `-j N` must be byte-identical; two runs byte-identical). +- Renaming or repurposing any shared keystone name (parity clause). + +## 2. Constraints and sources of truth + +- **Keystone:** `cldk-devtools/skills/designing-cldk-changes/references/canonical-schema.md` (shapes, + levels, edge families, parity clause). Construction guides: + `codeanalyzer-backend/references/level-3-intraprocedural-dataflow.md`, + `level-4-interprocedural-sdg.md`, `testing-and-validation.md`, `neo4j-projection.md`, + `cli-contract.md`. Where the keystone and the **shipped siblings** disagree, this design follows + the shipped siblings (Python 1.6.x, Java 3.3.x) because the SDK is written against them; each such + case is called out. +- **Golden rule** (from the v2 migration): keep everything that computes facts; add L3/L4 as new + computation; the v2 emitter stays the single serialization point. +- **Substrate:** `golang.org/x/tools v0.46.0` is already a dependency — `go/cfg`, `go/ssa`, + `go/ssa/ssautil`, `go/callgraph/vta`, `go/callgraph/cha` are available with no new module. + `x/tools/go/pointer` is deprecated: **do not adopt**. +- Level flag semantics (`cli-contract.md`): `-a 3` implies `-a 2`; `-a 4` implies `-a 3`; an `-a 3` + run must not pay L4 cost; `--emit neo4j` runs at max implemented depth (now 4) and rejects `-a` and + `--graph-field-depth`. +- python-sdk rules: the Iron Rule (no public accessor changes name/signature/return type); new + capability only as new methods; models are `extra="forbid"` mirrors of v2. + +## 3. Current state (what exists) + +- `codeanalyzer-go` L1/L2 on v2, Neo4j projection live. Analysis loads packages once via + `go/packages` (`NeedSyntax|NeedTypes|NeedTypesInfo|NeedDeps`); the internal v1-shaped model + (`internal/schema/schema.go`) is re-serialized by `internal/syntactic_analysis/v2emit`. +- No per-statement facts survive into the internal model; L3 must work from `pkg.Syntax` + + `pkg.TypesInfo` directly. +- `internal/schema/v2/ids.go` already defines `OrdinalID` (`@:`) and `TagID` + (`@`). `internal/neo4j/schema.go` already reserves the L3/L4 relationships. +- Known defects to fix on the way: `GO_DDG` discriminant is `var` only (must be `(var, prov)` per + CLAUDE.md K1 and siblings); `typeKind` never emits `alias`/`defined`; `is_deferred` is declared but + never populated; L2 call-site resolution re-walks the whole package AST per site (parked memory + `parked-callsite-resolution-perf`) — must be fixed before L3/L4 multiply call-site lookups. +- L2 dispatch: an interface-method call's callee is the interface method (CHA-style); no concrete + targets. This is insufficient for must-have flow (3). +- python-sdk `main` (2.0.0, c125d48) has **no Go support at all** — verified 2026-10-23: no tracked + Go files, `CLDK` has only `java`/`python`/`typescript` factories, the legacy `analysis()` shim + dispatches only those three. Branch `add-go-support` (July 2026, never merged, 332 commits behind) + is an old **v1-shaped** implementation and is obsolete. Since then PR #434 (unmerged) adds v2 + L1/L2 Go support; the SDK work builds on it (§7). + +## 4. Decisions + +Decisions marked **[user]** were made with the user; **[call]** are the designer's calls, made on +sibling precedent and taint needs, open to override on review. + +### 4.1 Identity: `can:///go/…` [user] + +Switch from `can://go//…` to the app-outermost grammar Python (`codeanalyzer/schema/ids.py`) +and Java (`SCHEMA_DECISIONS.md` D8) ship: + +``` +application can:// +module can:///go/ +type can:///go// +callable can:///go/[/]/ +body node @: | @ +external can:///@external// (language-neutral, as Python/TS) +stand-in see §6 +``` + +Reason: polyglot applications — one `` prefix scopes every language's nodes, the SDK's +`under_callable` / Neo4j prune prefixes assume app-outermost, and `@external` homes merge across +languages. Ships as its own release **before** L3 (Phase 0), so L3/L4 are built on final ids. +Breaking change; acceptable now because no released SDK consumes Go output. Update CLAUDE.md +§ Identity, `v2-schema-structure.md`, Neo4j prune/scope Cypher (`cypher.go`, `bolt.go`), and the +`:GoApplication` id. Mirror Python's application-root handling (`id = can://` matched +separately from the `can:///` descendant prefix). + +### 4.2 Body nodes (L3) [call] + +Body node ids are positional `line:col` local keys, global id `@` +(`global_ordinal` rule: synthetic `@…` keys concatenate as-is). Every body node carries its global +`id` field (Python #176 precedent, which the SDK reads) **at every level, from `-a 1` on** — the +L1 call nodes included. Python parity: codeanalyzer-python runs `stamp_body_ids` at L1, so a +consumer reads `id` off an L1 document the same way in both languages. (Supersedes the earlier +"id at L3+ only" ruling; additive at L1/L2.) `parent` stays an L3 refinement. + +**Kinds — shared vocabulary only, no Go-specific kinds:** + +| Go construct | `kind` | +|---|---| +| `if` (incl. init stmt folded in) | `branch` | +| `for` (3-clause, cond-only, infinite), `for … range` | `loop` | +| `switch`, type `switch`, `select` | `switch` | +| `return` | `return` | +| everything else (assign, `:=`, `var` decl, inc/dec, send `ch <- v`, expression stmt, `go`, `defer`, `break`/`continue`/`goto`/`fallthrough`, labeled stmt's inner stmt, `panic(...)` call stmt) | `statement` | +| each `case`/`default` clause head of switch/type switch/select | `statement` (the clause head — anchors its `switch_case` edge and holds the case expression / comm-clause defs/uses) | +| synthetic | `entry`, `exit` | +| existing L1 call | `call` | + +(The `defer` statement is two nodes — registration `statement` + deferred `call`; §4.3.1.) + +Rationale for not adding `select`/`go`/`defer` kinds: the keystone lets a language add kinds, but the +SDK's `body_node_kind` and Neo4j queries treat unknown kinds as opaque; `is_goroutine`/`is_deferred` +on the call node already carry those facts. Fewer new names = less parity debt. + +**Call nodes stay satellites** (Python #115, Java `nodeIdFor`): a call nested inside a statement keeps +its own L1 `line:col` key, is not a CFG step, and gains `parent` = the enclosing statement's local +id at L3 (sanctioned `parent` refinement). A bare call statement (`foo(x)` as an `ExprStmt`) **shares +its key** with the statement node: the CFG node *is* the call node (kind stays `call`). Same for +`go f(x)`: the statement node is the call node, keyed at the call's position, carrying +`is_goroutine`. `defer f(x)` is the exception — two nodes (§4.3.1). Co-positioned calls (`f(g(x))` where both start at the same column, or +`a.b().c()`) disambiguate with `/2`, `/3` suffixes, outermost first, exactly as Python +`call_body_keys` (#215). Fix this in L1 too (today a collision silently drops a call). + +Statement span = the statement's `ast.Node` span; ids anchor at `stmt.Pos()`. + +### 4.3 CFG construction (L3) [call] + +**Substrate: hand-built from `go/ast`** (statement-granular, positions are native), with a +**differential test against `x/tools/go/cfg`** for block reachability/successor structure (the +construction guide's "stdlib as answer key"). Rejected: building L3 from `go/ssa` — SSA instructions +lose statement granularity and need a position back-map (Java's WALA pain, `line:0` sentinels); +rejected `go/cfg` as the sole source because it does not model panics/defers/exits as edges we need. + +Edge `kind`s — shared vocabulary, **plus one Go add**: + +| Construct | Edges | +|---|---| +| straight-line | `fallthrough` | +| `if` | `true` → then, `false` → else/next | +| `for cond` / 3-clause | `true` into body, `false` out, `loop_back` tail → header; `post` stmt folded into header node | +| `for {}` (no cond) | body entry is `fallthrough`; **no** `false` edge; synthetic `exception` edge header → `@exit` to restore post-dominance (Python precedent, `cfg.py ~520`) | +| `for range` | header `true` into body, `false` out, `loop_back` | +| `switch` / type switch | `switch_case` header → each clause head; to next-stmt if no `default`; clause head → first body stmt `fallthrough`; Go `fallthrough` stmt → next clause **body** with kind `fallthrough` | +| `select` | `switch_case` header → each comm clause; no "no default" edge (select blocks); empty `select {}` → `exception` → `@exit` | +| `break`/`continue` (labeled or not) | `break` / `continue` to resolved target | +| `goto L` | `fallthrough` to labeled stmt (no new kind; rare) | +| `return` | `return` → `@exit` (via defer chain, below) | +| falling off end | `return` → `@exit` | +| panic-capable stmt | `exception` → `@exit` (via defer chain) | +| deferred-call resume | **`defer_resume`** (Go add, §4.3.1) | + +**4.3.1 `defer` — two nodes, no cycle [call, from builder check].** A `defer f(x)` is split into +two body nodes with distinct keys: + +- **Registration node:** the `defer` statement itself, `kind:"statement"`, keyed at the `defer` + keyword (`stmt.Pos()`), on the straight-line path. It **uses** the call's receiver and arguments + (evaluated here, per Go semantics) and weak-defines a pseudo access path `::` + (the call's key), carrying those evaluation-time values to the exit. +- **Deferred call node:** the L1 `call` node (keyed at the call expression's position — a different + column from the `defer` keyword, so no collision), `is_deferred:true`, placed **only** in the exit + chain. It uses `::` (argument values from registration) plus, at the exit + point, named results and (for `defer func(){…}()`) everything the closure captures (§4.6 + closures rule). + +Every edge that reaches `@exit` (`return`, fall-off, `exception`) is routed through the **defer +chain** — deferred call nodes in reverse source order — then to `@exit`. The edge from an exit point +into the chain keeps its original kind (`return`/`exception`); chain-internal edges and the tail → +`@exit` are `defer_resume` (Go add, pre-registered in CLAUDE.md and the keystone rubric). A +deferred call that can panic gets an `exception` edge to the next chain member (remaining defers +still run). Over-approximation: every defer is assumed registered on every path (conditional and +in-loop defers included; one chain member per `defer` statement, not per execution). The CFG +stays acyclic through the chain, so post-dominance and the "reaches `@exit`" gate hold. +`go f(x)` is **not** split: the `go` statement shares its key with the call node (the call is the +step; arguments are evaluated there; the spawned body is not a CFG successor). + +**4.3.2 panic / recover.** A statement *can panic* if it contains a call (any call — callee bodies +can panic), an index/slice expression, a type assertion without `ok`, a pointer dereference/selector +on a pointer, a channel send/close, a division, or a `panic(...)`. Go has no handlers, so `exception` +edges go to the defer chain then `@exit`. A `panic(...)` statement has **only** the `exception` +edge (no fall-through). `recover()` inside a deferred closure is not modelled as resuming normal +flow (over-approximate: the function still "exits"; data defined in the defer still reaches +`@formal_out` via the defer chain, which is what taint needs). `os.Exit`/`log.Fatal`-style no-return +calls are treated as ordinary calls (over-approximation; differs from `go/cfg`'s `mayReturn`, which +the differential test must account for). + +**4.3.3 goroutines.** `go f(x)` is a CFG step whose successor is the next statement; the spawned +function is not a CFG successor (keystone). Its data effect is the call (§4.6). + +**Gates:** single `@entry`/`@exit`; every node has a real span (synthetic excepted); every node +reachable from `@entry` and reaching `@exit`; edges sorted/deduped by `(src,dst,kind)`; node count +stable across runs; unreachable code after `return`/`panic`/`goto` pruned (Python precedent). + +### 4.4 CDG (L3) [call] + +Post-dominators via Cooper-Harvey-Kennedy on the reverse CFG rooted at `@exit`; control dependence +via Ferrante-Ottenstein-Warren. Nodes with no controller get an edge from `@entry` (Python +precedent — every statement anchored). Self-loops skipped. `cdg` edges `{src,dst}`, no attrs. +Gate: hand-computed control dependences on fixtures match exactly (incl. infinite loop, `select`, +defer chain). + +### 4.5 DDG (L3, syntactic) [call] + +Monotone forward may-reaching-definitions over **k-limited access paths** +`base(.field|[*])*`, `k` = `--graph-field-depth` (default 3; `x.f.g.h` → `x.f.g.*`; bases never +truncated). Match Java's `AccessPath` / Python's `access_paths` semantics: + +- **Bases:** locals, parameters, named results, the receiver, captured variables, package-level + variables (qualified `pkgpath.Name`, see §4.6). Shadowed variables are distinct bases — identity is + the `types.Object`, not the spelling (fixes the "shadowed variables leak" gate for free via + `TypesInfo.Uses/Defs`). The emitted `var` is the source spelling (`x.f`), as siblings. +- **Defs/uses per node:** from the node's own header expressions only (`if` cond + init, `for` + cond/post/init, `range` key/value defs + range-expr use, switch tag / type-switch guard, + case expressions, comm-clause send/recv, assignment LHS/RHS, inc/dec, `return` operands). + `@entry` defines every parameter, the receiver, named results (zero value), captures and every + package-level base the callable reads. +- **Writes:** plain `x = …` strong-kills `x` and its extensions (`x.f`, `x[*]`); `x.f = …` kills only + `x.f…` (no kill of `x`); `x[i] = …` is a weak def of `x[*]` (kills nothing); `*p = …` is a weak def + of `p.*` (deref collapses to `.*`, kills nothing). Compound assignment and `++`/`--` def+use. +- **Calls:** a call uses all its argument expressions and the receiver; it **weak-defines `a.*` for + every argument `a` that is pointer-, slice-, map-, chan-, func- or interface-typed, and the + receiver if it is a pointer receiver or reference-typed** (pointer, slice, map, chan, func or + interface — e.g. `hs.Write(b)` on an interface, `h.Set(k, v)` on a map-typed `http.Header`; the + callee may write through it; Python's suffixed-weak-def precedent). Value-typed struct arguments and + receivers are not weak-defined (copy semantics). A **slice expression** argument (`r.Read(buf[:n])`, + `io.ReadFull(r, hdr[:])`, and the destination of `copy(dst[i:], src)` / `clear(s[:n])`) shares its + elements with its base, so it weak-defines the base's elements **`base[*]`** — slice, array or + pointer-to-array base alike (`&`, `*` and parentheses around the base are stripped). `[*]` rather + than `.*` because it is the element step the builtin rule (`copy`/`clear` → `dst[*]`) and index + reads (`buf[0]` uses `buf[*]`) already use; the slice bounds are still uses. +- **Channels (goal 4, coarse):** `ch <- v` weak-defines `ch[*]` and uses `v`, `ch`; `v := <-ch` / + `v, ok := <-ch` / `range ch` / recv in a comm clause uses `ch[*]` and defines `v`. Cross-goroutine + flow then rides parameter/capture/field paths — a channel passed to a goroutine is an argument, so + its `[*]` contents flow via the port wiring (§4.7) and the "weak-def through reference arguments" + rule. +- **Maps/slices:** `m[k] = v` weak-defines `m[*]`; `m[k]` reads `m[*]`; `append(s, v)` uses `s`, `v` + and its result flows to the LHS; it also weak-defines `s[*]` (`append(dst[:0], v)` gives + `dst[*]`), since it may write into the destination's backing array (spare capacity, or a + re-sliced `dst[:0]`). +- **Type errors (partial types):** a package that does not type-check still gets its defs. A field + selector with no recorded selection (its operand has an invalid type, e.g. `var req missing.Req`) + is still a field step (`req.Body`); an identifier go/types recorded nothing for (undeclared, or a + member of a package that failed to import) is an unresolved base identified by its spelling, and a + write through it weak-defines its root `x.*`. An assignment target is never demoted to use-only + while it has a variable root: a target `pathOf` cannot walk (`x.(*T).f = v`) weak-defines its + root `x.*`. +- **Use matching:** a use links to a reaching def if the paths interfere — equal, either a prefix of + the other, or a wildcard covers it. +- **Edge attrs:** `{src, dst, var, prov:["ssa"]}`. `prov` value is `"ssa"` for **parity with Python + and Java**, whose L3 DDG is also syntactic reaching-defs labelled `ssa` (the SDK's prov + precision ranking and the keystone's monotonicity rule both key on it). Rejected: TS's + `reaching-defs` at L3 — the SDK ranks `reaching-defs` as a distinct, less-approximate token and + using it here would mislabel. Sorted by `(src,dst,var,prov)`. +- **Gates:** loop-carried `x = x + 1` self/back edge; shadowed variable does not leak; k-limit + contract example; backward intraprocedural slice over `cdg ∪ ddg` of a named var at a named line + equals the hand-computed set exactly. + +Differential check (non-gating, reported): compare scalar def-use against `go/ssa` (the +construction guide's answer key) on fixtures; mismatches are triaged, not asserted. + +#### L3 precision (known gaps) + +The L3 `ddg` is **syntactic and intraprocedural**. It over-approximates where it models a flow +(weak defs, bidirectional prefix matching), but it is **not refutation-complete**: the absence of +a `ddg` path does not prove the absence of a flow. It does not model: + +- **Writes to captured variables inside closures.** `func(){ x = secret }()` or + `once.Do(func(){ cfg = load() })` writes `x` / `cfg` in the closure's own body; the enclosing + callable sees only a use of the captures at the closure expression, never a def, so a later + read of `x` in the enclosing callable does not link to the closure's write. +- **Value-typed struct arguments or receivers that contain slice, map or pointer fields.** A + value struct is copied (no weak def, §4.5 Calls), but its reference-typed fields alias the + caller's: `fill(cfg)` with `cfg.items []T` may write `cfg.items[*]`, which L3 does not record. +- ~~Deferred calls reading receiver or argument state at exit~~ — **closed at L3, except for + reassignment**: the deferred call node also uses every path it may write through + (`defer w.Flush()` uses `w.*`, `defer fill(b[1:])` uses `b[*]`), so a write after the + registration reaches it. The paths are keyed on the *variable* at exit, not on the value + bound at registration, so a reassignment after the `defer` (`defer fill(b); …; b = o`) can + add a spurious edge, and can kill an earlier def on the normal path. + +The first two are addressed at **L4**: capture formals (`@formal_in`/`@formal_out` for captured +variables, §4.6 closures rule) carry closure writes back to the enclosing callable, and the +points-to delta (§4.10) links writes through reference fields of copied structs. + +### 4.6 L4 vertices [call; naming follows Python because the SDK reads it] + +Per callable (formals), all with `kind` + `of`, no span: + +| Vertex | `of` | When | +|---|---|---| +| `@formal_in:` | real parameter name; receiver name for the receiver | receiver first (index 0) if method, then declared params in order (unnamed `_`/blank params get `_`) | +| `@formal_in:` | `:` | each variable a closure captures (after declared params) | +| `@formal_in:` | `:::` | each package-level variable the callable (transitively, via summary) reads | +| `@formal_out` / `@formal_out:` | `` | **one per result position** — Go's multiple returns: `` for a single result; for >1 results, `@formal_out:` with `of` = the named result's name if named else `:` | +| `@formal_out:` | parameter name | each pointer-/reference-typed parameter (incl. pointer receiver) the callable may write through (by-ref out) | +| `@formal_out:` | `:::` | each package-level variable written | + +Per call site with a resolved callee (in-project **or stand-in**, §6): + +| Vertex | `of` | `parent` | +|---|---|---| +| `/actual_in:` | the bound callee formal's name (mirrors the callee's `formal_in:` `of`) | call-site local id | +| `/actual_out` / `…/actual_out:` | mirrors callee's `formal_out` `of` | call-site local id | + +**Why Python's naming, not the keystone's `$ret`/`argN` [call, the one non-trivial parity choice]:** +the SDK's `resolve_value` resolves a source/sink `(name, within)` against `formal_in` `of`, and +`value_candidate` translates exactly the Python markers (`:`, `:`, ``). +With Python's spelling the shared resolver works unchanged; Java/TS's `$ret`/`argN` would require a +Go-specific translation layer. The `:` multi-result form is a Go-only leaf extension of +``. Sources/sinks address `formal_in` only, so `:` never reaches +`value_candidate`; it does reach `body_node_kind` (which labels path hops), where `RETURN_MARKER` +is compared by equality — so an unnamed multi-result hop would be labelled `return` with name +`:1`. The Go backend normalises this locally before calling the shared helper; shared +`commons/` stays untouched. + +**Closures — by-reference captures, flow-insensitive binding [call, from builder check].** Go +closures capture variables by reference, and the closure may run later (`defer func(){…}()`, +`go func(){…}()`, stored callbacks), so binding a capture to the defs reaching the `FuncLit` would +miss later assignments (false refutation). Rule: + +- The closure callable gets `@formal_in:` `of:":"` per captured variable and + `@formal_out:` `of:":"` per captured variable it **writes** (strong or weak + def). +- In the enclosing callable, the `FuncLit` has a **capture site**: the statement containing the + `FuncLit` is the host; vertices `/actual_in:` and `/actual_out:` + (`` = `line:col` of the `func` keyword; `parent` = host statement's local id). + `param_in`/`param_out` connect them to the closure's capture formals. No `summary` edges at a + capture site (it is not a call; flow through the closure is via its own body). +- **Flow-insensitive wiring for captured variables** (port wiring, `prov:["reaching-defs"]`): + *every* def of a captured variable anywhere in the enclosing callable → the capture site's + `actual_in`; the capture site's `actual_out` → *every* use of that variable in the enclosing + callable. This over-approximates timing (covers defer, goroutines and callbacks alike) and is the + same weak-update posture as the rest of the design. +- Calls *of* the closure value (`f := func(){…}; f()`) resolve to the closure callable via VTA at + L4 and get ordinary call-site vertices for its declared parameters. + +Go specifics: +- **Variadic** `f(xs ...T)`: excess actuals collapse onto the last formal (weak, as Java). +- **Multi-value call as args** `f(g())`: each `g` result binds to the corresponding `f` formal. +- **Method value / expression** (`f := s.M; f(x)`; `T.M(s, x)`): resolved callee is `M`; receiver + binds `formal_in:0`. +- **Global state as extra parameters** (keystone L4 rule): globals read/written are computed in the + summary pass and propagated up the call graph, so a caller's `formal_in`/`formal_out` set includes + globals its callees touch (Python precedent). +- **Global filter — only globals that can carry data [call, from builder check].** Python emits + 546k `:` formals on one real app; transitive propagation is quadratic at cockroach scale. + A package-level variable is modelled as an extra parameter only if it is **mutable in a way that + can carry a value**: it has at least one write (strong or weak def, incl. a method call through it + or `&v` escaping) in any project function body including `init`, **or** its initializer contains a + call (e.g. `os.Getenv`). A global initialized only from literals/constants and never written + cannot carry taint, so dropping it removes no flow. Constants and functions are never globals. + This filter is sound under the over-approximation rule; output size is measured in the eval (§8) + and a size budget set from the first measurement. + +### 4.7 L4 edges [call] + +- `param_in` (application scope, global ids): `@/actual_in:` → + `@formal_in:`, **`var` = callee formal `of`** on every edge (Python #195 / Java #250 — + the SDK's sanitizer cuts match on `var`). +- `param_out`: `@formal_out[:]` → `@/actual_out[:]`, `var` = `of`. +- **Port wiring** (`ddg`, `prov:["reaching-defs"]`, Python precedent; Java uses `ssa` — following + Python because both projections of the SDK accept it and it keeps port edges distinguishable from + statement def-use): + `@formal_in:k → first-use statements of that base` (replaces the `@entry` def for that base for + port purposes — keep both, the `@entry` edge stays for L3 monotonicity); `return stmt → + @formal_out[:i]` (`var` ``/name), statement writing a by-ref param/global → + corresponding `@formal_out`; `host stmt → /actual_in:i` (`var` = arg spelling or `argN` if + not an access path); `/actual_out[:i] → host stmt` (`var` = the LHS base it defines). **This + wiring is what the SDK's `_ports_carry_dependence` probe checks; without it taint certifies false + refutations.** +- `summary` (callable scope, local ids): `/actual_in:i → /actual_out[:j]` iff the + callee's summary says formal `i` may flow to output `j`. +- Edge-only-when-resolved: a call site whose callee cannot be resolved at all (e.g. a call through a + `func` value with no VTA target) gets no vertices — **but** to avoid a false refutation it gets + conservative intra-callable `ddg` edges `arg host → host` (the existing L3 "call uses args, defines + results" already provides this: L3 DDG treats the call statement as using every argument and + defining its LHS). Taint therefore still flows *through* an unresolved call within the caller; it + just cannot enter the unknown callee. +- No dangling endpoints; `param_in`/`param_out` sorted by `(src,dst)`. + +### 4.8 Dispatch resolution: VTA for L2 and L4 [call] + +Must-have flow (3) needs concrete targets for interface and function-value calls. Use +`x/tools/go/callgraph/vta` over `go/ssa` (built with `ssautil.AllPackages` from the same +`go/packages` load, `ssa.InstantiateGenerics`), seeded with a CHA call graph +(`callgraph/cha`) — the construction guide's recommended Go oracle. + +- **L2 refinement:** `call_graph` gains VTA-derived edges for interface/func-value call sites, `prov` + `["vta"]`, merged with existing `["go/types"]` edges via `MergeEdges` (union prov, sum weight). + This is additive (monotone). Body `callee` stays a single id (the static callee — the interface + method for an interface call) — the schema has one slot. Multiple concrete targets are carried by + `call_graph` and, at L4, by one `param_in`/`param_out` set **per target** from the same + `actual_in` vertices (the actual vertices are per call site, not per target). +- **Interface methods are already callable nodes** (bodiless, in the interface type's + `callables{}` — `collectInterfaceMethods`), so an interface call's static `callee` resolves today. + At L4 a bodiless interface method gets `formal_in`/`formal_out` vertices (so the call site's + `actual_*` have a target for the static edge) and a **pass-through summary** (like a stand-in, §6), + so a call whose VTA target set is empty still does not drop flow. When VTA finds concrete targets, + `param_in`/`param_out` go to each concrete target **and** to the interface method; the + call-site `summary` edges are the union over targets. +- **SSA/VTA scope [call, from builder check]:** build SSA with `ssautil.AllPackages` over the + loaded packages (dependency SSA is needed for types and wrappers), but give VTA a function set of + **project functions only** (the SSA functions whose position lies in a project file, plus their + anonymous functions), seeded with a CHA graph restricted the same way. VTA needs no `main`/roots, + so libraries work. Library callbacks are therefore not discovered (§6, as intended) and cost stays + bounded by project size. Packages that fail to type-check get no SSA (`AllPackages` yields nil); + see §4.12 degradation. +- **SSA → `can://` mapping [call]:** map an SSA function to a callable by source position: + `fn.Origin()` for generic instantiations; synthetic wrappers (bound `$bound`, thunk `$thunk`, + promoted-method and pointer wrappers from embedding) unwrap to the method they wrap; anonymous + functions (`f$1`) map to the L1 closure whose `FuncLit` position equals `fn.Pos()` (L1 closure + naming `closure_N` stays; the position is the join key). A target that maps to nothing in-project + is an external stand-in (§6) if it has a package path outside the project, else dropped with a + debug log. Build an `ast` position → callable index once (also fixes the parked per-site AST walk). +- **VTA runs only at `-a 4`** (and `--emit neo4j`, which is `-a 4`): SSA construction is the + dominant cost; `-a 2`/`-a 3` keep today's cheap `go/types` resolver (an `-a 3` run must not pay + L4 cost). VTA edges therefore appear in `call_graph` only at `-a 4` (still monotone: L3 ⊆ L4). +- Unsound spots (reflection, `unsafe`, cgo, plugins): no targets ⇒ handled by the unresolved-call + rule above (§4.7). +- SSA is also the substrate for the may-alias oracle (§4.10). + +### 4.9 Summary pass [call] + +Bottom-up over the call graph (L2 `go/types` edges ∪ VTA edges), condensed with iterative Tarjan +SCC (sorted for determinism), callees first; non-recursive singletons solved once, recursive SCCs +iterated to a monotone fixpoint over finite domains (formal indices × outputs; globals sets). +Context-insensitive. Per callable compute: + +- `flows ⊆ inputs × outputs`, inputs = formals (incl. receiver, captures, globals), outputs = + results, by-ref params, written globals — by forward reachability over the callable's + `ddg ∪ cdg` from each input's `@entry`/`formal_in` def, bridging call sites via callees' + summaries (`summary` edges). Including `cdg` (implicit flow) matches Python and over-approximates. +- `global_reads`, `global_writes`, `mutated_params` (written through, from the weak-def rule), + propagated up the SCC DAG. +- No hammock regions (neither sibling built them; recorded as a known simplification). +- Stand-ins (§6) contribute their modelled summary (default all-inputs → all-outputs). +- Caching of summaries in `cache_dir` and SCC-wavefront parallelism: **builder's choice** whether + to implement in this train; determinism is mandatory either way. + +### 4.10 Alias oracle and the L4 semantic DDG delta [call] + +L4 adds `ddg` edges with `prov:["points-to"]` **in addition** to L3's `ssa` edges (strictly +additive). Oracle: **type-based may-alias MVP** — two access paths with different bases may alias iff +the bases' types are reference types (pointer/slice/map/chan/interface/func) with compatible +pointee types (identical, or one is an interface the other's dynamic type may satisfy), and their +field suffixes are prefix-compatible. The oracle answers `may_alias(path1, path2)`; the DDG solver +re-runs with it consulted only when a suffix exists (Python precedent). This catches +`p := &req; p.UserID = id; … q.UserID` (flow 2) without SSA points-to. Rejected for this train: +an Andersen solve over `go/ssa` (precision upgrade, own PR later, keystone's "MVP may stub the +oracle"); `go/pointer` (deprecated). + +The oracle's API is frozen as an interface so an Andersen implementation can replace it later +without schema change (`prov` stays `points-to`). + +### 4.11 Neo4j projection [call] + +Fill the reserved overlay, no contract renames: + +- `:GoBodyNode` for every body node incl. synthetic vertices; props add `of` → **`var`** and + `parent` → **`call_node`** (the **local** parent key, JSON `parent` verbatim, resolved against + the owning callable — Python parity: codeanalyzer-python's `project.py` stores `node.parent` and + python-sdk's `reconstruct.py` maps `call_node` straight back to `parent`; a global id would not + round-trip) — Python's projection names, which the SDK's Cypher reads + (`projection` includes `n.var`). Keep `kind`, `is_goroutine`, `is_deferred`, `span_json`; add the + flat span props siblings use if the SDK backend needs them (builder confirms against + `PyNeo4jBackend`). +- `GO_HAS_BODY_NODE` callable → every body node (incl. synthetic). +- `GO_CFG_NEXT {kind}` `_k=kind`; `GO_CDG`; `GO_DDG {var, prov}` **`_k=var|prov`** (fix the + current `var`-only discriminant); `GO_SUMMARY`; `GO_PARAM_IN {var}` / `GO_PARAM_OUT {var}`. +- Stand-ins (§6) project onto the existing `:GoExternal:GoSymbol:GoCanNode` ghost (same + `can:///@external/…` id, so today's call-graph ghosts and stand-ins are one node), gaining + `signature`/`module` props and `GO_HAS_BODY_NODE` to their `formal_*` vertices. Their vertex ids + are `@formal_in:`, so the SDK's `under_callable` id-prefix rule covers them. + `RelHasBodyNode.From` widens to `{GoCallable, GoExternal}` in `schema.go`. +- Flip `Reserved:false`; conformance test must see every declared relationship emitted on the + fixture; cross-projection gate: Neo4j counts == JSON counts modulo `HAS_*`. +- `--emit neo4j` now runs `-a 4`. + +### 4.12 CLI and envelope [call] + +- `-a 3`, `-a 4` accepted; `--graph-field-depth ` (default 3, error below `-a 3`, error with + `--emit neo4j`). +- Envelope: emit `k_limit` at L3+ (Python precedent; harmless superset of the keystone's "at L4"). +- `max_level` = requested level. +- `--strict`: fail non-zero when SSA/VTA construction fails at `-a 4` (default: degrade — keep L3, + derived vertices and param edges from `go/types` dispatch, log a warning to stderr, and record + the `degraded` envelope marker, §4.15), Java precedent. +- `-j N`: parallelize L3 per callable and L4 per SCC wavefront if implemented; collect then sort by + id; never mint ids or emit inside a worker. Byte-identical to `-j 1`. + +### 4.13 Caps [call] + +No hard node/time caps (neither sibling has them; a cap silently drops edges ⇒ false refutation). +Termination comes from k-limiting and finite summary domains. Memory: build L3 per callable and +discard ASTs-derived scratch after emission; SSA program is shared and freed after L4. + +### 4.14 L2 external calls — fixing the current gap (Phase 0) [user: fix it; details are calls] + +Today the L2 resolver resolves library callees (`internal/semantic_analysis/call_graph.go:105-131`) +and then drops them: no `call_graph` edge, and the emitter leaves the call node's `callee` `null` +(`v2emit/emit.go:207-216`). Python and TS emit these; Java emits them behind `--external-calls` +(always under `--emit neo4j`). Go fixes this at L2: + +- **`application.external_symbols{}`** (the Python/Java name) — one entry per distinct external + callee: `id` (`can:///@external//`, §6), `kind:"external"`, `name`, + `module` (import path), `signature`. Parameters/return type are added here too (cheap from + `go/types`; span-less `{name, type, is_variadic?}`); the L4 `body{}` vertices come in Phase 2. + An external id may name a library *interface* method (e.g. `@external/io/Writer.Write`), and a + method on a generic library type records the declared (origin) signature. +- **`call_graph`** keeps edges to external targets (`dst` = external id, `prov:["go/types"]`). +- **Body `callee`** is backfilled to the external id (Python precedent: a callee is "a symbol-table + callable or an `@external` home"). Still a monotone `null → id` refinement. +- **On by default, no flag** (Python precedent; Java's opt-in flag exists only for its v1 parity, + which Go does not need — Go is already breaking ids in this release). +- Still `null`: builtins (`len`, `append`, `make`, …, which `go/types` does not model as + `*types.Func`), calls through function values, and anything unresolvable. Type conversions + (today minted as `.__new__`) stop being minted and stay `null` — they are not calls. Taint + still flows through a conversion (`string(b)`) because L3 DDG treats it as an expression: the + statement defining the LHS uses `b`. (Builtins likewise: L3 "call uses every argument, defines its + LHS".) +- This is an intended **L2 output change shipped in Phase 0** (more `call_graph` edges, non-null + `callee` on library calls, a new `external_symbols` map). +- Neo4j: the existing `:GoExternal` ghost path in `project.go` (`callEndpoint`) starts receiving + rows; `GO_RESOLVES_TO` extends to external callees. `GO_CALLS` endpoint set already allows + `GoExternal`. +- Gate update: the L2 fixture gate (`gate_l2_test.go`) currently asserts external callees stay + `null`; it changes to assert the external id plus a matching `external_symbols` entry, and no + dangling endpoint. + +### 4.15 Degradation marker [user] + +When `-a 4` cannot build what L4 needs — SSA/VTA construction fails, or some packages failed to +type-check and got no SSA — the analyzer still emits (L3 + derived vertices + `go/types` dispatch) +but marks the envelope: + +```json +"degraded": [{"code": "no_vta", "detail": "...", "packages": ["example.com/x/pkg"]}] +``` + +- Envelope leaf, sibling of `max_level`; **absent** when nothing degraded (absence = no fact). + Codes (open vocabulary, plain strings): `no_ssa` (SSA build failed entirely), `no_vta` (VTA + failed), `partial_ssa` (listed packages had type errors / no SSA), `partial_types` (packages with + `go/packages` errors at any level). `max_level` still states the requested level. +- Projected to Neo4j as a `degraded_json` property on `:GoApplication`. +- `--strict` turns any degradation into a non-zero exit. +- **SDK:** the Go backends read it at attach. When present, `taint()` returns **no `exhausted` + pairs** and adds an `unresolved` diagnostic naming the degradation (`complete=False`), so a + "no flow" certificate is never issued over a degraded graph; witnesses (`paths`) are still + returned. Same rule for `paths_between`-style refutations. Implemented in the Go backend (the + diagnostic goes into the `blocked` ledger the walk already returns, which `taint_verdict` honours — + shared `commons/` unchanged). +- Go-only leaf now; candidate for the keystone alongside the cross-language taint revisit (§11). + +## 5. Flow coverage map (user goal 4 → mechanism) + +| Flow | Mechanism | Level | +|---|---|---| +| (1) through calls | formal/actual vertices, `param_in/out`, `summary`, port wiring | L4 | +| (3) interface / func-value dispatch | VTA targets → per-target `param_in/out` | L4 | +| (2) fields & pointers | k-limited access paths; weak-def through reference args; type-based may-alias `points-to` delta | L3 + L4 | +| closures | captures as `:` formals; nested callable's own SDG; capture bound at the `FuncLit` statement as `actual_in` (Python precedent) | L4 | +| goroutines | `go f(x)` is a call for data purposes (vertices + edges); no interleaving | L4 | +| channels | `ch[*]` container path; cross-goroutine via args/captures/fields | L3 + L4 | +| library calls | stand-ins with default pass-through summary | L4 | + +## 6. Library stand-ins (sinks/sources naming library functions) [user] + +**Problem:** the SDK resolves `(name, within)` by first resolving `within` to an *analysed* callable, +then a `formal_in` of it. `database/sql.(*DB).Query` is not in the tree, so it cannot be a sink. +**Decision:** the analyzer emits a **stand-in** for every external function/method the project +actually calls (resolved via `go/types`; VTA targets outside the project included). Go-only leaf +addition now; the cross-language SDK taint revisit is deferred (§11). + +- **Where in the tree:** a new optional application-scope map + `application.external_symbols{ : external }` — the same name and shape Python + (`PyApplication.external_symbols`) and Java (D19) already ship for call-graph endpoint homes — + **extended** on Go with `callables`-like content: each external entry carries `kind:"external"`, + `name`, `module` (import path), `signature`, `parameters[]` (real names from `go/types`; unnamed + → `_`), `return_type`, and at L4 a `body{}` holding only its `@formal_in:` / + `@formal_out[:i]` vertices. The map itself lands at L2 in Phase 0 (§4.14); Phase 2 adds only + `body`. Reachable by containment from `application` ⇒ keystone-legal. +- **Id:** `can:///@external//` where `` is the existing + `calleeSignatureOf` canonical form minus the package path (`DB.Query`, `Sprintf`). Language-neutral + `@external` per Python/TS/Java. +- **Vertices:** `formal_in` per receiver+param (`of` = real name), `formal_out` per result + (`` / `:`), plus by-ref `formal_out` for each pointer/slice/map/interface param + and pointer receiver (library may write through them — `json.Unmarshal(data, &v)`, `buf.Write`). +- **Stand-in summary (pass-through default):** every `formal_in` → every `formal_out`. Materialized + as `summary` edges at each **caller's** call site (`actual_in:i → actual_out:j`), *not* as edges + inside the stand-in. The stand-in has no outgoing `param_out` into unrelated callers beyond the + ones each call site owns (param_out goes stand-in `formal_out` → *that* call site's + `actual_out`), so — exactly as for in-project callees — an HRB-unaware shortest-path walk + could enter at call site A and leave at call site B. **This is the existing SDK behaviour for every + shared callee in every language** (the SDK's context-sensitivity gate is "if summary edges + present"); not made worse by stand-ins. Recorded as a known over-approximation. +- **Models file (optional overrides):** `--library-models ` (builder may also embed a + small default set), mapping an external id/signature to an explicit flow list, e.g. + `strconv.Atoi: [] ` (no flow), `strings.TrimSpace: [[0,""]]`. Starts empty or + near-empty; no catalogue of sources/sinks — that stays the caller's (SDK rule). +- **Not modelled:** callbacks invoked by library code (`sort.Slice(xs, less)`, + `http.HandleFunc(path, h)`): the function-value argument flows in, but the library calling it back + is not an edge. Taint *starting* in a handler is unaffected (sources are the handler's own + params). Recorded limitation. +- **SDK addressing:** `("query", "database/sql.DB.Query")` resolves through the Go backend's + `resolve_callable` (domain = analysed callables ∪ stand-ins) then `resolve_value` over the stand-in + body's `formal_in` `of`. Shared `commons/` code is untouched. + +## 7. SDK (python-sdk) [user: in scope; Claude may write it] + +Build on python-sdk PR #434 (L1/L2 Go support: models, ABC, local + Neo4j backends, facade, factory; +unmerged as of 2026-10-23), fixing it first against the v2-only analyzer. `add-go-support` is obsolete. + +- **Models** `cldk/models/go/`: `extra="ignore"` (PR #434's choice, matching TS #386) mirror of Go v2 (module/type/field/callable/body + node/edges/external symbols), mirroring how `cldk/models/python` and `java` model L3/L4. +- **Facade** `GoAnalysis` via `CLDK.go(...)` and the legacy shim; `GoAnalysisBackend` ABC with + `N="Go"`, `P="GO_"`. +- **Surface:** the decided CLAUDE.md SDK facade (type-centric: `get_types`/`get_type`, + `get_functions`, `get_methods_of`/`get_method`, `get_fields`, `get_callers`/`get_callees`) **plus** + the L3/L4 accessors with the **same names and signatures as `PythonAnalysisBackend`**: + `get_cfg`, `get_cdg`, `get_ddg`, `resolve_callable`, `resolve_value`, `slice_backward`, + `slice_forward`, `paths_between`, `reaches`, `backward_cone`, `flows_to_call`, + `flows_to_argument`, `taint`, `get_source`, `get_external_symbols`. Implementations route through + `commons/graphs.py`, `commons/resolve.py`, `commons/results.py` exactly as Python's do. + `resolve_callable` for Go accepts receiver forms `DB.Query`, `(*DB).Query`, `sql.DB.Query`, + `database/sql.DB.Query` (segment matching via existing `resolve_callable_signature`). +- **Backends:** local `GoCodeanalyzer` (runs `cango` subprocess, parses JSON, builds networkx + graphs) and `GoNeo4jBackend` (Cypher via `sdg_path_query`/`sdg_taint_query` with + `node_label="GoBodyNode"`, `P="GO_"`, `endpoint_scope`/`interior_scope` per the multi-application + scope audit). Contract test `test_go_backend_contract.py`; parity: both backends answer identically + including miss paths. +- Gate `_require_dataflow` (local) at level 3; port-connectivity probe — refuse slicing/taint if no + dependence edge leaves any `formal_in`. This exists only on the Java backend + (`java/backend.py:2329`, abstract there, implemented in both Java backends); Go declares and + implements it the same way (Python has none). +- **Satisfying the shared `AnalysisBackend` ABC [call, from builder check].** The ABC + (`commons/backend.py:101-184`) requires class-centric abstracts (`get_all_classes`, `get_class`, + `get_all_methods_in_class`, `get_method`, `get_all_fields`, `get_method_parameters`) and the + `Py*`-typed artifact getters. Go implements them as required, without changing the ABC: "class" + = any named Go type (`GoType`, all four kinds), methods = its receiver methods, fields = struct + fields; artifact getters return the shared `Py*` artifact classes (as the ABC documents). The + CLAUDE.md type-centric names (`get_types`, `get_functions`, `get_methods_of`, …) are the + **documented** Go surface; the ABC names are thin aliases over them. This refines CLAUDE.md + decision SDK1 ("not `get_classes`") — the ABC makes the class-named methods unavoidable; + update CLAUDE.md accordingly. +- CLAUDE.md (python-sdk) language table updated in the same change. + +## 8. Verification + +**Fixtures** (`testdata/`, new `testdata/dataflow/` module + augment existing): +if/else + loop; early return + panic + defer; `recover`; `select`; labeled break/continue; `goto`; +`fallthrough`; type switch; range over chan/map/slice; infinite `for {}`; closure capture; +goroutine + channel; struct field + pointer write-through (`svc.Handle(&req)`); interface dispatch +with ≥2 implementations; func-value call; `a→b→c` chain; mutual recursion; global written in one +function read in another; multi-file flow; cross-file method; multi-result returns; variadic; +library pass-through and library sink (`database/sql`, `os/exec`, `fmt`); generics (existing +`testdata/generics`: instantiation → `Origin()` mapping, generic interface dispatch); an HTTP-handler → +SQL-sink end-to-end taint fixture with a sanitizer variant (the "done" fixture). + +**Analyzer gates** (each level gated before advancing — `codeanalyzer-backend` HARD-GATE): +CFG gate, dominance gate (exact CDG), PDG-slice gate (exact set), DFG gates, L4 gates (no dangling; +arity; summary exists for known transitive flow; `points-to` ⊇ adds-only; ports carry dependence), +determinism (`-j 1` vs `-j N`, two runs), monotonicity (`-a 1..4` superset via a test like +Python's `test_v2_superset.py`), cross-projection (Neo4j vs JSON), Neo4j conformance, SDK model +validation of every level's output. + +**SDK gates:** slice gate (exact set on a `can://…@line:col` criterion); taint gate (known +source→sink found; sanitizer variant reported cut, not dropped; contiguous witness); a pair with no +flow lands in `exhausted`; context-sensitivity only as far as summaries support; JSON and Neo4j +backends agree; e2e level matrix. + +**"Done"** (user-confirmed intent, designer-worded): through `CLDK.go(...)`, on the HTTP→SQL +fixture, `taint()` finds the flow from a handler's request parameter to `database/sql.DB.Query`'s +`query` through an interface call; a callable sanitizer cuts it; an unrelated pair is `exhausted`; +identical answers from the JSON and Neo4j backends. + +**Real-world evaluation** (non-gating metrics + crash/determinism/time gates): the prior eval set — +cobra (small), dae, openbao, cockroach (scale) — plus listmonk and memos (HTTP → SQL; memos through +the `store.Driver` interface) and nats-server (concurrency / CFG stress) [user, 2026-10-23]. dae and +cockroach need their code-generation step (bpf2go, protobuf) before analysis. Record per app: wall time and peak RSS at `-a 3`/`-a 4`, body/edge counts, % call +sites with resolved targets (go/types vs VTA), % externals with stand-ins, determinism. Gate: all +four complete `-a 4` without crash and deterministically; no hard time budget, but timings are +reported and a regression budget is set after the first measurement. + +## 9. Phasing and release + +| Phase | Repo | Content | Release | + +> **Phases 0 and 1 ship together in one PR / one release** [user, 2026-10-23]; they stay separate rows for traceability. + +|---|---|---|---| +| 0 | analyzer | id grammar `can:///go/…`; **L2 external calls (§4.14)**; L1 co-positioned call keys; `is_deferred`; `typeKind` alias/defined; `GO_DDG` `_k`; call-site position index (perf) | analyzer next **breaking** version (wheel is at 0.1.0 with no local tags — builder confirms the released version and bumps the major, or the minor if still 0.x) | +| 1 | analyzer | L3: body statements, CFG (+defer chain, panic), CDG, syntactic DDG, CLI `-a 3`/`--graph-field-depth`, Neo4j CFG/CDG/DDG, gates | minor | +| 2 | analyzer | L4: SSA+VTA, `degraded` marker (§4.15), external stand-ins (`external_symbols`), vertices, param edges, summaries, port wiring, type-based alias delta, Neo4j L4 overlay, `--emit neo4j` at L4, real-app eval | minor | +| 3 | python-sdk | Go v2 models, facade, local + Neo4j backends, L1–L4 accessors, taint e2e | minor; pins the analyzer only after Phase 2 is released | + +Tracking: `codellm-devkit/.github` parent issue + one sub-issue per PR (Phase 0+1 as one) (filed just-in-time), +per `issue-and-pr-tracking.md`. Each phase gets its own implementation plan. +Docs updated per phase: CLAUDE.md § Schema decisions (new Go leaves: `defer_resume`, stand-ins, +`:`, kind mapping), `docs/design/roadmap.md` (L3/L4 un-deferred), +`v2-schema-structure.md`, README "Architecture & Tooling" (substrate slots: CFG = go/ast hand-built +with go/cfg differential; def-use = syntactic reaching-defs with go/ssa differential; oracle = VTA +dispatch + type-based may-alias). + +## 10. Left to the builder + +- Package layout under `internal/` (guide: a `dataflow/` subpackage, one module per stage, `core` + stays a pure orchestrator), internal IR shapes, worker-pool design. +- Whether summary caching and SCC-wavefront parallelism land in Phase 2 or later. +- Exact differential-test tolerance vs `go/cfg` / `go/ssa`. + +## 11. Deferred / follow-ups + +- **Cross-language SDK taint revisit** [user]: library sinks/sources for Python/Java/TS (stand-ins or + multi-node selectors in `commons`); promote `external_symbols.body` stand-ins into the keystone if + adopted. +- Andersen points-to over `go/ssa` replacing the type-based oracle (`prov` unchanged). +- Context-sensitive (HRB two-phase) taint traversal in the SDK. +- Library callback modelling (`sort.Slice`, `http.HandleFunc`). +- Promote `callable.source_file` to the keystone (existing deferral). +- Struct-tag modelling (existing deferral). + +## 12. Risks + +| Risk | Mitigation | +|---|---| +| SSA/VTA cost on cockroach-scale | SSA only at `-a 4`; measure in Phase 2 eval; `--strict` / degrade path | +| False refutations from a missed construct | over-approximation posture; "ports carry dependence" probe; per-construct fixtures; unresolved-call intra-caller pass-through | +| Over-approximation noise (defer chain, pass-through, CDG in summaries) | acceptable by anti-goal; library models file narrows hot spots | +| Breaking id change disrupts users | ship alone in Phase 0, before SDK pin; no released SDK consumes Go output | +| Determinism with parallel L3 / map iteration | collect-then-sort, ids minted outside workers, `-j` gate | +| Divergence between JSON and Neo4j | cross-projection gate; shared projector over the same v2 tree | diff --git a/docs/design/specs/v2-schema-structure.md b/docs/design/specs/v2-schema-structure.md index f833b43..aa7b45a 100644 --- a/docs/design/specs/v2-schema-structure.md +++ b/docs/design/specs/v2-schema-structure.md @@ -11,12 +11,13 @@ the JSON tags from that file. ``` ├─ schema_version # "2.0.0" ├─ language # "go" -├─ max_level # 1 (symbol table) | 2 (+ call graph) +├─ max_level # 1 (symbol table) | 2 (+ call graph) | 3 (+ cfg/cdg/ddg) +├─ k_limit # L3+ only: access-path bound (--graph-field-depth, default 3) ├─ analyzer │ ├─ name # "codeanalyzer-go" │ └─ version └─ application - ├─ id # "can://go/" + ├─ id # "can://" ├─ kind # "application" ├─ symbol_table # { "": module } │ └─ @@ -40,12 +41,23 @@ the JSON tags from that file. │ │ └─ # ← callable node, defined below │ └─ functions # { "": callable } │ └─ # ← callable node, defined below - └─ call_graph[] # L2+; empty [] at L1 - └─ - ├─ src # can:// callable id - ├─ dst # can:// callable id - ├─ prov[] # optional: e.g. ["go/types"] - └─ weight # optional + ├─ call_graph[] # L2+; empty [] at L1 + │ └─ + │ ├─ src # can:// callable id + │ ├─ dst # can:// callable id, or an @external id + │ ├─ prov[] # optional: e.g. ["go/types"] + │ └─ weight # optional + └─ external_symbols # L2+; optional: { "<@external id>": external } + └─ # library callee: `can:///@external//` + ├─ id + ├─ kind # "external" + ├─ name # e.g. "Println" + ├─ module # import path, e.g. "fmt" + ├─ signature # [Type.]Name, e.g. "Builder.WriteString"; may name a + │ # library interface method (e.g. "Writer.Write") + ├─ parameters[] # { name, type, is_variadic? } (no span: not in-project); + │ # receiver excluded; unnamed params are "_" + └─ return_type # optional ``` ## `` node (recursive) @@ -58,24 +70,70 @@ The same shape whether it sits under a type's `callables` (methods), a module's ├─ id ├─ kind # function | method | lambda ├─ signature -├─ source_file # optional: set only for a method declared in a -│ # different file than its receiver type; its -│ # span.bytes then index symbol_table[source_file].source +├─ source_file # optional: set only when the declaring file differs +│ # from the containing module — a method declared in +│ # a different file than its receiver type, and every +│ # closure (nested ones too) inside such a method; its +│ # span.bytes, and its body nodes' spans, then index +│ # symbol_table[source_file].source ├─ span { start, end, bytes } ├─ parameters[] { name, type, span, is_variadic? } ├─ return_type # optional ├─ error_channel[] # optional: error-typed returns ├─ metrics # optional: { cyclomatic: N } ├─ callables # optional: nested closures (same shape) -└─ body # { "": call node } - └─ - ├─ kind # "call" - ├─ span { start, end, bytes } - ├─ callee # null | can:// id (key always present) - ├─ is_goroutine # optional: present only when true - └─ is_deferred # optional: present only when true +├─ body # { "": body node }; key = "[/n]" | "@entry" | "@exit" +│ ├─ # L1+ +│ │ ├─ kind # "call" +│ │ ├─ id # L1+: @ +│ │ ├─ span { start, end, bytes } # the call expression (unchanged at L3) +│ │ ├─ parent # L3+, optional: key of the statement a nested call sits in +│ │ ├─ callee # null | can:// id (key always present) +│ │ ├─ is_goroutine # optional: present only when true +│ │ └─ is_deferred # optional: present only when true +│ └─ # L3+ +│ ├─ kind # statement | branch | loop | switch | return | entry | exit +│ ├─ id # @ (@entry → @entry) +│ └─ span { start, end, bytes } # the statement; absent on entry / exit +├─ cfg[] # L3+, optional: { src, dst, kind } — local body keys +├─ cdg[] # L3+, optional: { src, dst } +└─ ddg[] # L3+, optional: { src, dst, var, prov } (prov = ["ssa"]) ``` +### L3 body nodes and edges + +- **Kinds** (shared vocabulary, no Go-specific kinds): `if` → `branch`; `for` / `for … range` → + `loop`; `switch` / type `switch` / `select` → `switch`; `return` → `return`; every other + statement (assignment, `:=`, `var`, inc/dec, send, `go`, `defer`, `break` / `continue` / + `goto` / `fallthrough`) and each `case` / `default` / comm-clause head → `statement`; + `@entry` / `@exit` → `entry` / `exit`. +- **Call nodes.** A bare call statement and a `go` statement *are* their call node (kind + `call`, the L1 key, the CFG step). A call nested in a statement keeps its L1 key and span + and carries `parent` = that statement's key. It is not a CFG step and has no edges. + `defer f(x)` is two nodes: a `statement` at the `defer` keyword, and the `call` node + (`is_deferred`) placed in the exit chain. An L1 call node in unreachable code is still + emitted, but it has no edges. +- **Keys.** A statement that starts at the same position as an L1 call (`f().x = v`) takes + the next free `/n` suffix, so the L1 call key never moves. +- **`cfg` kinds:** `fallthrough`, `true`, `false`, `loop_back`, `switch_case`, `break`, + `continue`, `return`, `exception`, and the Go add `defer_resume`. Sorted by + `(src, dst, kind)`. +- **`cdg`** is control dependence from post-dominators. A node with no controller gets an + edge from `@entry`. Sorted by `(src, dst)`. +- **`ddg`** comes from syntactic reaching definitions over k-limited access paths + (`base(.field|[*])*`, past `k` → `.*`). `var` is the source spelling of the path, and + `prov` is always `["ssa"]` at L3. Sorted by `(src, dst, var)`. It is **not + refutation-complete**: writes to captured variables inside closures and writes through + reference-typed fields of value-typed struct arguments/receivers are not modelled at L3 + (no edge ≠ no flow); L4 closes them. A deferred call's receiver/argument paths are keyed on + the variable at exit, so a reassignment after the `defer` can add a spurious edge or kill an + earlier def. See `v2-l3-l4-dataflow.md` §4.5 "L3 precision". +- **Spans.** A statement node's span is its AST node's range, and a clause head ends at its + colon. For a callable with `source_file` (a method declared apart from its receiver type, + or a closure inside one), body spans index `symbol_table[source_file].source`, as the + callable's own span does. A closure never inherits `source_file` implicitly: each callable + carries its own. + ## `span` ``` @@ -87,22 +145,34 @@ span ## Presence rules -- **Always present:** every `id`, `kind`, `span`; the envelope fields; `symbol_table`, - `types`, `callables`, `functions`, `body` maps (may be empty `{}`); `call_graph` (empty - `[]` at L1); a body node's `callee` key (value `null` when unresolved/external). +- **Always present:** every `id` (body nodes included, from L1 on — parity with + codeanalyzer-python, which stamps body ids at L1), `kind`, `span` (except: `@entry` / + `@exit` have no `span`); the envelope fields; `symbol_table`, `types`, + `callables`, `functions`, `body` maps (may be empty `{}`); `call_graph` (empty `[]` at L1); + a call node's `callee` key (value `null` when unresolved/external). Non-call body nodes + have no `callee`. +- **L3+ only:** `k_limit`; body node `parent`; non-call body nodes; a callable's + `cfg` / `cdg` / `ddg`. These are omitted for a callable with no body. Each is a pure + addition: the `-a 2` document is a subset of the `-a 3` one. - **`omitempty` — emitted only when non-empty:** `content_hash`, `imports[].alias`, `base_types`, `interfaces`, `fields`, `source_file`, `return_type`, `error_channel`, `metrics`, a callable's nested `callables`, `parameters[].is_variadic`, edge `prov` / - `weight`. + `weight`, `external_symbols` (absent when no library call was resolved), + `external_symbols[].return_type`. +- A call node's `callee` is an in-tree callable id, an `external_symbols` key (library + callee), or `null` (conversion, builtin, call through a function value, unresolved). - **Emitted only when `true`:** `is_goroutine`, `is_deferred`. ## `id` shape ``` -application can://go/ -module can://go// -type can://go/// -callable can://go//// +application can:// +module can:///go/ +type can:///go// +callable can:///go/// (functions omit the segment) -body call keyed by ":" within the callable's body map +external can:///@external// (language-neutral) +body node keyed by ":" (co-positioned: ":/") or "@entry" / "@exit" + within the callable's body map; its id (every level) is @ + (a synthetic key already carries the "@": @entry) ``` diff --git a/go.mod b/go.mod index e514b7e..dcc5ee5 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,16 @@ module github.com/codellm-devkit/codeanalyzer-go -go 1.25.0 +go 1.26.0 require ( github.com/neo4j/neo4j-go-driver/v5 v5.28.5 github.com/spf13/cobra v1.8.0 - golang.org/x/tools v0.46.0 + golang.org/x/tools v0.52.0 ) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/mod v0.37.0 // indirect - golang.org/x/sync v0.21.0 // indirect + golang.org/x/mod v0.42.0 // indirect + golang.org/x/sync v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index c30bbc2..33b67a0 100644 --- a/go.sum +++ b/go.sum @@ -10,11 +10,11 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk= -golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/core/analyzer.go b/internal/core/analyzer.go index fb4e5b7..2329528 100644 --- a/internal/core/analyzer.go +++ b/internal/core/analyzer.go @@ -9,6 +9,7 @@ // 2. Symbol table construction (syntactic_analysis) // 3. Resolver-based call graph (semantic_analysis) — if level >= 2 // 4. Pass pipeline (analysis/registry) +// 5. Intraprocedural dataflow (dataflow) — if level >= 3 package core import ( @@ -19,6 +20,7 @@ import ( "path/filepath" "github.com/codellm-devkit/codeanalyzer-go/internal/analysis" + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" "github.com/codellm-devkit/codeanalyzer-go/internal/options" "github.com/codellm-devkit/codeanalyzer-go/internal/schema" "github.com/codellm-devkit/codeanalyzer-go/internal/semantic_analysis" @@ -30,6 +32,24 @@ import ( // Analyzer is the top-level analysis driver. Construct with New() and call Analyze(). type Analyzer struct { opts options.AnalysisOptions + // l3 is the Phase 5 side table (nil below L3). It stays off + // schema.GoApplication: schema must not depend on dataflow, and the + // cached v1 model has no L3 facts. + l3 map[dataflow.L3Key]dataflow.Result +} + +// L3 returns the dataflow side table of the last Analyze run: one result per +// callable with a body, keyed by (declaring file, signature). Nil below +// level 3. Pass it to RenderJSON / WriteOutput / v2emit.Emit. +func (a *Analyzer) L3() map[dataflow.L3Key]dataflow.Result { return a.l3 } + +// KLimit is the access-path bound used at L3: opts.FieldDepth, or the +// --graph-field-depth default of 3 when a non-CLI caller left it 0. +func KLimit(opts options.AnalysisOptions) int { + if opts.FieldDepth > 0 { + return opts.FieldDepth + } + return 3 } // New creates an Analyzer for the given options. @@ -78,6 +98,7 @@ func (a *Analyzer) Analyze() (*schema.GoApplication, error) { ) edges := cgBuilder.Build(symbolTable) app.CallGraph = edges + app.Externals = cgBuilder.Externals() utils.Info("call graph: %d edges", len(edges)) // ── Phase 4: Pass pipeline ──────────────────────────────────────────────── @@ -89,6 +110,12 @@ func (a *Analyzer) Analyze() (*schema.GoApplication, error) { utils.Warn("pass pipeline error: %v", err) } + // ── Phase 5: Intraprocedural dataflow (L3) ──────────────────────────────── + if a.opts.Level >= options.LevelDataflow { + a.l3 = dataflow.Run(builder.Fset(), builder.L3Units(symbolTable), KLimit(a.opts), a.opts.Jobs) + utils.Info("dataflow: %d callables", len(a.l3)) + } + return a.finalizeAndCache(app) } @@ -153,7 +180,10 @@ func (a *Analyzer) saveCache(app *schema.GoApplication) error { // tree via the v2emit emitter. This is the single serialization point, so both // the stdout and file output paths agree. Only "json" is supported; other // formats return an explicit error rather than silently falling back. -func RenderJSON(app *schema.GoApplication, opts options.AnalysisOptions) ([]byte, error) { +// +// l3 is the Analyzer's L3() side table (nil below level 3). At level 3 the +// envelope carries k_limit = KLimit(opts) whether or not l3 is set. +func RenderJSON(app *schema.GoApplication, l3 map[dataflow.L3Key]dataflow.Result, opts options.AnalysisOptions) ([]byte, error) { format := opts.Format if format == "" { format = "json" @@ -167,14 +197,14 @@ func RenderJSON(app *schema.GoApplication, opts options.AnalysisOptions) ([]byte return nil, fmt.Errorf("unsupported output format %q; supported: json", format) } - payload := v2emit.Emit(app, opts.AppName, opts.InputPath, int(opts.Level), opts.AnalyzerVersion) + payload := v2emit.Emit(app, opts.AppName, opts.InputPath, int(opts.Level), opts.AnalyzerVersion, KLimit(opts), l3) return json.Marshal(payload) } // WriteOutput serializes the analysis (in the selected schema) and writes it to // outputDir/analysis.json, or to stdout when outputDir is empty. -func WriteOutput(app *schema.GoApplication, opts options.AnalysisOptions) error { - data, err := RenderJSON(app, opts) +func WriteOutput(app *schema.GoApplication, l3 map[dataflow.L3Key]dataflow.Result, opts options.AnalysisOptions) error { + data, err := RenderJSON(app, l3, opts) if err != nil { return err } diff --git a/internal/core/analyzer_test.go b/internal/core/analyzer_test.go index 72be465..3cc70a5 100644 --- a/internal/core/analyzer_test.go +++ b/internal/core/analyzer_test.go @@ -109,7 +109,8 @@ func TestCallGraph_NoDanglingEdges(t *testing.T) { if !sigs[e.Source] { t.Errorf("dangling edge source: %s", e.Source) } - if !sigs[e.Target] { + // A target is either an in-project callable or a recorded library callee. + if _, ext := sharedGreeterL2.Externals[e.Target]; !sigs[e.Target] && !ext { t.Errorf("dangling edge target: %s", e.Target) } } @@ -138,7 +139,7 @@ func TestCallGraph_CallSitesBackfilled(t *testing.T) { func TestWriteOutput_ValidJSON(t *testing.T) { outDir := t.TempDir() - if err := core.WriteOutput(sharedGreeterL2, options.AnalysisOptions{OutputDir: outDir, Format: "json"}); err != nil { + if err := core.WriteOutput(sharedGreeterL2, nil, options.AnalysisOptions{OutputDir: outDir, Format: "json"}); err != nil { t.Fatalf("WriteOutput: %v", err) } data, err := os.ReadFile(filepath.Join(outDir, "analysis.json")) @@ -160,7 +161,7 @@ func TestWriteOutput_ValidJSON(t *testing.T) { func TestWriteOutput_EmptyFormatDefaultsToJSON(t *testing.T) { outDir := t.TempDir() - if err := core.WriteOutput(sharedGreeterL1, options.AnalysisOptions{OutputDir: outDir, Format: ""}); err != nil { + if err := core.WriteOutput(sharedGreeterL1, nil, options.AnalysisOptions{OutputDir: outDir, Format: ""}); err != nil { t.Fatalf("WriteOutput with empty format: %v", err) } if _, err := os.Stat(filepath.Join(outDir, "analysis.json")); err != nil { @@ -170,14 +171,14 @@ func TestWriteOutput_EmptyFormatDefaultsToJSON(t *testing.T) { func TestWriteOutput_MsgpackNotImplemented(t *testing.T) { outDir := t.TempDir() - if err := core.WriteOutput(sharedGreeterL1, options.AnalysisOptions{OutputDir: outDir, Format: "msgpack"}); err == nil { + if err := core.WriteOutput(sharedGreeterL1, nil, options.AnalysisOptions{OutputDir: outDir, Format: "msgpack"}); err == nil { t.Fatal("expected error for --format msgpack, got nil") } } func TestWriteOutput_UnknownFormatErrors(t *testing.T) { outDir := t.TempDir() - if err := core.WriteOutput(sharedGreeterL1, options.AnalysisOptions{OutputDir: outDir, Format: "csv"}); err == nil { + if err := core.WriteOutput(sharedGreeterL1, nil, options.AnalysisOptions{OutputDir: outDir, Format: "csv"}); err == nil { t.Fatal("expected error for unknown format, got nil") } } diff --git a/internal/core/chi_test.go b/internal/core/chi_test.go index 48742fe..c2f5532 100644 --- a/internal/core/chi_test.go +++ b/internal/core/chi_test.go @@ -123,7 +123,8 @@ func TestChi_NoDanglingEdges(t *testing.T) { if !sigs[e.Source] { t.Errorf("dangling edge source: %s", e.Source) } - if !sigs[e.Target] { + // A target is either an in-project callable or a recorded library callee. + if _, ext := sharedChiL2.Externals[e.Target]; !sigs[e.Target] && !ext { t.Errorf("dangling edge target: %s", e.Target) } } diff --git a/internal/core/dataflow_test.go b/internal/core/dataflow_test.go new file mode 100644 index 0000000..362bf18 --- /dev/null +++ b/internal/core/dataflow_test.go @@ -0,0 +1,51 @@ +package core_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/core" + "github.com/codellm-devkit/codeanalyzer-go/internal/options" +) + +// TestAnalyze_Level3RunsDataflow: Phase 5 fills the side table at -a 3 only, +// and a non-CLI caller that leaves FieldDepth 0 still gets k_limit 3. +func TestAnalyze_Level3RunsDataflow(t *testing.T) { + opts := options.AnalysisOptions{ + InputPath: greeterDir(t), Level: options.LevelDataflow, SkipTests: true, CacheDir: t.TempDir(), + } + a := core.New(opts) + app, err := a.Analyze() + if err != nil { + t.Fatal(err) + } + if len(a.L3()) == 0 { + t.Fatal("-a 3 produced no dataflow results") + } + data, err := core.RenderJSON(app, a.L3(), opts) + if err != nil { + t.Fatal(err) + } + var v struct { + KLimit int `json:"k_limit"` + } + if err := json.Unmarshal(data, &v); err != nil { + t.Fatal(err) + } + if v.KLimit != 3 { + t.Errorf("k_limit = %d, want the default 3", v.KLimit) + } + if !strings.Contains(string(data), `"cfg":[`) { + t.Error("-a 3 JSON has no cfg") + } + + opts.Level = options.LevelCallGraph + a2 := core.New(opts) + if _, err := a2.Analyze(); err != nil { + t.Fatal(err) + } + if a2.L3() != nil { + t.Error("-a 2 ran dataflow") + } +} diff --git a/internal/core/testsetup_test.go b/internal/core/testsetup_test.go index 7446447..db04684 100644 --- a/internal/core/testsetup_test.go +++ b/internal/core/testsetup_test.go @@ -19,8 +19,8 @@ var ( sharedGreeterL2 *schema.GoApplication // greeter, full call-graph sharedMultipackageL1 *schema.GoApplication // multipackage, symbol-table only sharedMultipackageL2 *schema.GoApplication // multipackage, full call-graph - sharedGenericsL1 *schema.GoApplication // generics, symbol-table only - sharedChiL2 *schema.GoApplication // chi (external dep), full call-graph + sharedGenericsL1 *schema.GoApplication // generics, symbol-table only + sharedChiL2 *schema.GoApplication // chi (external dep), full call-graph ) func TestMain(m *testing.M) { @@ -76,7 +76,7 @@ func runTestMain(m *testing.M) int { fmt.Fprintf(os.Stderr, "testsetup %s: Analyze: %v\n", f.name, err) return 1 } - if err := core.WriteOutput(app, opts); err != nil { + if err := core.WriteOutput(app, nil, opts); err != nil { fmt.Fprintf(os.Stderr, "testsetup %s: WriteOutput: %v\n", f.name, err) return 1 } @@ -97,4 +97,3 @@ func testdataDir() string { abs, _ := filepath.Abs(filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata")) return abs } - diff --git a/internal/dataflow/accesspath.go b/internal/dataflow/accesspath.go new file mode 100644 index 0000000..c860636 --- /dev/null +++ b/internal/dataflow/accesspath.go @@ -0,0 +1,301 @@ +package dataflow + +import ( + "go/ast" + "go/types" + "strings" +) + +// Path is a k-limited access path base(.field|[*])* (spec §4.5). +// +// - Base is the path's identity: the *types.Var of a local, parameter, +// receiver, named result, capture or package-level variable. Shadowed +// variables are distinct bases even when spelled alike. nil for a +// pseudo-base such as ":12:8", and for an unresolved identifier +// (a name go/types recorded no object for, in a package with type +// errors); either's identity is its spelling. +// - Spell is the emitted spelling: the source name for a local +// ("p.f.g.*", "ch[*]"), "pkgpath::Name" for a package-level variable +// ("example.com/lib::V.F"), or the pseudo-base itself. +// +// Steps after the base are ".name" (a field), "[*]" (any index / map key / +// channel element) and the wildcard ".*" (anything below: a dereference, a +// k-truncated tail, or the contents a callee may write). A ".*" is always the +// last step. Path is comparable, so it can be a map key. +type Path struct { + Base types.Object + Spell string +} + +// Covers reports whether p covers q: the same base, and p's step list a +// prefix of q's, except that a final wildcard step of p (".*" or "[*]") +// matches any non-empty remaining suffix of q. Steps compare as whole +// segments, so "p.f" does not cover "p.fg"; a wildcard stands for at least +// one step, so "x.f.*" does not cover "x.f"; a non-final "[*]" is a plain +// segment ("m[*].f" does not cover "m[*].g"). +// +// Task 10 kills with it (a strong def of p kills every reaching def d with +// p.Covers(d)) and matches uses with it in both directions (a use u links +// from d when d.Covers(u) || u.Covers(d)). +func (p Path) Covers(q Path) bool { + if p.Base != q.Base { + return false + } + ph, ps := splitSpell(p.Spell) + qh, qs := splitSpell(q.Spell) + if ph != qh || len(ps) > len(qs) { + return false + } + for i, s := range ps { + if i == len(ps)-1 && isWildcard(s) { + return true // len(qs) > i: a non-empty suffix remains + } + if s != qs[i] { + return false + } + } + return true +} + +func isWildcard(step string) bool { return step == ".*" || step == "[*]" } + +// splitSpell splits a spelling into its base spelling and its steps. A +// package-level base "pkgpath::Name" may contain dots before the "::"; a +// pseudo-base ("<…>") has no steps. +func splitSpell(s string) (string, []string) { + if strings.HasPrefix(s, "<") { + return s, nil + } + start := 0 + if i := strings.Index(s, "::"); i >= 0 { + start = i + 2 + } + end := strings.IndexAny(s[start:], ".[") + if end < 0 { + return s, nil + } + head, rest := s[:start+end], s[start+end:] + var steps []string + for rest != "" { + n := 1 + strings.IndexAny(rest[1:], ".[") + if rest[0] == '[' { + n = strings.IndexByte(rest, ']') + 1 + } else if n == 0 { + n = len(rest) + } + steps, rest = append(steps, rest[:n]), rest[n:] + } + return head, steps +} + +// pathB is a path under construction: a base and its untruncated steps. +type pathB struct { + base types.Object + head string + steps []string +} + +// with returns b extended by one step; nothing is appended after a ".*". +func (b pathB) with(step string) pathB { + if n := len(b.steps); n > 0 && b.steps[n-1] == ".*" { + return b + } + steps := make([]string, len(b.steps), len(b.steps)+1) + copy(steps, b.steps) + b.steps = append(steps, step) + return b +} + +// weak reports whether a write to b may not overwrite all of it: an index +// step (one element of many) or a wildcard (a dereference, a truncated tail). +func (b pathB) weak(k int) bool { + if len(b.steps) > max(k, 1) { + return true + } + for _, s := range b.steps { + if isWildcard(s) { + return true + } + } + return false +} + +// limit spells b with at most k steps after the base: a longer path keeps its +// first k-1 steps and ends in the ".*" wildcard as its k-th (k=3: +// p.f.g.h.i → p.f.g.*; k=1: p.f.g → p.*). k < 1 is treated as 1. +func (b pathB) limit(k int) Path { + k = max(k, 1) + steps := b.steps + if len(steps) > k { + steps = append(steps[:k-1:k-1], ".*") + } + return Path{Base: b.base, Spell: b.head + strings.Join(steps, "")} +} + +// PathOf returns the k-limited access path e names, and false when e is not a +// path (a literal, a call, a constant, a function, a method value, the blank +// identifier, an expression rooted in a call). info must be non-nil. +// +// - an identifier naming a variable (not a struct field) is a base; +// - a field selector x.f appends ".f", spelling out implicit embedded +// fields (e.f promoted from an embedded T is "e.T.f"); a selector through +// a pointer reads as Go spells it: p.f and (*p).f are both "p.f". When +// go/types recorded no selection (x has an invalid type, in a package with +// type errors), x.f is still x's path plus ".f"; +// - an identifier go/types recorded nothing for (an undeclared name, in a +// package with type errors) is an unresolved base: nil Base, its name as +// spelling; +// - a qualified identifier pkg.V is that package-level variable's base; +// - x[i] (slice, array, map, string, array pointer) appends "[*]"; a +// generic instantiation is not a path; +// - *x appends the ".*" wildcard; +// - parentheses are transparent. +func PathOf(e ast.Expr, info *types.Info, k int) (Path, bool) { + b, ok := pathOf(e, info) + if !ok { + return Path{}, false + } + return b.limit(k), true +} + +func pathOf(e ast.Expr, info *types.Info) (pathB, bool) { + if info == nil { + return pathB{}, false + } + switch e := ast.Unparen(e).(type) { + case *ast.Ident: + return baseOf(e, info) + + case *ast.SelectorExpr: + if b, ok, qualified := qualifiedBase(e, info); qualified { + return b, ok + } + // No selection means type errors left X without a usable type: read + // .Sel as a field step anyway, so a write to it is not lost. + sel := info.Selections[e] + if sel != nil && sel.Kind() != types.FieldVal { + return pathB{}, false + } + x := ast.Unparen(e.X) + if st, ok := x.(*ast.StarExpr); ok { + x = st.X // (*p).f is p.f + } + b, ok := pathOf(x, info) + if !ok { + return pathB{}, false + } + if sel != nil { + for _, name := range embeddedPath(sel) { + b = b.with("." + name) + } + } + return b.with("." + e.Sel.Name), true + + case *ast.IndexExpr: + if isType(e.Index, info) { + return pathB{}, false + } + x := ast.Unparen(e.X) + if st, ok := x.(*ast.StarExpr); ok { + x = st.X // (*a)[i] is a[i] + } + b, ok := pathOf(x, info) + if !ok { + return pathB{}, false + } + return b.with("[*]"), true + + case *ast.StarExpr: + b, ok := pathOf(e.X, info) + if !ok { + return pathB{}, false + } + return b.with(".*"), true + } + return pathB{}, false +} + +// baseOf is the base path of an identifier naming a variable, or the +// unresolved base (nil, spelled by name) of an identifier go/types recorded +// neither a definition nor a use of. +func baseOf(id *ast.Ident, info *types.Info) (pathB, bool) { + if id.Name == "_" { + return pathB{}, false + } + if unresolved(id, info) { + return pathB{head: id.Name}, true + } + v, ok := info.ObjectOf(id).(*types.Var) + if !ok || v.IsField() { + return pathB{}, false + } + return pathB{base: v, head: baseSpell(v)}, true +} + +// qualifiedBase is the base of qualified identifier pkg.Name (qualified +// reports whether e is one). A member of a package that failed to import is +// unresolved: nil base, spelled "pkgpath::Name" like a package-level variable. +func qualifiedBase(e *ast.SelectorExpr, info *types.Info) (b pathB, ok, qualified bool) { + x, isIdent := e.X.(*ast.Ident) + if !isIdent { + return pathB{}, false, false + } + pn, isPkg := info.Uses[x].(*types.PkgName) + if !isPkg { + return pathB{}, false, false + } + if unresolved(e.Sel, info) { + return pathB{head: pn.Imported().Path() + "::" + e.Sel.Name}, true, true + } + b, ok = baseOf(e.Sel, info) + return b, ok, true +} + +// unresolved reports whether go/types recorded nothing for id: not in Defs +// (a type switch's symbolic variable is there, with a nil object) and not in +// Uses. Only a package with type errors has such an identifier. +func unresolved(id *ast.Ident, info *types.Info) bool { + if _, ok := info.Defs[id]; ok { + return false + } + _, ok := info.Uses[id] + return !ok +} + +// baseSpell spells a base variable: "pkgpath::Name" at package level, else +// its name. +func baseSpell(v types.Object) string { + if isPkgVar(v) { + return v.Pkg().Path() + "::" + v.Name() + } + return v.Name() +} + +// isPkgVar reports whether obj is a package-level variable. It checks +// membership of the package scope by identity, which holds for packages +// type-checked from source and imported from export data alike. +func isPkgVar(obj types.Object) bool { + v, ok := obj.(*types.Var) + return ok && !v.IsField() && v.Pkg() != nil && v.Pkg().Scope().Lookup(v.Name()) == v +} + +// embeddedPath returns the names of the implicit embedded fields a field +// selection walks before its final field. +func embeddedPath(sel *types.Selection) []string { + idx := sel.Index() + var out []string + t := sel.Recv() + for _, i := range idx[:len(idx)-1] { + if p, ok := t.Underlying().(*types.Pointer); ok { + t = p.Elem() + } + st, ok := t.Underlying().(*types.Struct) + if !ok { + return out + } + f := st.Field(i) + out = append(out, f.Name()) + t = f.Type() + } + return out +} diff --git a/internal/dataflow/accesspath_test.go b/internal/dataflow/accesspath_test.go new file mode 100644 index 0000000..8efee74 --- /dev/null +++ b/internal/dataflow/accesspath_test.go @@ -0,0 +1,385 @@ +package dataflow_test + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" + v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" +) + +// ---- a hermetic two-package source for access-path and def/use tests ---- + +const duLibSrc = `package lib + +var V struct{ F int } + +var W int + +const C = 1 + +func F() {} +` + +const duSrc = `package p + +import "example.com/lib" + +var G int + +type H struct{ i int } +type Gs struct { + h H + x int +} +type Fs struct{ g Gs } +type P struct{ f Fs } + +type T struct { + f int + s []int +} + +type E2 struct{ *T } + +func (t *T) Ptr() {} +func (t T) Val() {} +func (t *T) M() *T { return t } + +type Mu struct{} + +func (m *Mu) Unlock() {} + +func sink(a ...any) {} +func id(x int) int { return x } +func pid(p *T) *T { return p } +func mut(p *T, s []int, t T, m map[int]int, i int, fn func(), e any, ch chan int) {} +func mut2(s []int, x int) {} + +func Exprs(p *P, m map[int]int, q *T, a []T, k, i int, x T, e E2) { + _ = p.f.g.h.i + _ = p.f.g.x + _ = p.f.g + _ = m[k] + _ = *q + _ = (*q).f + _ = a[i].f + _ = lib.V.F + _ = lib.W + _ = G + _ = (x) + _ = lib.C + _ = lib.F + _ = id(k) + _ = 42 + _ = x.Val + _ = e.f + _ = **(&q) +} + +func Assign(x T, m map[int]int, q *int, k, v int) { + x.f = v + m[k] = v + *q = v + x.f += 1 + k++ + m[k]++ + var z, w int = k, v + _ = z + w +} + +func Calls(req T, s []int, p *T, t T, m map[int]int) { + mut(&req, s, t, m, 1, nil, p, nil) + t.Ptr() + p.Ptr() + t.Val() + n := len(s) + s = append(s, n) + copy(s, s) + _ = T(t) + r := pid(&req) + _ = r +} + +func Chans(ch chan int, out chan<- int, v int) { + for x := range ch { + out <- x + } + select { + case y, ok := <-ch: + _, _ = y, ok + case out <- v: + case <-ch: + } + sink(<-ch) +} + +func Defers(mu *Mu, x int, s []int) (n int, err error) { + defer mu.Unlock() + defer func() { + if x > 0 { + err = nil + } + }() + defer mut2(s, x) + n = x + return +} + +func Goes(s []int, x int) { + go mut2(s, id(x)) + go func() { sink(x) }() +} + +func (r *T) Entry(a int, _ int, b []int) (n int, err error) { + G = a + _ = lib.V.F + lib.W + h := func() int { + k := 1 + return func() int { return a + k + r.f }() + } + _ = h + return +} + +func NamedRet(x int) (n int) { + return x +} + +func Mixed(a int) int { + b, a := 1, 2 + return a + b +} + +func src() int { return 0 } + +func ForPost(n int) int { + x := 1 + for i2 := 0; i2 < 3; i2, x = i2+1, src() { + } + return x +} + +func ForInit(n int) { + for i, v := 0, 0; i < n; i++ { + sink(v) + v = src() + } +} + +func RangeAssign(a []int) int { + last := 0 + for _, last = range a { + } + return last +} + +type Hm map[string]string + +func (h Hm) Set(k, v string) { h[k] = v } + +type Writer interface{ Write(p []byte) } + +func Recv(w Writer, h Hm, t T, data []byte) { + w.Write(data) + h.Set("a", "b") + t.Val() +} + +func IfShadow(x int) int { + if x := id(x); x > 0 { + return x + } + return x +} +` + +type mapImporter map[string]*types.Package + +func (m mapImporter) Import(path string) (*types.Package, error) { return m[path], nil } + +type duFixture struct { + fset *token.FileSet + file *ast.File + info *types.Info + decls map[string]*ast.FuncDecl +} + +func newInfo() *types.Info { + return &types.Info{ + Types: map[ast.Expr]types.TypeAndValue{}, + Defs: map[*ast.Ident]types.Object{}, + Uses: map[*ast.Ident]types.Object{}, + Implicits: map[ast.Node]types.Object{}, + Selections: map[*ast.SelectorExpr]*types.Selection{}, + Scopes: map[ast.Node]*types.Scope{}, + } +} + +func loadDU(t *testing.T) *duFixture { + t.Helper() + fset := token.NewFileSet() + lf, err := parser.ParseFile(fset, "lib.go", duLibSrc, 0) + if err != nil { + t.Fatal(err) + } + lib, err := (&types.Config{}).Check("example.com/lib", fset, []*ast.File{lf}, newInfo()) + if err != nil { + t.Fatal(err) + } + pf, err := parser.ParseFile(fset, "p.go", duSrc, 0) + if err != nil { + t.Fatal(err) + } + info := newInfo() + conf := &types.Config{Importer: mapImporter{"example.com/lib": lib}} + if _, err := conf.Check("example.com/p", fset, []*ast.File{pf}, info); err != nil { + t.Fatal(err) + } + fx := &duFixture{fset: fset, file: pf, info: info, decls: map[string]*ast.FuncDecl{}} + for _, d := range pf.Decls { + if fd, ok := d.(*ast.FuncDecl); ok { + fx.decls[fd.Name.Name] = fd + } + } + return fx +} + +// expr finds the first expression in fn whose source text is text. +func (fx *duFixture) expr(t *testing.T, fn, text string) ast.Expr { + t.Helper() + var found ast.Expr + ast.Inspect(fx.decls[fn].Body, func(n ast.Node) bool { + if e, ok := n.(ast.Expr); ok && found == nil && types.ExprString(e) == text { + found = e + } + return found == nil + }) + if found == nil { + t.Fatalf("%s: no expression %q", fn, text) + } + return found +} + +// keyAt is the local key of the first occurrence of sub in duSrc, plus +// delta bytes. +func (fx *duFixture) keyAt(t *testing.T, sub string, delta int) string { + t.Helper() + off := strings.Index(duSrc, sub) + if off < 0 { + t.Fatalf("no %q in duSrc", sub) + } + p := fx.fset.Position(fx.fset.File(fx.file.Pos()).Pos(off + delta)) + return v2.LocalID(p.Line, p.Column) +} + +func TestPathOf_KLimit(t *testing.T) { + fx := loadDU(t) + pObj := fx.info.Defs[fx.decls["Exprs"].Type.Params.List[0].Names[0]] + cases := []struct { + text string + k int + want string + }{ + {"p.f.g.h.i", 3, "p.f.g.*"}, + {"p.f.g.h.i", 4, "p.f.g.h.i"}, + {"p.f.g.h.i", 2, "p.f.*"}, + {"p.f.g.x", 3, "p.f.g.x"}, // exactly k steps: not truncated + {"p.f.g", 1, "p.*"}, + {"p.f.g", 3, "p.f.g"}, + } + for _, c := range cases { + p, ok := dataflow.PathOf(fx.expr(t, "Exprs", c.text), fx.info, c.k) + if !ok || p.Spell != c.want { + t.Errorf("PathOf(%s, k=%d) = %q, %v; want %q", c.text, c.k, p.Spell, ok, c.want) + } + if p.Base != pObj { + t.Errorf("PathOf(%s): Base = %v, want param p", c.text, p.Base) + } + } +} + +func TestPathOf_IndexAndDeref(t *testing.T) { + fx := loadDU(t) + paths := []struct{ text, want string }{ + {"m[k]", "m[*]"}, + {"*q", "q.*"}, + {"(*q).f", "q.f"}, // spelled as Go's auto-deref q.f + {"**(&q)", ""}, // &q is not a path + {"a[i].f", "a[*].f"}, + {"lib.V.F", "example.com/lib::V.F"}, + {"lib.W", "example.com/lib::W"}, + {"G", "example.com/p::G"}, + {"(x)", "x"}, + {"e.f", "e.T.f"}, // the implicit embedded field is spelled out + } + for _, c := range paths { + if c.want == "" { + continue + } + p, ok := dataflow.PathOf(fx.expr(t, "Exprs", c.text), fx.info, 3) + if !ok || p.Spell != c.want { + t.Errorf("PathOf(%s) = %q, %v; want %q", c.text, p.Spell, ok, c.want) + } + } + for _, text := range []string{"lib.C", "lib.F", "id(k)", "42", "x.Val", "**(&q)"} { + if p, ok := dataflow.PathOf(fx.expr(t, "Exprs", text), fx.info, 3); ok { + t.Errorf("PathOf(%s) = %q, want not a path", text, p.Spell) + } + } + blank := &ast.Ident{Name: "_"} + if _, ok := dataflow.PathOf(blank, fx.info, 3); ok { + t.Errorf("PathOf(_) is a path") + } +} + +func TestPath_Covers(t *testing.T) { + x := types.NewVar(token.NoPos, nil, "x", nil) + x2 := types.NewVar(token.NoPos, nil, "x", nil) // a shadowing x + m := types.NewVar(token.NoPos, nil, "m", nil) + pkg := types.NewPackage("example.com/p", "p") + g := types.NewVar(token.NoPos, pkg, "V", types.Typ[types.Int]) + pkg.Scope().Insert(g) // package-level: spelled pkgpath::V + P := func(b types.Object, s string) dataflow.Path { return dataflow.Path{Base: b, Spell: s} } + cases := []struct { + p, q dataflow.Path + want bool + }{ + {P(x, "x"), P(x, "x"), true}, // exact + {P(x, "x"), P(x, "x.f"), true}, // prefix + {P(x, "x.f"), P(x, "x"), false}, // longer never covers shorter + {P(x, "x.f"), P(x, "x.f.g"), true}, // prefix + {P(x, "x.f"), P(x, "x.g"), false}, // sibling + {P(x, "x.f"), P(x, "x.fg"), false}, // string prefix, not segment prefix + {P(x, "x.f.*"), P(x, "x.f.g"), true}, // wildcard + {P(x, "x.f.*"), P(x, "x.f.g.h"), true}, // wildcard, any remaining suffix + {P(x, "x.f.*"), P(x, "x.f"), false}, // a wildcard stands for ≥1 step + {P(x, "x.f"), P(x, "x.*"), false}, // kill direction: a field def does not kill x.* + {P(x, "x.f.g"), P(x, "x.f.*"), false}, // kill direction: nor a deeper one x.f.* + {P(x, "x.*"), P(x, "x[*]"), true}, // wildcard + {P(x, "x.*"), P(x, "x[*].f.g"), true}, // wildcard + {P(m, "m[*]"), P(m, "m[*].f"), true}, // prefix + {P(m, "m[*]"), P(m, "m[*]"), true}, // [*] is every index + {P(m, "m[*]"), P(m, "m.*"), true}, // a terminal [*] matches any remaining suffix + {P(m, "m[*].f"), P(m, "m[*].g"), false}, // sibling under an element: a non-terminal [*] is a plain segment + {P(m, "m[*].f"), P(m, "m[*].f.g"), true}, + {P(m, "m.f"), P(m, "m[*]"), false}, + {P(m, "m"), P(m, "m[*]"), true}, // prefix + {P(m, "m[*]"), P(m, "m"), false}, // longer never covers shorter + {P(x, "x"), P(x2, "x"), false}, // shadowed: different base + {P(x, "x.f"), P(x2, "x.f"), false}, // shadowed: different base + {P(nil, ":1:2"), P(nil, ":1:2"), true}, // pseudo base + {P(nil, ":1:2"), P(nil, ":1:20"), false}, + {P(g, "example.com/p::V"), P(g, "example.com/p::V.f"), true}, + {P(g, "example.com/p::V.f"), P(g, "example.com/p::V.g"), false}, + } + for _, c := range cases { + if got := c.p.Covers(c.q); got != c.want { + t.Errorf("%q.Covers(%q) = %v, want %v", c.p.Spell, c.q.Spell, got, c.want) + } + } +} diff --git a/internal/dataflow/bodynodes.go b/internal/dataflow/bodynodes.go new file mode 100644 index 0000000..cdaafca --- /dev/null +++ b/internal/dataflow/bodynodes.go @@ -0,0 +1,311 @@ +package dataflow + +import ( + "go/ast" + "go/token" + "sort" + "strconv" + + v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" +) + +// BuildNodes maps a callable body onto its L3 body nodes (spec §4.2) and +// returns them with a map from AST node to node key. +// +// Node rules: +// - if → branch; for / for-range → loop; switch / type switch / select → +// switch; return → return; every other statement → statement. The init +// statement of if/for/switch, the post statement of a 3-clause for, a type +// switch's guard and a comm clause's send/receive fold into their header or +// clause-head node; they get no node of their own. +// - each case / default / comm clause head is a statement node keyed at the +// clause's Pos(); its span ends at the clause's colon. +// - a bare call statement (ExprStmt whose expression is an L1 call) and a +// `go` statement are the call node: kind call, keyed with the L1 call key. +// Calls nested in the call (a `go` statement's function value and +// arguments, evaluated at the statement) are satellites of that node. +// - `defer f(x)` is two nodes (§4.3.1): a statement node at the `defer` +// keyword and the L1 call node with Deferred set. Calls in the deferred +// call's receiver and arguments run at registration and are satellites of +// the defer statement. +// - every other L1 call is a satellite call node whose Parent is the key of +// the innermost enclosing statement / header / clause-head node. +// - blocks and labels get no node (a label's inner statement does); an empty +// statement gets one only as a label target. Function literal bodies are +// skipped: closures are their own callables. +// +// calls holds the L1 call-node keys in L1 recording order (v2.CallBodyKeys +// over the callable's call sites). They are paired with the body's call +// expressions by recording index (ruling R5), never by position: under a +// //line directive the recorded line:col does not name the call's token.Pos. +// The caller checks len(calls) == RecordedCallCount(body); surplus keys are +// ignored and calls past the end of the list get no call node. +// +// Keys: call nodes take their L1 key. Statement nodes take +// v2.LocalID(line, col) of their anchor; a statement starting with a call +// (`f().x = v`) would collide with the L1 call key, so it takes the next free +// "/n" suffix instead and the L1 key is never moved. +// +// Order (deterministic): @entry first, then the real nodes in source order +// (by Pos; co-positioned nodes in traversal order — a statement before its +// calls, an outer call before the calls nested in it), then @exit last. The +// synthetic nodes have NoPos and a nil Stmt. A nil body yields an empty slice +// and an empty map (no entry/exit: a bodyless callable has no flow). +// +// The returned map keys every node's Stmt to its key. It also maps the AST +// nodes folded into a node to that node's key: a bare or `go` call's +// *ast.CallExpr, an if/for/switch init, a for post, a type switch guard, a comm +// clause's statement, and a *ast.LabeledStmt (to its inner statement's node). +func BuildNodes(fset *token.FileSet, body *ast.BlockStmt, calls []string) ([]Node, map[ast.Node]string) { + if body == nil { + return []Node{}, map[ast.Node]string{} + } + b := &builder{ + fset: fset, + callKey: assignCallKeys(body, calls), + taken: make(map[string]bool, len(calls)), + keyOf: map[ast.Node]string{}, + } + for _, k := range calls { + b.taken[k] = true + } + b.stmtList(body.List) + + sort.SliceStable(b.nodes, func(i, j int) bool { return b.nodes[i].Pos < b.nodes[j].Pos }) + out := make([]Node, 0, len(b.nodes)+2) + out = append(out, Node{Key: EntryKey, Kind: KindEntry}) + for _, n := range b.nodes { + p, e := fset.Position(n.Pos), fset.Position(n.End) + n.StartLC, n.EndLC = [2]int{p.Line, p.Column}, [2]int{e.Line, e.Column} + out = append(out, n) + } + out = append(out, Node{Key: ExitKey, Kind: KindExit}) + return out, b.keyOf +} + +// recordedCalls lists body's call expressions in the L1 recording order of +// buildCallSites (internal/syntactic_analysis): pre-order — an outer call +// before the calls nested in it, a `go` / `defer` statement's call before the +// calls in its function value and arguments — and no function literal bodies. +// The two walks must stay identical: keys pair with calls by index (R5). +func recordedCalls(body *ast.BlockStmt) []*ast.CallExpr { + var out []*ast.CallExpr + ast.Inspect(body, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncLit: + return false + case *ast.CallExpr: + out = append(out, n) + } + return true + }) + return out +} + +// RecordedCallCount is the number of L1 call sites body records: the length +// the calls argument of BuildNodes must have. +func RecordedCallCount(body *ast.BlockStmt) int { + if body == nil { + return 0 + } + return len(recordedCalls(body)) +} + +// assignCallKeys pairs the i-th recorded call with the i-th L1 key. +func assignCallKeys(body *ast.BlockStmt, calls []string) map[*ast.CallExpr]string { + out := map[*ast.CallExpr]string{} + for i, call := range recordedCalls(body) { + if i < len(calls) { + out[call] = calls[i] + } + } + return out +} + +type builder struct { + fset *token.FileSet + callKey map[*ast.CallExpr]string + taken map[string]bool + keyOf map[ast.Node]string + nodes []Node +} + +func (b *builder) stmtList(list []ast.Stmt) { + for _, s := range list { + b.stmt(s) + } +} + +func (b *builder) stmt(s ast.Stmt) { + switch s := s.(type) { + case *ast.BlockStmt: + b.stmtList(s.List) + + case *ast.LabeledStmt: + if e, ok := s.Stmt.(*ast.EmptyStmt); ok { + b.add(KindStatement, e, s.Stmt.Pos(), s.Stmt.End()) + } else { + b.stmt(s.Stmt) + } + if k, ok := b.keyOf[s.Stmt]; ok { + b.keyOf[s] = k + } + + case *ast.EmptyStmt: + // No node: gofmt'd code has an empty statement only as a label target. + + case *ast.ExprStmt: + if call, ok := ast.Unparen(s.X).(*ast.CallExpr); ok { + if key, ok := b.callKey[call]; ok { + b.addCall(key, s, call) + b.satellites(key, s.X, call) + return + } + } + b.simple(s) + + case *ast.GoStmt: + if key, ok := b.callKey[s.Call]; ok { + b.addCall(key, s, s.Call) + b.satellites(key, s.Call, s.Call) + return + } + b.simple(s) + + case *ast.DeferStmt: + reg := b.add(KindStatement, s, s.Pos(), s.End()) + if key, ok := b.callKey[s.Call]; ok { + b.nodes = append(b.nodes, Node{Key: key, Kind: KindCall, Pos: s.Call.Pos(), End: s.Call.End(), Stmt: s.Call, Deferred: true}) + b.keyOf[s.Call] = key + } + b.satellites(reg, s.Call.Fun, nil) + for _, a := range s.Call.Args { + b.satellites(reg, a, nil) + } + + case *ast.IfStmt: + k := b.add(KindBranch, s, s.Pos(), s.End()) + b.folded(k, s.Init) + b.satellites(k, s.Cond, nil) + b.stmt(s.Body) + if s.Else != nil { + b.stmt(s.Else) + } + + case *ast.ForStmt: + k := b.add(KindLoop, s, s.Pos(), s.End()) + b.folded(k, s.Init) + b.satellites(k, s.Cond, nil) + b.folded(k, s.Post) + b.stmt(s.Body) + + case *ast.RangeStmt: + k := b.add(KindLoop, s, s.Pos(), s.End()) + b.satellites(k, s.Key, nil) + b.satellites(k, s.Value, nil) + b.satellites(k, s.X, nil) + b.stmt(s.Body) + + case *ast.SwitchStmt: + k := b.add(KindSwitch, s, s.Pos(), s.End()) + b.folded(k, s.Init) + b.satellites(k, s.Tag, nil) + b.caseClauses(s.Body) + + case *ast.TypeSwitchStmt: + k := b.add(KindSwitch, s, s.Pos(), s.End()) + b.folded(k, s.Init) + b.folded(k, s.Assign) + b.caseClauses(s.Body) + + case *ast.SelectStmt: + b.add(KindSwitch, s, s.Pos(), s.End()) + for _, c := range s.Body.List { + cc := c.(*ast.CommClause) + k := b.add(KindStatement, cc, cc.Pos(), cc.Colon+1) + b.folded(k, cc.Comm) + b.stmtList(cc.Body) + } + + case *ast.ReturnStmt: + k := b.add(KindReturn, s, s.Pos(), s.End()) + b.satellites(k, s, nil) + + default: + // assign, :=, var decl, inc/dec, send, non-call expression statement, + // break / continue / goto / fallthrough. + b.simple(s) + } +} + +func (b *builder) caseClauses(body *ast.BlockStmt) { + for _, c := range body.List { + cc := c.(*ast.CaseClause) + k := b.add(KindStatement, cc, cc.Pos(), cc.Colon+1) + for _, e := range cc.List { + b.satellites(k, e, nil) + } + b.stmtList(cc.Body) + } +} + +// simple adds a statement node for a statement with no nested statements and +// makes every call in it a satellite. +func (b *builder) simple(s ast.Stmt) { + k := b.add(KindStatement, s, s.Pos(), s.End()) + b.satellites(k, s, nil) +} + +// folded attributes a header's init / post / guard statement to the header +// node k: no node of its own, its calls are k's satellites. +func (b *builder) folded(k string, s ast.Stmt) { + if s == nil { + return + } + b.keyOf[s] = k + b.satellites(k, s, nil) +} + +// add appends a non-call node anchored at pos and returns its key: the +// position's LocalID, or the next free "/n" suffix when an L1 call (or an +// earlier node) already holds it. +func (b *builder) add(kind NodeKind, n ast.Node, pos, end token.Pos) string { + p := b.fset.Position(pos) + base := v2.LocalID(p.Line, p.Column) + key := base + for i := 2; b.taken[key]; i++ { + key = base + "/" + strconv.Itoa(i) + } + b.taken[key] = true + b.nodes = append(b.nodes, Node{Key: key, Kind: kind, Pos: pos, End: end, Stmt: n}) + b.keyOf[n] = key + return key +} + +// addCall appends a statement that is its call node (a bare call or `go`). +func (b *builder) addCall(key string, s ast.Stmt, call *ast.CallExpr) { + b.nodes = append(b.nodes, Node{Key: key, Kind: KindCall, Pos: s.Pos(), End: s.End(), Stmt: s}) + b.keyOf[s] = key + b.keyOf[call] = key +} + +// satellites adds a call node with Parent = parent for every L1 call inside n, +// except skip (the statement's own call node). Function literal bodies are +// not entered. +func (b *builder) satellites(parent string, n ast.Node, skip *ast.CallExpr) { + if n == nil { + return + } + ast.Inspect(n, func(x ast.Node) bool { + switch x := x.(type) { + case *ast.FuncLit: + return false + case *ast.CallExpr: + if key, ok := b.callKey[x]; ok && x != skip { + b.nodes = append(b.nodes, Node{Key: key, Kind: KindCall, Pos: x.Pos(), End: x.End(), Stmt: x, Parent: parent}) + b.keyOf[x] = key + } + } + return true + }) +} diff --git a/internal/dataflow/bodynodes_test.go b/internal/dataflow/bodynodes_test.go new file mode 100644 index 0000000..e84afb4 --- /dev/null +++ b/internal/dataflow/bodynodes_test.go @@ -0,0 +1,372 @@ +package dataflow_test + +import ( + "go/ast" + "go/token" + "go/types" + "path/filepath" + "runtime" + "sync" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" + "github.com/codellm-devkit/codeanalyzer-go/internal/schema" + v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" + "github.com/codellm-devkit/codeanalyzer-go/internal/syntactic_analysis" +) + +// flowFixture is the testdata/dataflow module loaded once through the real +// symbol-table builder, so call keys come from the L1 call-site list. +type flowFixture struct { + fset *token.FileSet + info *types.Info + decls map[string]*ast.FuncDecl + sites map[string][]schema.GoCallsite +} + +var ( + fixtureOnce sync.Once + fixture *flowFixture + fixtureErr error +) + +func loadFlow(t *testing.T) *flowFixture { + t.Helper() + fixtureOnce.Do(func() { + _, thisFile, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata", "dataflow") + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + fixtureErr = err + return + } + fx := &flowFixture{fset: b.Fset(), decls: map[string]*ast.FuncDecl{}, sites: map[string][]schema.GoCallsite{}} + for _, fn := range st[filepath.Join("flow", "flow.go")].Functions { + fx.sites[fn.Name] = fn.CallSites + } + pkg := b.Pkgs()["example.com/dataflow/flow"] + fx.info = pkg.TypesInfo + for _, f := range pkg.Syntax { + for _, d := range f.Decls { + if fd, ok := d.(*ast.FuncDecl); ok && fd.Recv == nil { + fx.decls[fd.Name.Name] = fd + } + } + } + fixture = fx + }) + if fixtureErr != nil { + t.Fatalf("build fixture: %v", fixtureErr) + } + return fixture +} + +// callKeys builds the L1 call-node keys exactly as the driver does: keys from +// v2.CallBodyKeys over the call sites' [line, col], in recording order. +// BuildNodes pairs them with the AST calls by index (ruling R5), never by +// rebuilding a token.Pos. +func (fx *flowFixture) callKeys(t *testing.T, fn string) []string { + t.Helper() + sites := fx.sites[fn] + pos := make([][2]int, len(sites)) + for i, cs := range sites { + pos[i] = [2]int{cs.StartLine, cs.StartColumn} + } + return v2.CallBodyKeys(pos) +} + +func (fx *flowFixture) decl(t *testing.T, fn string) *ast.FuncDecl { + t.Helper() + d, ok := fx.decls[fn] + if !ok { + t.Fatalf("fixture has no function %s", fn) + } + return d +} + +// build runs BuildNodes over fn and indexes the nodes by key, failing on a +// duplicate key. +func build(t *testing.T, fn string) ([]dataflow.Node, map[string]dataflow.Node, map[ast.Node]string) { + t.Helper() + fx := loadFlow(t) + nodes, keyOf := dataflow.BuildNodes(fx.fset, fx.decl(t, fn).Body, fx.callKeys(t, fn)) + byKey := map[string]dataflow.Node{} + for _, n := range nodes { + if _, dup := byKey[n.Key]; dup { + t.Fatalf("%s: duplicate key %s", fn, n.Key) + } + byKey[n.Key] = n + } + return nodes, byKey, keyOf +} + +func wantKind(t *testing.T, fn string, byKey map[string]dataflow.Node, key string, kind dataflow.NodeKind) dataflow.Node { + t.Helper() + n, ok := byKey[key] + if !ok { + t.Fatalf("%s: no node %s; have %v", fn, key, keysOf(byKey)) + } + if n.Kind != kind { + t.Errorf("%s: node %s kind = %q, want %q", fn, key, n.Kind, kind) + } + return n +} + +func wantAbsent(t *testing.T, fn string, byKey map[string]dataflow.Node, key, why string) { + t.Helper() + if _, ok := byKey[key]; ok { + t.Errorf("%s: unexpected node %s (%s)", fn, key, why) + } +} + +func keysOf(m map[string]dataflow.Node) []string { + var ks []string + for k := range m { + ks = append(ks, k) + } + return ks +} + +func TestBodyNodes_KindsPerConstruct(t *testing.T) { + // IfElse: the init statement folds into the branch header. + _, ie, _ := build(t, "IfElse") + wantKind(t, "IfElse", ie, "16:2", dataflow.KindBranch) + wantAbsent(t, "IfElse", ie, "16:5", "if init folds into the header") + wantKind(t, "IfElse", ie, "17:3", dataflow.KindReturn) + wantKind(t, "IfElse", ie, "19:3", dataflow.KindStatement) + wantKind(t, "IfElse", ie, "21:2", dataflow.KindReturn) + if len(ie) != 6 { // @entry, if, return, assign, return, @exit + t.Errorf("IfElse: %d nodes, want 6: %v", len(ie), keysOf(ie)) + } + + // ForLoop: init and post fold into the loop header. + _, fl, _ := build(t, "ForLoop") + wantKind(t, "ForLoop", fl, "26:2", dataflow.KindStatement) + wantKind(t, "ForLoop", fl, "27:2", dataflow.KindLoop) + wantAbsent(t, "ForLoop", fl, "27:6", "for init folds into the header") + wantAbsent(t, "ForLoop", fl, "27:21", "for post folds into the header") + wantKind(t, "ForLoop", fl, "28:3", dataflow.KindStatement) + wantKind(t, "ForLoop", fl, "30:2", dataflow.KindReturn) + + _, inf, _ := build(t, "Infinite") + wantKind(t, "Infinite", inf, "35:2", dataflow.KindLoop) + + _, rg, _ := build(t, "Range") + wantKind(t, "Range", rg, "46:2", dataflow.KindLoop) + + // Switch: header, one statement per clause head, fallthrough statement. + _, sw, _ := build(t, "Switch") + wantKind(t, "Switch", sw, "55:2", dataflow.KindSwitch) + for _, k := range []string{"56:2", "59:2", "61:2", "57:3", "58:3", "60:3", "62:3"} { + wantKind(t, "Switch", sw, k, dataflow.KindStatement) + } + wantKind(t, "Switch", sw, "64:2", dataflow.KindReturn) + + // TypeSwitch: the `t := v.(type)` guard folds into the header. + _, ts, _ := build(t, "TypeSwitch") + wantKind(t, "TypeSwitch", ts, "69:2", dataflow.KindSwitch) + wantAbsent(t, "TypeSwitch", ts, "69:9", "type-switch guard folds into the header") + for _, k := range []string{"70:2", "73:2", "75:2"} { + wantKind(t, "TypeSwitch", ts, k, dataflow.KindStatement) + } + wantKind(t, "TypeSwitch", ts, "72:3", dataflow.KindReturn) + + // Select2: comm clause heads are statements; the comm stmt folds into them. + _, sel, _ := build(t, "Select2") + wantKind(t, "Select2", sel, "82:2", dataflow.KindSwitch) + wantKind(t, "Select2", sel, "83:2", dataflow.KindStatement) + wantKind(t, "Select2", sel, "85:2", dataflow.KindStatement) + wantAbsent(t, "Select2", sel, "83:7", "comm stmt folds into the clause head") + wantAbsent(t, "Select2", sel, "85:7", "comm stmt folds into the clause head") + wantKind(t, "Select2", sel, "84:3", dataflow.KindReturn) + + _, es, _ := build(t, "EmptySelect") + wantKind(t, "EmptySelect", es, "121:2", dataflow.KindSwitch) + + // Labeled: the label gets no node; its inner for does. + _, lb, _ := build(t, "Labeled") + wantAbsent(t, "Labeled", lb, "93:1", "a label is not a node") + wantKind(t, "Labeled", lb, "94:2", dataflow.KindLoop) + wantKind(t, "Labeled", lb, "97:5", dataflow.KindStatement) // continue outer + wantKind(t, "Labeled", lb, "100:5", dataflow.KindStatement) // break outer + + // GotoBack: the goto target is the label's inner statement. + _, gb, keyOf := build(t, "GotoBack") + inc := wantKind(t, "GotoBack", gb, "112:2", dataflow.KindStatement) + wantKind(t, "GotoBack", gb, "114:3", dataflow.KindStatement) + for n, k := range keyOf { + if ls, ok := n.(*ast.LabeledStmt); ok && ls.Label.Name == "loop" && k != inc.Key { + t.Errorf("GotoBack: keyOf[label loop] = %q, want %q", k, inc.Key) + } + } + + // Synthetic nodes bracket every body. + nodes, _, _ := build(t, "IfElse") + if first, last := nodes[0], nodes[len(nodes)-1]; first.Key != dataflow.EntryKey || first.Kind != dataflow.KindEntry || + last.Key != dataflow.ExitKey || last.Kind != dataflow.KindExit { + t.Errorf("IfElse: want @entry first and @exit last, got %s ... %s", first.Key, last.Key) + } + for _, n := range []dataflow.Node{nodes[0], nodes[len(nodes)-1]} { + if n.Pos != token.NoPos || n.Stmt != nil { + t.Errorf("synthetic %s has Pos %v / Stmt %T", n.Key, n.Pos, n.Stmt) + } + } + for i := 2; i < len(nodes)-1; i++ { + if nodes[i].Pos < nodes[i-1].Pos { + t.Errorf("IfElse: nodes out of source order at %s", nodes[i].Key) + } + } +} + +func TestBodyNodes_BareCallSharesKey(t *testing.T) { + fx := loadFlow(t) + l1 := map[string]bool{} + for _, k := range fx.callKeys(t, "EarlyReturnDefer") { + l1[k] = true + } + _, byKey, keyOf := build(t, "EarlyReturnDefer") + n := wantKind(t, "EarlyReturnDefer", byKey, "131:2", dataflow.KindCall) + if !l1["131:2"] { + t.Fatalf("131:2 is not an L1 call key: %v", l1) + } + if _, ok := n.Stmt.(*ast.ExprStmt); !ok { + t.Errorf("bare call node Stmt = %T, want *ast.ExprStmt", n.Stmt) + } + if n.Parent != "" || n.Deferred { + t.Errorf("bare call node Parent=%q Deferred=%v, want \"\"/false", n.Parent, n.Deferred) + } + if keyOf[n.Stmt] != "131:2" { + t.Errorf("keyOf[ExprStmt] = %q, want 131:2", keyOf[n.Stmt]) + } + + // `go work(f(x))` is the call node, keyed at the call; f(x), evaluated + // at the go statement, is its satellite. + _, sp, _ := build(t, "Spawn") + g := wantKind(t, "Spawn", sp, "202:5", dataflow.KindCall) + if _, ok := g.Stmt.(*ast.GoStmt); !ok { + t.Errorf("go node Stmt = %T, want *ast.GoStmt", g.Stmt) + } + wantAbsent(t, "Spawn", sp, "202:2", "go statement shares the call's key") + if arg := wantKind(t, "Spawn", sp, "202:10", dataflow.KindCall); arg.Parent != "202:5" { + t.Errorf("Spawn: f(x) Parent = %q, want the go node 202:5", arg.Parent) + } + + // s.A().B(): the outer call (L1 "234:2") is the statement's node; the + // co-positioned inner s.A() (L1 "234:2/2") is its satellite. + _, ch, _ := build(t, "Chain") + if outer := wantKind(t, "Chain", ch, "234:2", dataflow.KindCall); outer.Parent != "" { + t.Errorf("Chain: outer call Parent = %q, want \"\"", outer.Parent) + } + if inner := wantKind(t, "Chain", ch, "234:2/2", dataflow.KindCall); inner.Parent != "234:2" { + t.Errorf("Chain: s.A() Parent = %q, want 234:2", inner.Parent) + } + + // panic("negative") is a bare call statement too: it keeps kind call. + _, pn, _ := build(t, "Panics") + wantKind(t, "Panics", pn, "138:3", dataflow.KindCall) + if len(sp) != 4 { // @entry, go node, f(x) satellite, @exit + t.Errorf("Spawn: %d nodes, want 4: %v", len(sp), keysOf(sp)) + } +} + +func TestBodyNodes_DeferIsTwoNodes(t *testing.T) { + _, byKey, keyOf := build(t, "EarlyReturnDefer") + reg := wantKind(t, "EarlyReturnDefer", byKey, "126:2", dataflow.KindStatement) + if _, ok := reg.Stmt.(*ast.DeferStmt); !ok { + t.Errorf("registration Stmt = %T, want *ast.DeferStmt", reg.Stmt) + } + call := wantKind(t, "EarlyReturnDefer", byKey, "126:8", dataflow.KindCall) + if !call.Deferred { + t.Errorf("126:8 not marked Deferred") + } + if _, ok := call.Stmt.(*ast.CallExpr); !ok { + t.Errorf("deferred call Stmt = %T, want *ast.CallExpr", call.Stmt) + } + if keyOf[call.Stmt] != "126:8" || keyOf[reg.Stmt] != "126:2" { + t.Errorf("keyOf: defer=%q call=%q", keyOf[reg.Stmt], keyOf[call.Stmt]) + } + + // defer work(g(a)): g(a) is evaluated at registration, so it is a + // satellite of the defer statement; work(...) is the deferred call. + wantKind(t, "EarlyReturnDefer", byKey, "127:2", dataflow.KindStatement) + if w := wantKind(t, "EarlyReturnDefer", byKey, "127:8", dataflow.KindCall); !w.Deferred { + t.Errorf("127:8 not marked Deferred") + } + inner := wantKind(t, "EarlyReturnDefer", byKey, "127:13", dataflow.KindCall) + if inner.Deferred || inner.Parent != "127:2" { + t.Errorf("g(a): Deferred=%v Parent=%q, want false/127:2", inner.Deferred, inner.Parent) + } + for _, k := range []string{"126:8", "127:8"} { + if byKey[k].Parent != "" { + t.Errorf("deferred call %s Parent = %q, want \"\"", k, byKey[k].Parent) + } + } + if byKey["131:2"].Deferred { + t.Errorf("non-deferred cleanup() marked Deferred") + } +} + +func TestBodyNodes_NestedCallHasParent(t *testing.T) { + _, byKey, _ := build(t, "Nested") + wantKind(t, "Nested", byKey, "196:2", dataflow.KindStatement) + for _, k := range []string{"196:7", "196:9"} { + if n := wantKind(t, "Nested", byKey, k, dataflow.KindCall); n.Parent != "196:2" { + t.Errorf("call %s Parent = %q, want 196:2", k, n.Parent) + } + } + + // A call in an if header is a satellite of the header. + _, ie, _ := build(t, "EarlyReturnDefer") + if ie["128:2"].Kind != dataflow.KindBranch { + t.Errorf("128:2 kind = %q, want branch", ie["128:2"].Kind) + } + + // The closure body is its own callable: no nodes from inside it, and the + // call h() is a satellite of the return. + _, cl, _ := build(t, "Closure") + wantAbsent(t, "Closure", cl, "217:3", "closure body is skipped") + wantAbsent(t, "Closure", cl, "217:10", "closure body is skipped") + if n := wantKind(t, "Closure", cl, "219:9", dataflow.KindCall); n.Parent != "219:2" { + t.Errorf("h() Parent = %q, want 219:2", n.Parent) + } +} + +// TestBodyNodes_StatementCoPositionedWithCall locks the collision rule: a +// statement that starts with a call (get().f.g.x = v) leaves the L1 call key +// "l:c" untouched and takes the next free suffix. +func TestBodyNodes_StatementCoPositionedWithCall(t *testing.T) { + _, byKey, _ := build(t, "CallLHS") + call := wantKind(t, "CallLHS", byKey, "211:2", dataflow.KindCall) + stmt := wantKind(t, "CallLHS", byKey, "211:2/2", dataflow.KindStatement) + if _, ok := stmt.Stmt.(*ast.AssignStmt); !ok { + t.Errorf("211:2/2 Stmt = %T, want *ast.AssignStmt", stmt.Stmt) + } + if call.Parent != "211:2/2" { + t.Errorf("get() Parent = %q, want 211:2/2", call.Parent) + } +} + +// TestBodyNodes_CallKeysMatchL1 locks ruling R1: every L1 call key appears as +// exactly one node key, for every fixture function. +func TestBodyNodes_CallKeysMatchL1(t *testing.T) { + fx := loadFlow(t) + for fn := range fx.decls { + _, byKey, _ := build(t, fn) + for _, k := range fx.callKeys(t, fn) { + if n, ok := byKey[k]; !ok || n.Kind != dataflow.KindCall { + t.Errorf("%s: L1 call key %s has no call node", fn, k) + } + } + } +} + +func TestBodyNodes_BodylessCallableEmitsNothing(t *testing.T) { + nodes, keyOf := dataflow.BuildNodes(token.NewFileSet(), nil, nil) + if len(nodes) != 0 || len(keyOf) != 0 { + t.Errorf("nil body: %d nodes, %d keys; want none", len(nodes), len(keyOf)) + } + if nodes == nil || keyOf == nil { + t.Errorf("nil body: want empty (non-nil) slice and map") + } +} diff --git a/internal/dataflow/cfg.go b/internal/dataflow/cfg.go new file mode 100644 index 0000000..6c7214a --- /dev/null +++ b/internal/dataflow/cfg.go @@ -0,0 +1,327 @@ +package dataflow + +import ( + "go/ast" + "go/token" + "go/types" + "sort" +) + +// CFG edge kinds (spec §4.3), shared vocabulary. The Go add EdgeDeferResume +// is in cfg_exits.go, with the defer chain. +const ( + EdgeFallthrough = "fallthrough" + EdgeTrue = "true" + EdgeFalse = "false" + EdgeLoopBack = "loop_back" + EdgeSwitchCase = "switch_case" + EdgeBreak = "break" + EdgeContinue = "continue" + EdgeReturn = "return" + EdgeException = "exception" +) + +// BuildCFG links the body nodes of one callable into its statement-level CFG +// (spec §4.3) and returns the kept nodes with the edges. nodes and keyOf are +// BuildNodes' output for body; every edge endpoint is a key from keyOf, +// EntryKey or ExitKey. info resolves the builtin panic and the types canPanic +// needs; it may be nil (panic by name, every call / selector can panic). +// +// The returned nodes are nodes minus the pruned ones, in nodes' order: a +// statement / header / clause-head node unreachable from @entry (code after a +// return / panic / goto, the implicit empty statement of `L: }`), a deferred +// call whose defer statement is unreachable, and every satellite call node of +// a pruned statement. Callers (the L3 emitter) must emit body nodes from this +// set, not from BuildNodes' output. A nil body yields empty, non-nil slices. +// +// The linker works backwards like Java's CfgBuilder: link(s, next, kind) +// wires statement s so its normal completion reaches next, and returns the +// key control enters s at. A frame stack resolves break / continue / +// fallthrough; goto edges are added once every label's entry is known. +// finishExits then adds panic edges and the defer chain, prunes, and repairs +// exit reachability (see there). +// +// Structured edge rules: +// - @entry → first statement: fallthrough (return when the body is empty). +// - straight-line: fallthrough. The last statement of a body falls to +// @exit with kind return; the last statement of a loop body goes back to +// the header with kind loop_back. Branching constructs keep their own +// kind on those terminal edges (an else-less if's false, a break, …). +// - if: header → then entry true; → else entry (or next) false. +// - for cond / 3-clause, for range: header → body entry true, → next false, +// body tail → header loop_back. Init and post fold into the header. +// - for {}: header → body entry fallthrough, no false edge, plus the +// synthetic header → @exit exception (restores post-dominance). +// - a loop body with no node: header → header loop_back instead of the +// body entry edge. +// - switch / type switch: header → each clause head switch_case; without a +// default, also header → next switch_case. Clause head → its first body +// statement fallthrough (an empty clause falls to next). A fallthrough +// statement → the next clause's first body statement, or that clause's +// continuation when its body is empty, kind fallthrough. +// - select: header → each comm clause head switch_case, never → next. +// select {} has only the synthetic header → @exit exception. +// - break / continue (labeled or not) → resolved target, kind break / +// continue. continue targets the loop header (a for post folds into it). +// - goto L → L's entry, kind fallthrough. A label on a statement with no +// node of its own (a block, the implicit empty statement before `}`) +// resolves to the first node inside it, or to its continuation (ruling R6). +// - return → @exit, kind return. Every other non-exception edge into +// @exit (fall-off after an else-less if, a loop's false edge, a goto to +// a label at the end, …) is normalized to kind return too, so the defer +// chain sees only return / exception edges into @exit. +// +// Satellite call nodes are never linked. Deferred call nodes are linked only +// in the exit chain; their defer statements are on the straight-line path. +// +// Edges are deduplicated and sorted by (Src, Dst, Kind). +func BuildCFG(nodes []Node, keyOf map[ast.Node]string, body *ast.BlockStmt, info *types.Info) ([]Node, []CfgEdge) { + if body == nil { + return []Node{}, []CfgEdge{} + } + l := &linker{keyOf: keyOf, edges: map[CfgEdge]bool{}, labels: map[string]string{}} + first := l.seq(body.List, ExitKey, EdgeReturn) + if first == ExitKey { + l.edge(EntryKey, ExitKey, EdgeReturn) + } else { + l.edge(EntryKey, first, EdgeFallthrough) + } + for _, g := range l.gotos { + if dst, ok := l.labels[g.label]; ok { + l.edge(g.src, dst, EdgeFallthrough) + } + } + return finishExits(l, nodes, body, info) +} + +// frame is one enclosing break / continue / fallthrough scope. +type frame struct { + label string // "" when unlabeled + brk string // break target (the construct's continuation) + cont string // continue target (loop header); "" for switch / select + fall string // fallthrough target of the clause being linked (switch) +} + +type pendingGoto struct{ src, label string } + +type linker struct { + keyOf map[ast.Node]string + edges map[CfgEdge]bool + frames []frame + labels map[string]string // label → entry key of its statement + gotos []pendingGoto +} + +// edge records src → dst. Every non-exception edge into @exit is a function +// exit (a return, or falling off the end of the body however the last +// construct completes), so it takes kind return. +func (l *linker) edge(src, dst, kind string) { + if dst == ExitKey && kind != EdgeException { + kind = EdgeReturn + } + if src != "" && dst != "" { + l.edges[CfgEdge{Src: src, Dst: dst, Kind: kind}] = true + } +} + +func (l *linker) sorted() []CfgEdge { + out := make([]CfgEdge, 0, len(l.edges)) + for e := range l.edges { + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.Src != b.Src { + return a.Src < b.Src + } + if a.Dst != b.Dst { + return a.Dst < b.Dst + } + return a.Kind < b.Kind + }) + return out +} + +// seq links a statement list: each statement falls through to the next, the +// last one reaches next with kind. It returns the list's entry (next when the +// list has no node). +func (l *linker) seq(list []ast.Stmt, next, kind string) string { + for i := len(list) - 1; i >= 0; i-- { + next = l.link(list[i], next, kind, "") + kind = EdgeFallthrough + } + return next +} + +// link wires s so its normal completion reaches next with kind, and returns +// the key control enters s at. label is the label naming s, if any. +func (l *linker) link(s ast.Stmt, next, kind, label string) string { + switch s := s.(type) { + case *ast.BlockStmt: + return l.seq(s.List, next, kind) + + case *ast.LabeledStmt: + entry := next // R6: implicit `L: }` resolves to the continuation + if e, ok := s.Stmt.(*ast.EmptyStmt); !ok || !e.Implicit { + entry = l.link(s.Stmt, next, kind, s.Label.Name) + } + l.labels[s.Label.Name] = entry + return entry + + case *ast.IfStmt: + h := l.keyOf[s] + l.edge(h, l.seq(s.Body.List, next, kind), EdgeTrue) + if s.Else != nil { + l.edge(h, l.link(s.Else, next, kind, ""), EdgeFalse) + } else { + l.edge(h, next, EdgeFalse) + } + return h + + case *ast.ForStmt: + return l.loop(s, s.Body, s.Cond == nil, next, label) + + case *ast.RangeStmt: + return l.loop(s, s.Body, false, next, label) + + case *ast.SwitchStmt: + return l.clauses(s, s.Body, true, next, kind, label) + + case *ast.TypeSwitchStmt: + return l.clauses(s, s.Body, true, next, kind, label) + + case *ast.SelectStmt: + h := l.clauses(s, s.Body, false, next, kind, label) + if len(s.Body.List) == 0 { + l.blocksForever(h) + } + return h + + case *ast.ReturnStmt: + k := l.keyOf[s] + l.edge(k, ExitKey, EdgeReturn) + return k + + case *ast.BranchStmt: + return l.branch(s, next) + + default: + // Simple statements, bare / go calls, defer registrations. A statement + // with no node (an unlabeled empty statement) is transparent. + k, ok := l.keyOf[s] + if !ok { + return next + } + l.edge(k, next, kind) + return k + } +} + +// loop links a for / for-range statement with header node keyOf[s]. +func (l *linker) loop(s ast.Stmt, body *ast.BlockStmt, infinite bool, next, label string) string { + h := l.keyOf[s] + l.frames = append(l.frames, frame{label: label, brk: next, cont: h}) + entry := l.seq(body.List, h, EdgeLoopBack) + l.frames = l.frames[:len(l.frames)-1] + switch { + case entry == h: + l.edge(h, h, EdgeLoopBack) // empty body: the header loops on itself + case infinite: + l.edge(h, entry, EdgeFallthrough) + default: + l.edge(h, entry, EdgeTrue) + } + if infinite { + l.blocksForever(h) + } else { + l.edge(h, next, EdgeFalse) + } + return h +} + +// blocksForever adds the synthetic exception edge h → @exit that keeps a +// header with no normal way out (for {}, select {}) post-dominated by @exit. +// The defer chain redirects it like any exit edge; the later "every node +// reaches @exit" repair then finds h's loop already exiting, and would add +// the same edge anyway (absorbed by the dedup). +func (l *linker) blocksForever(h string) { + l.edge(h, ExitKey, EdgeException) +} + +// clauses links a switch, type switch (noMatch: a missing default falls to +// next) or select (no such edge) with header node keyOf[s]. Clauses are +// linked last to first so each knows where a fallthrough into the next one +// lands. +func (l *linker) clauses(s ast.Stmt, body *ast.BlockStmt, noMatch bool, next, kind, label string) string { + h := l.keyOf[s] + hasDefault := false + l.frames = append(l.frames, frame{label: label, brk: next}) + top := len(l.frames) - 1 + fallInto := next + for i := len(body.List) - 1; i >= 0; i-- { + var stmts []ast.Stmt + switch c := body.List[i].(type) { + case *ast.CaseClause: + stmts, hasDefault = c.Body, hasDefault || c.List == nil + case *ast.CommClause: + stmts = c.Body + } + head := l.keyOf[body.List[i]] + l.frames[top].fall = fallInto + entry := l.seq(stmts, next, kind) + if entry != next { + l.edge(head, entry, EdgeFallthrough) + } else { + l.edge(head, next, kind) + } + l.edge(h, head, EdgeSwitchCase) + fallInto = entry + } + l.frames = l.frames[:top] + if noMatch && !hasDefault { + l.edge(h, next, EdgeSwitchCase) + } + return h +} + +// branch links break / continue / goto / fallthrough statement s. +func (l *linker) branch(s *ast.BranchStmt, next string) string { + k, ok := l.keyOf[s] + if !ok { + return next + } + name := "" + if s.Label != nil { + name = s.Label.Name + } + switch s.Tok { + case token.BREAK: + if f := l.target(name, false); f != nil { + l.edge(k, f.brk, EdgeBreak) + } + case token.CONTINUE: + if f := l.target(name, true); f != nil { + l.edge(k, f.cont, EdgeContinue) + } + case token.FALLTHROUGH: + if n := len(l.frames); n > 0 { + l.edge(k, l.frames[n-1].fall, EdgeFallthrough) + } + case token.GOTO: + l.gotos = append(l.gotos, pendingGoto{src: k, label: name}) + } + return k +} + +// target returns the innermost frame a break (loop=false) or continue +// (loop=true) with label name ("" = unlabeled) resolves to, or nil for +// ill-formed code. +func (l *linker) target(name string, loop bool) *frame { + for i := len(l.frames) - 1; i >= 0; i-- { + f := &l.frames[i] + if (name == "" || f.label == name) && (!loop || f.cont != "") { + return f + } + } + return nil +} diff --git a/internal/dataflow/cfg_exits.go b/internal/dataflow/cfg_exits.go new file mode 100644 index 0000000..4c7c39a --- /dev/null +++ b/internal/dataflow/cfg_exits.go @@ -0,0 +1,509 @@ +package dataflow + +import ( + "go/ast" + "go/token" + "go/types" + "sort" +) + +// EdgeDeferResume is the Go add (spec §4.3.1): a deferred call's normal +// completion resuming the exit chain (the next deferred call, or @exit). +const EdgeDeferResume = "defer_resume" + +// finishExits applies the exit passes to the structured CFG in l.edges and +// returns the kept nodes (in nodes' order) and the sorted edges: +// +// 1. panic edges: every canPanic node gets exception → @exit; a panic(...) +// statement keeps only that edge. +// 2. defer chain: the registered deferred calls in reverse source order are +// D1..Dn. Every edge into @exit is redirected to D1 with its kind kept; +// Di → Di+1 and Dn → @exit are defer_resume, plus exception when Di can +// panic. A deferred call is registered when its defer statement is +// reachable from @entry; one that never is joins no chain. +// 3. pruning: nodes unreachable from @entry are dropped (the chain, @entry +// and @exit are always kept), with their edges and satellites. +// 4. exit reachability: each set of nodes that cannot reach @exit — a +// bottom strongly connected component of the graph — gets one synthetic +// exception edge to the exit target (D1, else @exit) from its header, +// the component's first node in depth-first order from @entry. +// 5. dedup (the edge set) and sort. +func finishExits(l *linker, nodes []Node, body *ast.BlockStmt, info *types.Info) ([]Node, []CfgEdge) { + // 1. Panic edges. Satellite and deferred call nodes are not CFG steps + // here; a satellite's panic is its parent statement's. + panics := map[string]bool{} + for _, n := range nodes { + if n.Stmt == nil || n.Parent != "" || n.Deferred { + continue + } + if isPanicStmt(n.Stmt, info) { + panics[n.Key] = true + } else if canPanic(n.Stmt, info) { + l.edges[CfgEdge{Src: n.Key, Dst: ExitKey, Kind: EdgeException}] = true + } + } + for e := range l.edges { + if panics[e.Src] { + delete(l.edges, e) + } + } + for k := range panics { + l.edges[CfgEdge{Src: k, Dst: ExitKey, Kind: EdgeException}] = true + } + + // 2. Defer chain over the registered deferred calls. + reach := reachable(l.edges, EntryKey) + regOf := registrations(body, l.keyOf) + var chain []Node + for i := len(nodes) - 1; i >= 0; i-- { + if n := nodes[i]; n.Deferred && reach[regOf[n.Stmt]] { + chain = append(chain, n) + } + } + exitTarget := ExitKey + if len(chain) > 0 { + exitTarget = chain[0].Key + for e := range l.edges { + if e.Dst == ExitKey { + delete(l.edges, e) + l.edges[CfgEdge{Src: e.Src, Dst: exitTarget, Kind: e.Kind}] = true + } + } + for i, d := range chain { + next := ExitKey + if i+1 < len(chain) { + next = chain[i+1].Key + } + l.edges[CfgEdge{Src: d.Key, Dst: next, Kind: EdgeDeferResume}] = true + if canPanic(d.Stmt, info) { + l.edges[CfgEdge{Src: d.Key, Dst: next, Kind: EdgeException}] = true + } + } + } + + // 3. Pruning. + keep := map[string]bool{EntryKey: true, ExitKey: true} + for k := range reach { + keep[k] = true + } + for _, d := range chain { + keep[d.Key] = true + } + for e := range l.edges { + if !keep[e.Src] { + delete(l.edges, e) + } + } + kept := make([]Node, 0, len(nodes)) + for _, n := range nodes { + if keep[n.Key] && n.Parent == "" || n.Parent != "" && keep[n.Parent] { + kept = append(kept, n) + } + } + + // 4. Exit reachability. + for _, h := range stuckHeaders(l.edges, kept) { + l.edges[CfgEdge{Src: h, Dst: exitTarget, Kind: EdgeException}] = true + } + + // 5. Dedup and sort. + return kept, l.sorted() +} + +// registrations maps each deferred call expression in body to the key of its +// defer statement. Function literal bodies are not entered. +func registrations(body *ast.BlockStmt, keyOf map[ast.Node]string) map[ast.Node]string { + out := map[ast.Node]string{} + ast.Inspect(body, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncLit: + return false + case *ast.DeferStmt: + out[n.Call] = keyOf[n] + } + return true + }) + return out +} + +// successors returns the adjacency of edges, each list sorted, so walks are +// deterministic. +func successors(edges map[CfgEdge]bool) map[string][]string { + succ := map[string][]string{} + seen := map[[2]string]bool{} + for e := range edges { + if p := [2]string{e.Src, e.Dst}; !seen[p] { + seen[p] = true + succ[e.Src] = append(succ[e.Src], e.Dst) + } + } + for _, s := range succ { + sort.Strings(s) + } + return succ +} + +// reachable returns the keys reachable from src, src included. +func reachable(edges map[CfgEdge]bool, src string) map[string]bool { + succ := successors(edges) + seen := map[string]bool{src: true} + stack := []string{src} + for len(stack) > 0 { + x := stack[len(stack)-1] + stack = stack[:len(stack)-1] + for _, y := range succ[x] { + if !seen[y] { + seen[y] = true + stack = append(stack, y) + } + } + } + return seen +} + +// stuckHeaders returns, for each bottom strongly connected component among +// the CFG nodes that cannot reach @exit, its header: the member first visited +// by a depth-first walk from @entry. One exit edge per bottom component makes +// every stuck node reach @exit (each one reaches some bottom component). The +// loop is found structurally, never by edge kind. +func stuckHeaders(edges map[CfgEdge]bool, kept []Node) []string { + pred := map[string][]string{} + for e := range edges { + pred[e.Dst] = append(pred[e.Dst], e.Src) + } + reachesExit := map[string]bool{ExitKey: true} + stack := []string{ExitKey} + for len(stack) > 0 { + x := stack[len(stack)-1] + stack = stack[:len(stack)-1] + for _, p := range pred[x] { + if !reachesExit[p] { + reachesExit[p] = true + stack = append(stack, p) + } + } + } + stuck := map[string]bool{} + for _, n := range kept { + if n.Parent == "" && !reachesExit[n.Key] { + stuck[n.Key] = true + } + } + if len(stuck) == 0 { + return nil + } + + succ := successors(edges) + // Depth-first preorder from @entry: the header is the first member seen. + order := map[string]int{} + var dfs func(string) + dfs = func(x string) { + order[x] = len(order) + for _, y := range succ[x] { + if _, ok := order[y]; !ok { + dfs(y) + } + } + } + dfs(EntryKey) + + // Tarjan's SCCs over the stuck subgraph (closed under successors). + index, low, onStack := map[string]int{}, map[string]int{}, map[string]bool{} + var st []string + var headers []string + var strong func(string) + strong = func(v string) { + index[v], low[v] = len(index), len(index) + st = append(st, v) + onStack[v] = true + for _, w := range succ[v] { + if _, ok := index[w]; !ok { + strong(w) + low[v] = min(low[v], low[w]) + } else if onStack[w] { + low[v] = min(low[v], index[w]) + } + } + if low[v] != index[v] { + return + } + var comp []string + for { + w := st[len(st)-1] + st = st[:len(st)-1] + onStack[w] = false + comp = append(comp, w) + if w == v { + break + } + } + in := map[string]bool{} + for _, c := range comp { + in[c] = true + } + h := comp[0] + for _, c := range comp { + for _, y := range succ[c] { + if !in[y] { + return // not a bottom component + } + } + if order[c] < order[h] { + h = c + } + } + headers = append(headers, h) + } + keys := make([]string, 0, len(stuck)) + for k := range stuck { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { return order[keys[i]] < order[keys[j]] }) + for _, k := range keys { + if _, ok := index[k]; !ok { + strong(k) + } + } + return headers +} + +// isPanicStmt reports whether s is a statement calling the builtin panic. +// The name may be shadowed, so the callee is resolved through info; without +// info the name decides. +func isPanicStmt(s ast.Node, info *types.Info) bool { + es, ok := s.(*ast.ExprStmt) + if !ok { + return false + } + call, ok := ast.Unparen(es.X).(*ast.CallExpr) + if !ok { + return false + } + id, ok := ast.Unparen(call.Fun).(*ast.Ident) + if !ok { + return false + } + if info == nil { + return id.Name == "panic" + } + b, ok := info.Uses[id].(*types.Builtin) + return ok && b.Name() == "panic" +} + +// canPanic reports whether body node n (a node's Stmt) can panic (spec +// §4.3.2). It inspects only the node's own expressions: +// - a header: if Init/Cond; for Init/Cond/Post; range Key/Value/X; switch +// Init/Tag; type switch Init/Assign; a select header has none; +// - a clause head: a case's List, a comm clause's Comm; +// - a defer / go statement: the call's function value (its receiver) and +// arguments, evaluated at the statement — not the call, which runs at +// exit or in the new goroutine; +// - any other node (a simple statement, a return, a deferred call node's +// *ast.CallExpr): the whole node. +// +// Nested statement bodies and function literals are never entered. Within +// those expressions it is true for: a call that is not a type conversion +// (builtins included, close among them); an index or slice expression (a +// generic instantiation is not one); a type assertion other than x.(type) +// and the comma-ok form; a pointer dereference; a selector through a +// pointer, or an interface method value; a channel send; a / or % (or /=, +// %=) with a non-constant divisor; a shift (or <<=, >>=) by a non-constant +// signed count; a slice → array or array-pointer conversion. A range over a +// function calls the iterator, so its header can panic. +// With info nil, every call, selector and star expression counts. +func canPanic(n ast.Node, info *types.Info) bool { + var parts []ast.Node + switch s := n.(type) { + case *ast.IfStmt: + parts = []ast.Node{s.Init, s.Cond} + case *ast.ForStmt: + parts = []ast.Node{s.Init, s.Cond, s.Post} + case *ast.RangeStmt: + if rangesOverFunc(s.X, info) { + return true // each iteration calls the iterator + } + parts = []ast.Node{s.Key, s.Value, s.X} + case *ast.SwitchStmt: + parts = []ast.Node{s.Init, s.Tag} + case *ast.TypeSwitchStmt: + parts = []ast.Node{s.Init, s.Assign} + case *ast.SelectStmt, *ast.BlockStmt, *ast.LabeledStmt: + return false + case *ast.CaseClause: + for _, x := range s.List { + parts = append(parts, x) + } + case *ast.CommClause: + parts = []ast.Node{s.Comm} + case *ast.DeferStmt: + parts = callOperands(s.Call) + case *ast.GoStmt: + parts = callOperands(s.Call) + default: + parts = []ast.Node{n} + } + for _, p := range parts { + if p != nil && exprPanics(p, info) { + return true + } + } + return false +} + +// callOperands is what a defer / go statement evaluates: the function value +// and the arguments. +func callOperands(call *ast.CallExpr) []ast.Node { + out := []ast.Node{call.Fun} + for _, a := range call.Args { + out = append(out, a) + } + return out +} + +// exprPanics walks n (no function literal bodies) for a panic-capable +// operation. +func exprPanics(n ast.Node, info *types.Info) bool { + commaOK := map[ast.Expr]bool{} + found := false + ast.Inspect(n, func(x ast.Node) bool { + if found { + return false + } + switch x := x.(type) { + case *ast.FuncLit: + return false + case *ast.AssignStmt: + if len(x.Lhs) == 2 && len(x.Rhs) == 1 { + commaOK[ast.Unparen(x.Rhs[0])] = true + } + switch x.Tok { + case token.QUO_ASSIGN, token.REM_ASSIGN: + found = !isConst(x.Rhs[0], info) + case token.SHL_ASSIGN, token.SHR_ASSIGN: + found = signedCount(x.Rhs[0], info) + } + case *ast.ValueSpec: + if len(x.Names) == 2 && len(x.Values) == 1 { + commaOK[ast.Unparen(x.Values[0])] = true + } + case *ast.CallExpr: + found = !isType(x.Fun, info) || sliceToArray(x, info) + case *ast.IndexExpr: + found = !isType(x.Index, info) + case *ast.SliceExpr, *ast.SendStmt: + found = true + case *ast.TypeAssertExpr: + found = x.Type != nil && !commaOK[x] + case *ast.StarExpr: + found = !isType(x, info) + case *ast.SelectorExpr: + found = viaPointer(x, info) + case *ast.BinaryExpr: + switch x.Op { + case token.QUO, token.REM: + found = !isConst(x.Y, info) + case token.SHL, token.SHR: + found = signedCount(x.Y, info) + } + } + return !found + }) + return found +} + +// isType reports whether e denotes a type (false without info). +func isType(e ast.Expr, info *types.Info) bool { + if info == nil { + return false + } + tv, ok := info.Types[e] + return ok && tv.IsType() +} + +// isConst reports whether e is a constant expression. Without info only a +// basic literal is. +func isConst(e ast.Expr, info *types.Info) bool { + if info == nil { + _, ok := ast.Unparen(e).(*ast.BasicLit) + return ok + } + tv, ok := info.Types[e] + return ok && tv.Value != nil +} + +// viaPointer reports whether selector x can panic on a nil receiver: its +// receiver is pointer-typed, the selection walks an embedded pointer, or it +// is a method value on an interface (nil interface). A qualified identifier +// (pkg.Name) and a method expression do not. Without info every selector +// does. +func viaPointer(x *ast.SelectorExpr, info *types.Info) bool { + if info == nil { + return true + } + sel, ok := info.Selections[x] + if !ok || sel.Kind() == types.MethodExpr { + return false + } + if sel.Kind() == types.MethodVal && types.IsInterface(sel.Recv()) { + return true + } + _, ptr := sel.Recv().Underlying().(*types.Pointer) + return ptr || sel.Indirect() +} + +// signedCount reports whether shift count e can be negative: non-constant +// and of a signed integer type. Without info any non-literal count can. +func signedCount(e ast.Expr, info *types.Info) bool { + if isConst(e, info) { + return false + } + if info == nil { + return true + } + t := info.Types[e].Type + if t == nil { + return true + } + b, ok := t.Underlying().(*types.Basic) + return !ok || b.Info()&types.IsUnsigned == 0 +} + +// sliceToArray reports whether conversion call converts a slice to an array +// or an array pointer, which panics when the slice is too short. +func sliceToArray(call *ast.CallExpr, info *types.Info) bool { + if info == nil || len(call.Args) != 1 { + return false + } + to := info.Types[call.Fun].Type + if to == nil { + return false + } + to = to.Underlying() + if p, ok := to.(*types.Pointer); ok { + to = p.Elem().Underlying() + } + if _, ok := to.(*types.Array); !ok { + return false + } + from := info.Types[call.Args[0]].Type + if from == nil { + return false + } + _, ok := from.Underlying().(*types.Slice) + return ok +} + +// rangesOverFunc reports whether a range statement over x iterates a +// function (range-over-func). Without info it cannot tell and says no; the +// header's own expressions are still inspected. +func rangesOverFunc(x ast.Expr, info *types.Info) bool { + if info == nil || x == nil { + return false + } + t := info.Types[x].Type + if t == nil { + return false + } + _, ok := t.Underlying().(*types.Signature) + return ok +} diff --git a/internal/dataflow/cfg_test.go b/internal/dataflow/cfg_test.go new file mode 100644 index 0000000..90d3bd3 --- /dev/null +++ b/internal/dataflow/cfg_test.go @@ -0,0 +1,1303 @@ +package dataflow_test + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "golang.org/x/tools/go/cfg" + "golang.org/x/tools/go/packages" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" + v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" +) + +// Statement keys in testdata/dataflow/flow/flow.go (line:col of each node's +// anchor). Synthetic endpoints are dataflow.EntryKey / dataflow.ExitKey. +const ( + entry = dataflow.EntryKey + exit = dataflow.ExitKey + + // IfElse + ieIf, ieThenRet, ieElseAsg, ieRet = "16:2", "17:3", "19:3", "21:2" + // ForLoop + flInit, flFor, flBody, flRet = "26:2", "27:2", "28:3", "30:2" + // Infinite + infFor, infRecv, infIf, infRet = "35:2", "36:3", "37:3", "38:4" + // Range + rgInit, rgFor, rgBody, rgRet = "45:2", "46:2", "47:3", "49:2" + // Switch + swInit, swHdr, swCase1, swAsg1, swFall, swCase2, swAsg2, swDefault, swAsgD, swRet = "54:2", "55:2", "56:2", "57:3", "58:3", "59:2", "60:3", "61:2", "62:3", "64:2" + // TypeSwitch + tsHdr, tsInt, tsUse, tsRetInt, tsStr, tsRetStr, tsDefault, tsRetD = "69:2", "70:2", "71:3", "72:3", "73:2", "74:3", "75:2", "76:3" + // Select2 + selHdr, selRecv, selRetX, selSend, selRet0 = "82:2", "83:2", "84:3", "85:2", "86:3" + // Labeled + lbInit, lbOuter, lbInner, lbIfNeg, lbCont, lbIfZero, lbBreak, lbAdd, lbRet = "92:2", "94:2", "95:3", "96:4", "97:5", "99:4", "100:5", "102:4", "105:2" + // GotoBack + gbInit, gbInc, gbIf, gbGoto, gbRet = "110:2", "112:2", "113:2", "114:3", "116:2" + // EmptySelect + esSel = "121:2" + // SwitchNoDefault + sndHdr, sndCase1, sndAsg, sndFall, sndCase2, sndRet = "239:2", "240:2", "241:3", "242:3", "243:2", "245:2" + // BreakContinue + bcFor, bcDec, bcIf5, bcCont, bcIf2, bcBreak, bcRet = "250:2", "251:3", "252:3", "253:4", "255:3", "256:4", "259:2" + // GotoBlock + gkIf, gkGotoM, gkDec, gkInc, gkIf5, gkGotoE, gkAsg, gkRet = "265:2", "266:3", "268:2", "271:3", "273:2", "274:3", "276:2", "280:2" + // GotoEnd + geIf, geGoto, geInc, geEmpty = "285:2", "286:3", "288:2", "290:1" + // EarlyReturnDefer: d1 is the last defer in source (first in the chain). + erdReg1, erdReg2, erdIf, erdRetNeg, erdCall, erdRet, erdD1, erdD2 = "126:2", "127:2", "128:2", "129:3", "131:2", "132:2", "127:8", "126:8" + // Panics + pIf, pPanic, pIndex, pRet = "137:2", "138:3", "140:2", "141:2" +) + +// edges runs Tasks 5–7 over fixture function fn with its real L1 call keys +// and checks the output contract. +func edges(t *testing.T, fn string) []dataflow.CfgEdge { + t.Helper() + _, es := cfgOf(t, fn) + return es +} + +// cfgOf is edges, also returning the pruned node set. +func cfgOf(t *testing.T, fn string) ([]dataflow.Node, []dataflow.CfgEdge) { + t.Helper() + fx := loadFlow(t) + body := fx.decl(t, fn).Body + nodes, keyOf := dataflow.BuildNodes(fx.fset, body, fx.callKeys(t, fn)) + kept, es := dataflow.BuildCFG(nodes, keyOf, body, fx.info) + checkContract(t, fn, kept, es) + return kept, es +} + +// checkContract asserts the output contract: every endpoint is a returned +// node and never a satellite call node; a deferred call node is linked only +// by exit-chain edges (it is entered by an exit edge, left by defer_resume / +// exception); the edges are sorted by (src, dst, kind) with no duplicates. +func checkContract(t *testing.T, fn string, nodes []dataflow.Node, es []dataflow.CfgEdge) { + t.Helper() + byKey := map[string]dataflow.Node{} + for _, n := range nodes { + byKey[n.Key] = n + } + for _, g := range es { + for _, k := range []string{g.Src, g.Dst} { + n, ok := byKey[k] + if !ok { + t.Errorf("%s: edge %v endpoint %s is not a node", fn, g, k) + } else if n.Parent != "" { + t.Errorf("%s: edge %v links satellite call %s", fn, g, k) + } + } + if byKey[g.Src].Deferred && g.Kind != "defer_resume" && g.Kind != "exception" { + t.Errorf("%s: deferred call leaves by %v", fn, g) + } + if byKey[g.Dst].Deferred && g.Kind != "defer_resume" && g.Kind != "exception" && g.Kind != "return" { + t.Errorf("%s: deferred call entered by %v", fn, g) + } + } + for i := 1; i < len(es); i++ { + a, b := es[i-1], es[i] + if a == b { + t.Errorf("%s: duplicate edge %v", fn, b) + } + if a.Src > b.Src || a.Src == b.Src && (a.Dst > b.Dst || a.Dst == b.Dst && a.Kind > b.Kind) { + t.Errorf("%s: edges out of order: %v before %v", fn, a, b) + } + } +} + +func e(src, dst, kind string) dataflow.CfgEdge { + return dataflow.CfgEdge{Src: src, Dst: dst, Kind: kind} +} + +// wantEdges asserts the CFG of fn is exactly want (order-insensitive). +func wantEdges(t *testing.T, fn string, want ...dataflow.CfgEdge) []dataflow.CfgEdge { + t.Helper() + got := edges(t, fn) + have := map[dataflow.CfgEdge]bool{} + for _, g := range got { + have[g] = true + } + exp := map[dataflow.CfgEdge]bool{} + for _, w := range want { + exp[w] = true + if !have[w] { + t.Errorf("%s: missing edge %s→%s %s", fn, w.Src, w.Dst, w.Kind) + } + } + for _, g := range got { + if !exp[g] { + t.Errorf("%s: unexpected edge %s→%s %s", fn, g.Src, g.Dst, g.Kind) + } + } + return got +} + +func TestCFG_IfElse(t *testing.T) { + // The then-arm is a return, so it leaves to @exit instead of rejoining. + wantEdges(t, "IfElse", + e(entry, ieIf, "fallthrough"), + e(ieIf, ieThenRet, "true"), + e(ieIf, ieElseAsg, "false"), + e(ieThenRet, exit, "return"), + e(ieElseAsg, ieRet, "fallthrough"), + e(ieRet, exit, "return"), + ) +} + +func TestCFG_ForLoop(t *testing.T) { + // 3-clause for: init and post fold into the header. + wantEdges(t, "ForLoop", + e(entry, flInit, "fallthrough"), + e(flInit, flFor, "fallthrough"), + e(flFor, flBody, "true"), + e(flFor, flRet, "false"), + e(flBody, flFor, "loop_back"), + e(flRet, exit, "return"), + ) + // for range: same shape. + wantEdges(t, "Range", + e(entry, rgInit, "fallthrough"), + e(rgInit, rgFor, "fallthrough"), + e(rgFor, rgBody, "true"), + e(rgFor, rgRet, "false"), + e(rgBody, rgFor, "loop_back"), + e(rgRet, exit, "return"), + ) + // cond-only for with unlabeled continue / break. An else-less if at the + // body's tail keeps its `false` kind into the header. + wantEdges(t, "BreakContinue", + e(entry, bcFor, "fallthrough"), + e(bcFor, bcDec, "true"), + e(bcFor, bcRet, "false"), + e(bcDec, bcIf5, "fallthrough"), + e(bcIf5, bcCont, "true"), + e(bcIf5, bcIf2, "false"), + e(bcCont, bcFor, "continue"), + e(bcIf2, bcBreak, "true"), + e(bcIf2, bcFor, "false"), + e(bcBreak, bcRet, "break"), + e(bcRet, exit, "return"), + ) +} + +func TestCFG_Switch(t *testing.T) { + // With a default: no header→after edge. `fallthrough` enters the next + // clause's first body statement, not its head. + wantEdges(t, "Switch", + e(entry, swInit, "fallthrough"), + e(swInit, swHdr, "fallthrough"), + e(swHdr, swCase1, "switch_case"), + e(swHdr, swCase2, "switch_case"), + e(swHdr, swDefault, "switch_case"), + e(swCase1, swAsg1, "fallthrough"), + e(swAsg1, swFall, "fallthrough"), + e(swFall, swAsg2, "fallthrough"), + e(swCase2, swAsg2, "fallthrough"), + e(swAsg2, swRet, "fallthrough"), + e(swDefault, swAsgD, "fallthrough"), + e(swAsgD, swRet, "fallthrough"), + e(swRet, exit, "return"), + ) + // No default: one extra switch_case header→after. `fallthrough` into an + // empty clause goes to that clause's continuation (after the switch). + wantEdges(t, "SwitchNoDefault", + e(entry, sndHdr, "fallthrough"), + e(sndHdr, sndCase1, "switch_case"), + e(sndHdr, sndCase2, "switch_case"), + e(sndHdr, sndRet, "switch_case"), + e(sndCase1, sndAsg, "fallthrough"), + e(sndAsg, sndFall, "fallthrough"), + e(sndFall, sndRet, "fallthrough"), + e(sndCase2, sndRet, "fallthrough"), + e(sndRet, exit, "return"), + ) + // Type switch: the guard folds into the header; every clause returns. + wantEdges(t, "TypeSwitch", + e(entry, tsHdr, "fallthrough"), + e(tsHdr, tsInt, "switch_case"), + e(tsHdr, tsStr, "switch_case"), + e(tsHdr, tsDefault, "switch_case"), + e(tsInt, tsUse, "fallthrough"), + e(tsUse, tsRetInt, "fallthrough"), + e(tsRetInt, exit, "return"), + e(tsStr, tsRetStr, "fallthrough"), + e(tsRetStr, exit, "return"), + e(tsDefault, tsRetD, "fallthrough"), + e(tsRetD, exit, "return"), + ) +} + +func TestCFG_Select(t *testing.T) { + // select blocks: header→each comm clause, never header→after (here @exit). + wantEdges(t, "Select2", + e(entry, selHdr, "fallthrough"), + e(selHdr, selRecv, "switch_case"), + e(selHdr, selSend, "switch_case"), + e(selRecv, selRetX, "fallthrough"), + e(selRetX, exit, "return"), + e(selSend, selRet0, "fallthrough"), + e(selSend, exit, "exception"), // a send on a closed channel panics + e(selRet0, exit, "return"), + ) + // select {} blocks forever: only the synthetic exception edge leaves it. + wantEdges(t, "EmptySelect", + e(entry, esSel, "fallthrough"), + e(esSel, exit, "exception"), + ) +} + +func TestCFG_LabeledBreakContinue(t *testing.T) { + // continue outer → outer header; break outer → after the outer loop. The + // inner loop's false edge leaves to the outer header (its continuation). + wantEdges(t, "Labeled", + e(entry, lbInit, "fallthrough"), + e(lbInit, lbOuter, "fallthrough"), + e(lbOuter, lbInner, "true"), + e(lbOuter, lbRet, "false"), + e(lbInner, lbIfNeg, "true"), + e(lbInner, lbOuter, "false"), + e(lbIfNeg, lbCont, "true"), + e(lbIfNeg, lbIfZero, "false"), + e(lbCont, lbOuter, "continue"), + e(lbIfZero, lbBreak, "true"), + e(lbIfZero, lbAdd, "false"), + e(lbBreak, lbRet, "break"), + e(lbAdd, lbInner, "loop_back"), + e(lbRet, exit, "return"), + ) + // Backward goto to a labeled statement: kind fallthrough. + wantEdges(t, "GotoBack", + e(entry, gbInit, "fallthrough"), + e(gbInit, gbInc, "fallthrough"), + e(gbInc, gbIf, "fallthrough"), + e(gbIf, gbGoto, "true"), + e(gbIf, gbRet, "false"), + e(gbGoto, gbInc, "fallthrough"), + e(gbRet, exit, "return"), + ) +} + +// TestCFG_GotoLabeledBlock locks ruling R6: a label on a non-node statement +// resolves to the first node inside it, or, when it holds none, to the +// statement's continuation. +func TestCFG_GotoLabeledBlock(t *testing.T) { + // goto M → `M: { a++ }` enters a++; goto E → `E: {}` reaches the return. + wantEdges(t, "GotoBlock", + e(entry, gkIf, "fallthrough"), + e(gkIf, gkGotoM, "true"), + e(gkIf, gkDec, "false"), + e(gkGotoM, gkInc, "fallthrough"), + e(gkDec, gkInc, "fallthrough"), + e(gkInc, gkIf5, "fallthrough"), + e(gkIf5, gkGotoE, "true"), + e(gkIf5, gkAsg, "false"), + e(gkGotoE, gkRet, "fallthrough"), + e(gkAsg, gkRet, "fallthrough"), + e(gkRet, exit, "return"), + ) + // `L: }` labels the implicit empty statement: goto L continues past the + // end of the body to @exit — and every non-exception edge into @exit is + // kind return. BuildNodes still emits a zero-width node for it (290:1); + // the CFG leaves it unlinked and unreachable, so pruning drops it from the + // returned node set. + got := wantEdges(t, "GotoEnd", + e(entry, geIf, "fallthrough"), + e(geIf, geGoto, "true"), + e(geIf, geInc, "false"), + e(geGoto, exit, "return"), + e(geInc, exit, "return"), + ) + for _, g := range got { + if g.Src == geEmpty || g.Dst == geEmpty { + t.Errorf("GotoEnd: implicit empty statement is linked: %v", g) + } + } + nodes, _ := cfgOf(t, "GotoEnd") + if hasNode(nodes, geEmpty) { + t.Errorf("GotoEnd: implicit empty statement %s not pruned", geEmpty) + } +} + +func hasNode(nodes []dataflow.Node, key string) bool { + for _, n := range nodes { + if n.Key == key { + return true + } + } + return false +} + +func TestCFG_Infinite(t *testing.T) { + got := wantEdges(t, "Infinite", + e(entry, infFor, "fallthrough"), + e(infFor, infRecv, "fallthrough"), + e(infFor, exit, "exception"), // synthetic: restores post-dominance + e(infRecv, infIf, "fallthrough"), + e(infIf, infRet, "true"), + e(infIf, infFor, "false"), + e(infRet, exit, "return"), + ) + for _, g := range got { + if g.Src == infFor && g.Kind == "false" { + t.Errorf("Infinite: for {} has a false edge %v", g) + } + } +} + +// TestCFG_EmptyLoopBody: a loop body with no node is a loop_back self-edge. +func TestCFG_EmptyLoopBody(t *testing.T) { + wantEdges(t, "Spin", + e(entry, "294:2", "fallthrough"), + e("294:2", "294:2", "loop_back"), + e("294:2", "296:2", "false"), + e("296:2", "296:2", "loop_back"), + e("296:2", exit, "exception"), + ) +} + +// TestCFG_DeferChainOrderAndKinds: deferred calls run only in the exit chain, +// last-registered first. Edges into the chain keep their kind (return / +// exception); chain-internal edges and the tail → @exit are defer_resume, and +// a deferred call that can panic also passes an exception edge down the chain. +func TestCFG_DeferChainOrderAndKinds(t *testing.T) { + wantEdges(t, "EarlyReturnDefer", + e(entry, erdReg1, "fallthrough"), + e(erdReg1, erdReg2, "fallthrough"), + e(erdReg2, erdIf, "fallthrough"), + e(erdReg2, erdD1, "exception"), // evaluating g(a) at registration + e(erdIf, erdRetNeg, "true"), + e(erdIf, erdCall, "false"), + e(erdRetNeg, erdD1, "return"), + e(erdCall, erdRet, "fallthrough"), + e(erdCall, erdD1, "exception"), + e(erdRet, erdD1, "return"), + e(erdD1, erdD2, "defer_resume"), + e(erdD1, erdD2, "exception"), + e(erdD2, exit, "defer_resume"), + e(erdD2, exit, "exception"), + ) +} + +// TestCFG_PanicOnlyException: a panic(...) statement leaves only by its +// exception edge; nothing falls through it. +func TestCFG_PanicOnlyException(t *testing.T) { + got := wantEdges(t, "Panics", + e(entry, pIf, "fallthrough"), + e(pIf, pPanic, "true"), + e(pIf, pIndex, "false"), + e(pPanic, exit, "exception"), + e(pIndex, pRet, "fallthrough"), + e(pIndex, exit, "exception"), + e(pRet, exit, "return"), + ) + if out := outEdges(got, pPanic); len(out) != 1 || out[0].Kind != "exception" { + t.Errorf("Panics: panic out-edges = %v, want one exception", out) + } + // A body ending in panic falls off nowhere. + wantEdges(t, "PanicLast", + e(entry, "412:2", "fallthrough"), + e("412:2", "413:2", "fallthrough"), + e("412:2", exit, "exception"), + e("413:2", exit, "exception"), + ) + // A local named panic is an ordinary call: it falls through too. + wantEdges(t, "ShadowPanic", + e(entry, "418:2", "fallthrough"), + e("418:2", "419:2", "fallthrough"), + e("419:2", "420:2", "fallthrough"), + e("419:2", exit, "exception"), + e("420:2", exit, "return"), + ) +} + +func outEdges(es []dataflow.CfgEdge, src string) []dataflow.CfgEdge { + var out []dataflow.CfgEdge + for _, g := range es { + if g.Src == src { + out = append(out, g) + } + } + return out +} + +// TestCFG_IndexCanPanic: `v := xs[i]` both falls through and may panic. +func TestCFG_IndexCanPanic(t *testing.T) { + got := edges(t, "Panics") + out := outEdges(got, pIndex) + if len(out) != 2 || out[0] != e(pIndex, pRet, "fallthrough") || out[1] != e(pIndex, exit, "exception") { + t.Errorf("Panics: %s out-edges = %v, want exception + fallthrough", pIndex, out) + } +} + +// TestCFG_UnreachableAfterReturnPruned: code after a return is dropped from +// both the edges and the returned node set, and a defer that is never +// registered stays out of the exit chain. Satellite calls of a kept statement +// are kept. +func TestCFG_UnreachableAfterReturnPruned(t *testing.T) { + wantEdges(t, "Unreachable", + e(entry, "189:2", "fallthrough"), + e("189:2", exit, "return"), + ) + nodes, _ := cfgOf(t, "Unreachable") + if keys := nodeKeys(nodes); strings.Join(keys, " ") != entry+" 189:2 "+exit { + t.Errorf("Unreachable: nodes = %v, want [@entry 189:2 @exit]", keys) + } + wantEdges(t, "DeadDefer", + e(entry, "425:2", "fallthrough"), + e("425:2", exit, "return"), + ) + nodes, _ = cfgOf(t, "DeadDefer") + if keys := nodeKeys(nodes); strings.Join(keys, " ") != entry+" 425:2 "+exit { + t.Errorf("DeadDefer: nodes = %v, want [@entry 425:2 @exit]", keys) + } + nodes, _ = cfgOf(t, "Nested") + if keys := nodeKeys(nodes); strings.Join(keys, " ") != entry+" 196:2 196:7 196:9 197:2 "+exit { + t.Errorf("Nested: nodes = %v, want the satellites kept", keys) + } +} + +func nodeKeys(nodes []dataflow.Node) []string { + keys := make([]string, len(nodes)) + for i, n := range nodes { + keys[i] = n.Key + } + return keys +} + +// TestCFG_GotoBackwardAndEmptySelect: a node that cannot reach @exit gets one +// synthetic exception edge to the exit target (the chain head under a defer) +// from the header of its endless loop — the goto target, found structurally. +// select {} keeps Task 6's synthetic edge, now routed through the chain. +func TestCFG_GotoBackwardAndEmptySelect(t *testing.T) { + wantEdges(t, "GotoForever", + e(entry, "398:2", "fallthrough"), + e("398:2", "400:2", "fallthrough"), + e("400:2", "401:2", "fallthrough"), + e("400:2", "398:8", "exception"), // repair: loop header → chain + e("401:2", "400:2", "fallthrough"), + e("398:8", exit, "defer_resume"), + e("398:8", exit, "exception"), + ) + wantEdges(t, "SelectForever", + e(entry, "406:2", "fallthrough"), + e("406:2", "407:2", "fallthrough"), + e("407:2", "406:8", "exception"), + e("406:8", exit, "defer_resume"), + e("406:8", exit, "exception"), + ) + wantEdges(t, "EmptySelect", + e(entry, esSel, "fallthrough"), + e(esSel, exit, "exception"), + ) +} + +// TestCFG_ConditionalAndLoopDefers: a conditional or in-loop defer is one +// chain member, assumed registered on every path (spec §4.3.1). +func TestCFG_ConditionalAndLoopDefers(t *testing.T) { + wantEdges(t, "CondDefer", + e(entry, "431:2", "fallthrough"), + e("431:2", "432:3", "true"), + e("431:2", "434:2", "false"), + e("432:3", "434:2", "fallthrough"), + e("434:2", "432:9", "return"), + e("432:9", exit, "defer_resume"), + e("432:9", exit, "exception"), + ) + // The loop's false edge into @exit (normalized to return) enters the chain. + wantEdges(t, "LoopDefer", + e(entry, "439:2", "fallthrough"), + e("439:2", "440:3", "true"), + e("439:2", "440:9", "return"), + e("440:3", "439:2", "loop_back"), + e("440:9", exit, "defer_resume"), + e("440:9", exit, "exception"), + ) +} + +// TestCFG_PanicInNestedBlock: a panic in a block nested in an if in a loop +// leaves only by its exception edge, never back to the loop. +func TestCFG_PanicInNestedBlock(t *testing.T) { + got := wantEdges(t, "LoopPanic", + e(entry, "446:2", "fallthrough"), + e("446:2", "447:2", "fallthrough"), + e("447:2", "448:3", "true"), + e("447:2", "455:2", "false"), + e("448:3", "450:5", "true"), + e("448:3", "453:3", "false"), + e("450:5", exit, "exception"), + e("453:3", "447:2", "loop_back"), + e("455:2", exit, "return"), + ) + if out := outEdges(got, "450:5"); len(out) != 1 { + t.Errorf("LoopPanic: panic out-edges = %v, want one exception", out) + } +} + +// TestCFG_AfterPanicPruned: everything after a panic is pruned, and a defer +// registered after it stays out of the chain. +func TestCFG_AfterPanicPruned(t *testing.T) { + wantEdges(t, "AfterPanic", + e(entry, "460:2", "fallthrough"), + e("460:2", "461:2", "fallthrough"), + e("461:2", "460:8", "exception"), + e("460:8", exit, "defer_resume"), + e("460:8", exit, "exception"), + ) + nodes, _ := cfgOf(t, "AfterPanic") + if keys := nodeKeys(nodes); strings.Join(keys, " ") != entry+" 460:2 460:8 461:2 "+exit { + t.Errorf("AfterPanic: nodes = %v, want [@entry 460:2 460:8 461:2 @exit]", keys) + } +} + +// ---- R9: regression locks for Task 6 constructs ---- + +func TestCFG_LabeledSwitchBreak(t *testing.T) { + // break Sw from a for nested in a clause leaves the switch. + wantEdges(t, "LabeledSwitchBreak", + e(entry, "303:2", "fallthrough"), + e("303:2", "304:2", "switch_case"), + e("303:2", "313:2", "switch_case"), + e("304:2", "305:3", "fallthrough"), + e("305:3", "306:4", "true"), + e("305:3", "311:3", "false"), + e("306:4", "307:5", "true"), + e("306:4", "309:4", "false"), + e("307:5", "313:2", "break"), + e("309:4", "305:3", "loop_back"), + e("311:3", "313:2", "fallthrough"), + e("313:2", exit, "return"), + ) +} + +func TestCFG_LabeledSelectBreak(t *testing.T) { + wantEdges(t, "LabeledSelectBreak", + e(entry, "319:2", "fallthrough"), + e("319:2", "320:2", "switch_case"), + e("320:2", "321:3", "fallthrough"), + e("321:3", "322:4", "true"), + e("321:3", "324:3", "false"), + e("322:4", "326:2", "break"), + e("324:3", "326:2", "fallthrough"), + e("326:2", exit, "return"), + ) +} + +func TestCFG_ContinueInSwitch(t *testing.T) { + // continue inside a switch targets the enclosing for header. + wantEdges(t, "ContinueInSwitch", + e(entry, "331:2", "fallthrough"), + e("331:2", "332:3", "true"), + e("331:2", "339:2", "false"), + e("332:3", "333:3", "fallthrough"), + e("333:3", "334:3", "switch_case"), + e("333:3", "337:3", "switch_case"), + e("334:3", "335:4", "fallthrough"), + e("335:4", "331:2", "continue"), + e("337:3", "331:2", "loop_back"), + e("339:2", exit, "return"), + ) +} + +func TestCFG_BreakInSelect(t *testing.T) { + // break inside a select leaves the select, not the for. + wantEdges(t, "BreakInSelect", + e(entry, "344:2", "fallthrough"), + e("344:2", "345:3", "true"), + e("344:2", "354:2", "false"), + e("345:3", "346:3", "switch_case"), + e("346:3", "347:4", "fallthrough"), + e("347:4", "348:5", "true"), + e("347:4", "350:4", "false"), + e("348:5", "352:3", "break"), + e("350:4", "352:3", "fallthrough"), + e("352:3", "344:2", "loop_back"), + e("354:2", exit, "return"), + ) +} + +func TestCFG_DefaultMiddle(t *testing.T) { + // default in the middle: still no header → after edge; fallthrough into + // and out of the default enters the next clause's body. + wantEdges(t, "DefaultMiddle", + e(entry, "359:2", "fallthrough"), + e("359:2", "360:2", "fallthrough"), + e("360:2", "361:2", "switch_case"), + e("360:2", "364:2", "switch_case"), + e("360:2", "367:2", "switch_case"), + e("361:2", "362:3", "fallthrough"), + e("362:3", "363:3", "fallthrough"), + e("363:3", "365:3", "fallthrough"), + e("364:2", "365:3", "fallthrough"), + e("365:3", "366:3", "fallthrough"), + e("366:3", "368:3", "fallthrough"), + e("367:2", "368:3", "fallthrough"), + e("368:3", "370:2", "fallthrough"), + e("370:2", exit, "return"), + ) +} + +func TestCFG_TypeSwitchNil(t *testing.T) { + // case nil: is an ordinary clause; no default, so header → after too. + wantEdges(t, "TypeSwitchNil", + e(entry, "375:2", "fallthrough"), + e("375:2", "376:2", "switch_case"), + e("375:2", "378:2", "switch_case"), + e("375:2", "381:2", "switch_case"), + e("376:2", "377:3", "fallthrough"), + e("377:3", exit, "return"), + e("378:2", "379:3", "fallthrough"), + e("379:3", exit, "return"), + e("381:2", exit, "return"), + ) +} + +func TestCFG_GotoInLoop(t *testing.T) { + // backward goto inside a loop body; the else-less if at the body's tail + // keeps its false kind into the header. + wantEdges(t, "GotoInLoop", + e(entry, "386:2", "fallthrough"), + e("386:2", "388:3", "true"), + e("386:2", "393:2", "false"), + e("388:3", "389:3", "fallthrough"), + e("389:3", "390:4", "true"), + e("389:3", "386:2", "false"), + e("390:4", "388:3", "fallthrough"), + e("393:2", exit, "return"), + ) +} + +// ---- CFG gates (spec §4.3) ---- + +// TestCFG_Gate runs the full CFG over every function and function literal +// body in testdata/dataflow and testdata/chi, with call keys built in L1 +// recording order, and checks the gates: exactly one @entry and @exit; every +// CFG node reachable from @entry and reaching @exit; every satellite's parent +// kept; every non-synthetic node has a valid Pos; edges sorted and deduped; +// two builds deep-equal. +func TestCFG_Gate(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata") + total, deferred := 0, 0 + for _, dir := range []string{"dataflow", "chi"} { + fset, cases := loadDiffCases(t, filepath.Join(root, dir)) + for _, c := range cases { + total++ + calls := gateCallKeys(fset, c.body) + nodes, keyOf := dataflow.BuildNodes(fset, c.body, calls) + kept, es := dataflow.BuildCFG(nodes, keyOf, c.body, c.info) + checkContract(t, c.name, kept, es) + deferred += gateOne(t, c.name, kept, es) + + nodes2, keyOf2 := dataflow.BuildNodes(fset, c.body, calls) + kept2, es2 := dataflow.BuildCFG(nodes2, keyOf2, c.body, c.info) + if !reflect.DeepEqual(kept, kept2) || !reflect.DeepEqual(es, es2) { + t.Errorf("%s: two builds differ", c.name) + } + } + } + if total < 100 || deferred == 0 { + t.Errorf("gate covered only %d bodies / %d deferred calls", total, deferred) + } + t.Logf("gate: %d bodies, %d deferred calls in exit chains", total, deferred) +} + +// gateOne checks one body's gates and returns its number of chain members. +func gateOne(t *testing.T, name string, nodes []dataflow.Node, es []dataflow.CfgEdge) int { + t.Helper() + entries, exits, chain := 0, 0, 0 + keys := map[string]bool{} + for _, n := range nodes { + keys[n.Key] = true + } + fwd := map[string]bool{entry: true} + for k := range closure(es)[entry] { + fwd[k] = true + } + rev := make([]dataflow.CfgEdge, len(es)) + for i, g := range es { + rev[i] = dataflow.CfgEdge{Src: g.Dst, Dst: g.Src, Kind: g.Kind} + } + bwd := map[string]bool{exit: true} + for k := range closure(rev)[exit] { + bwd[k] = true + } + for _, n := range nodes { + switch n.Key { + case entry: + entries++ + continue + case exit: + exits++ + continue + } + if !n.Pos.IsValid() || n.End < n.Pos || n.Stmt == nil { + t.Errorf("%s: node %s has no valid span", name, n.Key) + } + if n.Deferred { + chain++ + } + if n.Parent != "" { + if !keys[n.Parent] { + t.Errorf("%s: satellite %s kept without its parent %s", name, n.Key, n.Parent) + } + continue + } + if !fwd[n.Key] { + t.Errorf("%s: node %s unreachable from @entry", name, n.Key) + } + if !bwd[n.Key] { + t.Errorf("%s: node %s cannot reach @exit", name, n.Key) + } + } + if entries != 1 || exits != 1 { + t.Errorf("%s: %d @entry / %d @exit nodes, want 1 / 1", name, entries, exits) + } + if !fwd[exit] { + t.Errorf("%s: @exit unreachable from @entry", name) + } + return chain +} + +// gateCallKeys lists the key of every call in body in the L1 recording order +// (pre-order, an outer call before its nested calls; no function literal +// bodies), as +// v2.CallBodyKeys assigns them from each call's (line-directive adjusted) +// position. +func gateCallKeys(fset *token.FileSet, body *ast.BlockStmt) []string { + var calls []*ast.CallExpr + ast.Inspect(body, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncLit: + return false + case *ast.CallExpr: + calls = append(calls, n) + } + return true + }) + pos := make([][2]int, len(calls)) + for i, c := range calls { + p := fset.Position(c.Pos()) + pos[i] = [2]int{p.Line, p.Column} + } + return v2.CallBodyKeys(pos) +} + +// ---- canPanic rules (spec §4.3.2) ---- + +const panicSrc = `package p + +type T struct { + x int + p *T +} + +func (T) M() {} + +func f() int { return 0 } + +type Celsius float64 + +type Seq func(func(int) bool) + +func _(s []int, p *T, t T, i any, ch chan int, a, b int, fl float64, + r interface{ Close() error }, sh uint, seq func(func(int) bool), ns Seq, m map[int]int) { + _ = Celsius(fl) + _ = f() + _ = s[a] + _ = s[a:b] + _ = i.(int) + v, ok := i.(int) + _ = *p + _ = p.x + _ = t.x + ch <- a + _ = a / b + _ = a / 2 + _ = a % b + a /= b + _ = func() { _ = s[0] } + var q *T + _ = <-ch + _ = []*T{} + if a > 0 { + f() + } + for i := 0; i < len(s); i++ { + } + switch x := i.(type) { + case *T: + _ = x + } + switch a { + case f(): + } + defer f() + defer t.M() + defer p.M() + defer close(ch) + defer g(s[0]) + go f() + go g(f()) + _, _, _ = v, ok, q + _ = r.Close + defer r.Close() + go r.Close() + _ = a << b + _ = a >> sh + _ = a << 2 + a <<= b + a >>= sh + _ = [2]int(s) + _ = (*[2]int)(s) + _ = []int(s) + for x := range seq { + _ = x + } + for x := range ns { + _ = x + } + for range s { + } + _ = m[a] + _ = recover() + close(ch) +} + +func g(int) {} +` + +func TestCanPanic(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "p.go", panicSrc, 0) + if err != nil { + t.Fatal(err) + } + info := &types.Info{Types: map[ast.Expr]types.TypeAndValue{}, Uses: map[*ast.Ident]types.Object{}, Defs: map[*ast.Ident]types.Object{}, Selections: map[*ast.SelectorExpr]*types.Selection{}} + if _, err := (&types.Config{}).Check("p", fset, []*ast.File{file}, info); err != nil { + t.Fatal(err) + } + var body []ast.Stmt + for _, d := range file.Decls { + if fd, ok := d.(*ast.FuncDecl); ok && fd.Name.Name == "_" { + body = fd.Body.List + } + } + want := []bool{ + false, // conversion + true, // call + true, // index + true, // slice + true, // type assertion + false, // comma-ok type assertion + true, // dereference + true, // selector on a pointer + false, // selector on a value + true, // send + true, // non-constant divisor + false, // constant divisor + true, // non-constant % + true, // /= + false, // function literal body not entered + false, // pointer type in a declaration + false, // receive + false, // composite literal of pointer element type + false, // if header only (body has the call) + true, // for header calls len + false, // type switch guard + false, // expression switch with no init / tag panic + false, // defer f(): the call runs at exit + false, // defer t.M(): value receiver + true, // defer p.M(): *p is dereferenced at registration + false, // defer close(ch): close runs at exit + true, // defer g(s[0]): argument evaluated at registration + false, // go f() + true, // go g(f()): argument evaluated here + false, // _, _, _ = … + true, // interface method value (nil receiver) + true, // defer r.Close(): interface method value evaluated at registration + true, // go r.Close(): likewise + true, // shift by a signed non-constant count + false, // shift by an unsigned count + false, // shift by a constant + true, // <<= by a signed count + false, // >>= by an unsigned count + true, // slice → array conversion + true, // slice → array-pointer conversion + false, // slice → slice conversion + true, // range over a func (calls the iterator) + true, // range over a named func type + false, // range over a slice + true, // map index (nil map write / index of a map key type that panics) + true, // recover() + true, // close(ch) + } + if len(body) != len(want) { + t.Fatalf("snippet has %d statements, table %d", len(body), len(want)) + } + for i, s := range body { + if got := dataflow.CanPanic(s, info); got != want[i] { + t.Errorf("stmt %d (line %d): canPanic = %v, want %v", i, fset.Position(s.Pos()).Line, got, want[i]) + } + } + // Clause heads inspect their own list: `case *T:` is a type, `case f():` + // a call. + tsClause := body[20].(*ast.TypeSwitchStmt).Body.List[0] + swClause := body[21].(*ast.SwitchStmt).Body.List[0] + if dataflow.CanPanic(tsClause, info) || !dataflow.CanPanic(swClause, info) { + t.Errorf("clause heads: type case = %v, call case = %v; want false, true", dataflow.CanPanic(tsClause, info), dataflow.CanPanic(swClause, info)) + } + // A deferred call node itself (the CallExpr) can panic. + if !dataflow.CanPanic(body[25].(*ast.DeferStmt).Call, info) { + t.Errorf("deferred close(ch) call: canPanic = false, want true") + } + // The receive row stays false: `_ = <-ch` (stmt 16). + if dataflow.CanPanic(body[16], info) { + t.Errorf("receive: canPanic = true, want false") + } + // Without type info, every call and selector is conservatively true. + if !dataflow.CanPanic(body[0], nil) || !dataflow.CanPanic(body[8], nil) { + t.Errorf("nil info: conversion / value selector must be conservatively true") + } +} + +func TestCFG_EmptyBody(t *testing.T) { + // func cleanup() {}: falling off the end is a return. + wantEdges(t, "cleanup", e(entry, exit, "return")) +} + +func TestCFG_BodylessCallable(t *testing.T) { + nodes, keyOf := dataflow.BuildNodes(token.NewFileSet(), nil, nil) + kept, es := dataflow.BuildCFG(nodes, keyOf, nil, nil) + if len(es) != 0 || es == nil { + t.Errorf("nil body: edges = %#v, want empty non-nil", es) + } + if len(kept) != 0 || kept == nil { + t.Errorf("nil body: nodes = %#v, want empty non-nil", kept) + } +} + +// ---- differential test against golang.org/x/tools/go/cfg ---- + +// isBuiltinPanic reports whether call calls the builtin panic. +func isBuiltinPanic(call *ast.CallExpr, info *types.Info) bool { + id, ok := ast.Unparen(call.Fun).(*ast.Ident) + if !ok { + return false + } + b, ok := info.Uses[id].(*types.Builtin) + return ok && b.Name() == "panic" +} + +// diffCase is one function body for the differential test. +type diffCase struct { + name string + body *ast.BlockStmt + info *types.Info + fn ast.Node // the *ast.FuncDecl or *ast.FuncLit owning body +} + +func loadDiffCases(t *testing.T, dir string) (*token.FileSet, []diffCase) { + t.Helper() + fset := token.NewFileSet() + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo, + Dir: dir, + Fset: fset, + }, "./...") + if err != nil { + t.Fatalf("load %s: %v", dir, err) + } + var out []diffCase + for _, p := range pkgs { + if len(p.Errors) > 0 { + t.Fatalf("load %s: %v", p.PkgPath, p.Errors) + } + for _, f := range p.Syntax { + ast.Inspect(f, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncDecl: + if n.Body != nil { + name := n.Name.Name + if n.Recv != nil && len(n.Recv.List) > 0 { + name = types.ExprString(n.Recv.List[0].Type) + "." + name + } + out = append(out, diffCase{p.PkgPath + "." + name, n.Body, p.TypesInfo, n}) + } + case *ast.FuncLit: + pos := fset.Position(n.Pos()) + out = append(out, diffCase{fmt.Sprintf("%s.func@%d:%d", p.PkgPath, pos.Line, pos.Column), n.Body, p.TypesInfo, n}) + } + return true + }) + } + } + return fset, out +} + +// TestCFG_Differential checks that our CFG is at least as connected as +// go/cfg's: whenever go/cfg has a path from statement A to statement B, ours +// has a path from A's node to B's node. The reverse is not asserted (ours adds +// synthetic exception edges, panic edges and the defer chain). +// +// go/cfg's mayReturn is false exactly for a call to the builtin panic, which +// in ours has only its exception edge; every other call (os.Exit included) +// returns, as in ours (spec §4.3.2). Only live go/cfg blocks are compared: +// ours prunes what is unreachable from @entry. +// +// Mapping rule (go/cfg block Nodes → our keys), see diffKeyOf: +// - a statement, or a statement folded into a header (if/for/switch init, +// for post, type-switch guard) → keyOf[stmt]; +// - a `var` ValueSpec (go/cfg splits a DeclStmt per spec) → its DeclStmt's +// node; +// - a header expression (if/for cond, switch tag, range key/value/X) → the +// header node; +// - a switch case-list expression → the switch header. go/cfg tests case +// expressions in sequence (clause i's failed test falls into clause i+1's +// test); ours fans out header → every clause head, so the case tests are +// part of the header's dispatch; +// - a select comm statement → the select header. go/cfg evaluates every +// comm operation in one block before choosing (its builder's own TODO); +// ours folds each into its clause head, which are siblings; +// - a comm clause's receive LHS (go/cfg adds it in the chosen case body) → +// that clause head; +// - go/cfg's synthetic fall-off return (a *ast.ReturnStmt not in the +// source, at body.End()-1) → @exit; +// - an unlabeled or implicit empty statement → excluded (go/cfg adds every +// one; ours gives an unlabeled one no node, and leaves an implicit `L: }` +// unlinked: its label resolves to the continuation per ruling R6). An +// explicit labeled `L: ;` keeps its node and is compared. +// +// Defers: BuildNodes runs with real call keys (gateCallKeys, the L1 +// recording order), so deferred call nodes exist and every exit edge runs +// through the defer chain. go/cfg does not model defers: its DeferStmt is a +// plain statement (mapped to our registration node) and it has no node for +// the deferred call, so a deferred call node is never a compared endpoint +// (diffOne rejects one as a mapped key). Every go/cfg pair ending at its +// fall-off return (→ @exit) is still checked, so @exit must stay reachable +// through the chain from every live statement. +// +// Pairs whose two go/cfg nodes map to the same key are skipped. +func TestCFG_Differential(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata") + total, pairs, chained := 0, 0, 0 + for _, dir := range []string{"dataflow", "chi"} { + fset, cases := loadDiffCases(t, filepath.Join(root, dir)) + for _, c := range cases { + total++ + nodes, keyOf := dataflow.BuildNodes(fset, c.body, gateCallKeys(fset, c.body)) + kept, ours := dataflow.BuildCFG(nodes, keyOf, c.body, c.info) + checkContract(t, c.name, kept, ours) + pairs += diffOne(t.Errorf, c.name, c.body, c.info, keyOf, ours) + for _, n := range kept { + if n.Deferred { + chained++ + } + } + } + } + if total < 100 || pairs < 1000 || chained == 0 { + t.Errorf("differential covered only %d bodies / %d pairs / %d deferred calls", total, pairs, chained) + } + t.Logf("differential: %d bodies, %d reachable pairs checked, %d deferred calls chained", total, pairs, chained) +} + +// diffOne reports every go/cfg-reachable pair that ours does not reach and +// returns the number of distinct pairs checked. +func diffOne(errorf func(string, ...any), name string, body *ast.BlockStmt, info *types.Info, keyOf map[ast.Node]string, ours []dataflow.CfgEdge) int { + g := cfg.New(body, func(call *ast.CallExpr) bool { return !isBuiltinPanic(call, info) }) + km := diffKeyOf(body, keyOf) + deferred := map[string]bool{} + ast.Inspect(body, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncLit: + return false + case *ast.DeferStmt: + deferred[keyOf[n.Call]] = true + } + return true + }) + + // Map each live block's nodes to our keys, in block order. Dead blocks + // (code after a return / panic / goto) are pruned from ours. + blockKeys := make([][]string, len(g.Blocks)) + for _, b := range g.Blocks { + if !b.Live { + continue + } + for _, n := range b.Nodes { + k, ok := km[n] + if r, isRet := n.(*ast.ReturnStmt); !ok && isRet && r.Return == body.End()-1 { + k, ok = dataflow.ExitKey, true // go/cfg's synthetic fall-off return + } + if !ok { + errorf("%s: go/cfg node %T at %d has no mapped key", name, n, n.Pos()) + continue + } + if k == "" { // excluded by the mapping rule + continue + } + if deferred[k] { + errorf("%s: go/cfg node %T maps to deferred call %s", name, n, k) + continue + } + blockKeys[b.Index] = append(blockKeys[b.Index], k) + } + } + + // Block-level transitive closure (paths of length >= 1). + reachB := make([]map[int32]bool, len(g.Blocks)) + for _, b := range g.Blocks { + seen := map[int32]bool{} + stack := append([]*cfg.Block(nil), b.Succs...) + for len(stack) > 0 { + x := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if seen[x.Index] { + continue + } + seen[x.Index] = true + stack = append(stack, x.Succs...) + } + reachB[b.Index] = seen + } + + oursReach := closure(ours) + type pair struct{ a, b string } + checked := map[pair]bool{} + check := func(a, b string) { + if a == b || checked[pair{a, b}] { + return + } + checked[pair{a, b}] = true + if !oursReach[a][b] { + errorf("%s: go/cfg reaches %s → %s; ours does not", name, a, b) + } + } + for _, x := range g.Blocks { + ks := blockKeys[x.Index] + for i, a := range ks { + for _, b := range ks[i+1:] { // later in the same block + check(a, b) + } + for y := range reachB[x.Index] { + for _, b := range blockKeys[y] { + check(a, b) + } + } + } + } + return len(checked) +} + +// closure returns, for every key, the set of keys reachable by a path of +// length >= 1. +func closure(es []dataflow.CfgEdge) map[string]map[string]bool { + succ := map[string][]string{} + for _, e := range es { + succ[e.Src] = append(succ[e.Src], e.Dst) + } + out := map[string]map[string]bool{} + for src := range succ { + seen := map[string]bool{} + stack := append([]string(nil), succ[src]...) + for len(stack) > 0 { + x := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if seen[x] { + continue + } + seen[x] = true + stack = append(stack, succ[x]...) + } + out[src] = seen + } + return out +} + +// diffKeyOf extends keyOf with the go/cfg-only nodes per the mapping rule of +// TestCFG_Differential. Function literal bodies are not entered: go/cfg treats +// a FuncLit as an expression. +func diffKeyOf(body *ast.BlockStmt, keyOf map[ast.Node]string) map[ast.Node]string { + km := make(map[ast.Node]string, len(keyOf)) + for n, k := range keyOf { + km[n] = k + } + put := func(n ast.Node, k string) { + if n != nil && k != "" { + km[n] = k + } + } + ast.Inspect(body, func(n ast.Node) bool { + switch s := n.(type) { + case *ast.FuncLit: + return false + case *ast.EmptyStmt: + // go/cfg adds every empty statement. Ours has a node only for a + // label target, and an implicit one (`L: }`) is not linked + // (ruling R6: its label resolves to the continuation). Excluded. + if s.Implicit || keyOf[s] == "" { + km[s] = "" + } + case *ast.DeclStmt: + for _, sp := range s.Decl.(*ast.GenDecl).Specs { + put(sp, keyOf[s]) + } + case *ast.IfStmt: + put(s.Cond, keyOf[s]) + case *ast.ForStmt: + put(s.Cond, keyOf[s]) + case *ast.RangeStmt: + put(s.X, keyOf[s]) + put(s.Key, keyOf[s]) + put(s.Value, keyOf[s]) + case *ast.SwitchStmt: + put(s.Tag, keyOf[s]) + for _, c := range s.Body.List { + for _, x := range c.(*ast.CaseClause).List { + put(x, keyOf[s]) + } + } + case *ast.SelectStmt: + for _, c := range s.Body.List { + cc := c.(*ast.CommClause) + put(cc.Comm, keyOf[s]) + if as, ok := cc.Comm.(*ast.AssignStmt); ok { + put(as.Lhs[0], keyOf[cc]) + } + } + } + return true + }) + return km +} + +// TestCFG_DifferentialCatchesMissingEdge guards the differential itself: a +// CFG with one edge removed must be reported. +func TestCFG_DifferentialCatchesMissingEdge(t *testing.T) { + fx := loadFlow(t) + body := fx.decl(t, "Labeled").Body + nodes, keyOf := dataflow.BuildNodes(fx.fset, body, nil) + _, full := dataflow.BuildCFG(nodes, keyOf, body, fx.info) + var cut []dataflow.CfgEdge // drop every edge into the return + for _, g := range full { + if g.Dst != lbRet { + cut = append(cut, g) + } + } + var errs []string + diffOne(func(f string, a ...any) { errs = append(errs, fmt.Sprintf(f, a...)) }, "Labeled(cut)", body, fx.info, keyOf, cut) + if !strings.Contains(strings.Join(errs, "\n"), lbInit+" → "+lbRet) { + t.Errorf("differential missed the cut edges; got %v", errs) + } +} diff --git a/internal/dataflow/defuse.go b/internal/dataflow/defuse.go new file mode 100644 index 0000000..4162c57 --- /dev/null +++ b/internal/dataflow/defuse.go @@ -0,0 +1,504 @@ +package dataflow + +import ( + "go/ast" + "go/token" + "go/types" + "sort" +) + +// DefUse is the def/use summary of one body node (spec §4.5). +// +// - Defs are strong definitions: a reaching def d with p.Covers(d) for a p +// in Defs is killed (`x = …` kills x, x.f, x[*]; `x.f = …` kills x.f…). +// - WeakDefs are may-definitions: they reach on, but kill nothing (`m[k] = +// v`, `*p = v`, a k-truncated write, what a callee may write through a +// reference argument, the `:` pseudo path). +// - Uses are the paths the node reads. +// +// A strong def never has a wildcard step (any write through an index, a +// dereference or past k is weak), so a kill never removes more than the +// written path covers. The `:` pseudo path (nil Base) appears only as +// a registration's weak def and its deferred call node's use. Each list is +// deduplicated and sorted by (Spell, base position). +type DefUse struct { + Defs, WeakDefs, Uses []Path +} + +// DeferPrefix starts the pseudo path a defer registration weak-defines and its +// deferred call node uses: ":" + the deferred call node's Key. +const DeferPrefix = ":" + +// DefUseOf returns the def/use sets of body node n of callable fn (a +// *ast.FuncDecl or *ast.FuncLit). k is the access-path limit (PathOf). +// entryBases is EntryBases(fn, info), what @entry defines. keyOf is +// BuildNodes' map, used to find the deferred call node's key from a defer +// registration (keyOf[DeferStmt.Call]). info must be non-nil. +// +// Rules by node: +// +// - @entry defines a base path for every object in entryBases; @exit has +// none. +// - A satellite call node (Parent != "") has an EMPTY DefUse. Its uses and +// weak defs belong to its host statement (the Parent), whose DefUse is +// computed from the host's whole expression, which contains the call — +// so a flow is counted once, at the node on the CFG. Task 10 relies on +// this: satellites are never CFG nodes and need no gen/kill. +// - A header or clause head reads only its own header fields, never its +// body (R8): if Init+Cond; for Init+Cond+Post; range Key/Value (defs) and +// X (use); switch Init+Tag; type switch Init+Assign (defining each +// clause's implicit variable); case List; comm clause Comm; select none. +// A for header runs on entry and on every back edge, so only a variable +// both Init and Post strong-define stays strong; other folded defs are +// weak. Range Key/Value are weak with `=` (zero iterations leave them), +// strong with `:=`. If and switch inits run once per header: strong. +// - A bare call node (*ast.ExprStmt, `panic(x)` included) and a `go` node +// (*ast.GoStmt) apply the call rule below to their call — that covers the +// calls nested in its arguments, whose satellite nodes carry nothing. +// - defer (spec §4.3.1). The registration node (*ast.DeferStmt) uses the +// deferred call's function value / receiver and arguments (with the calls +// nested in them) and weak-defines the pseudo path ":", +// = keyOf[DeferStmt.Call], the deferred call node's Key. The +// deferred call node (Deferred) uses that pseudo path, every named result +// of fn and, for `defer func(){…}()`, what the closure captures; it +// weak-defines what the call may write through (call rule), uses those +// same paths (it reads receiver and argument state at exit, not at +// registration) and never strong-defines anything (R11: chain +// membership is unconditional). +// - Assignment: each LHS is a def (`:=` reuses an existing object for an +// already-declared name); `_` is none. An ident or a field path is a +// strong def; a path through an index, a dereference or past k is weak. +// Locating the target reads its index expressions and the pointer / +// map / slice it goes through (`m[k] = v` uses k, m; `*q = v` and +// `p.f = v` with p a pointer use q / p). Compound assignment and ++/-- +// also use the target. `var x, y = a, b` defines x, y. +// - `ch <- v` weak-defines ch[*] and uses ch, v. A receive (`<-ch`, a +// range over a channel) uses ch and ch[*]. +// - return uses its operands. With named results, a bare return uses them +// and a return with operands defines them. +// +// Expression uses: a path is used as a whole (m[k] uses m[*] and k); a +// function literal uses every variable it captures, never its body. The call +// rule: a call uses its function value / receiver and arguments; it +// weak-defines a.* for each argument of pointer, slice, map, chan, func or +// interface type (`&x` gives x.*; a slice expression of a path gives its +// base's elements: buf[:n], arr[:] and (*pa)[i:] give buf[*], arr[*], pa[*]) +// and recv.* for a receiver that is +// reference-typed (the same kinds) or bound to a pointer-receiver method. A value-typed struct +// argument is not weak-defined. Conversions write nothing; builtins write +// nothing except append(dst, …), copy(dst, …), delete(m, …) and clear(x), +// which weak-define dst[*], m[*], x[*] (copy(dst[i:], …) and append(dst[:0], +// …) give dst[*] too: append may write into dst's backing array); append's +// result also flows to the assignment's LHS. +func DefUseOf(n Node, info *types.Info, k int, entryBases []types.Object, fn ast.Node, keyOf map[ast.Node]string) DefUse { + c := &collector{info: info, k: k, seen: map[pathKind]bool{}} + switch { + case n.Kind == KindEntry: + for _, b := range entryBases { + c.add(kindDef, Path{Base: b, Spell: baseSpell(b)}) + } + case n.Kind == KindExit, n.Parent != "", info == nil: + case n.Deferred: + call, ok := n.Stmt.(*ast.CallExpr) + if !ok { + break + } + c.add(kindUse, Path{Spell: DeferPrefix + n.Key}) + for _, r := range namedResults(fn, info) { + c.add(kindUse, Path{Base: r, Spell: r.Name()}) + } + if lit, ok := ast.Unparen(call.Fun).(*ast.FuncLit); ok { + c.captures(lit) + } + c.callWrites(call) + // The call runs at exit and reads, then, the state behind what it may + // write through: a write after the registration (`defer w.Flush()` + // then `w.buf = …`) reaches it. + for _, p := range c.du.WeakDefs { + c.add(kindUse, p) + } + default: + c.fn = fn + c.keyOf = keyOf + c.stmt(n.Stmt) + } + sortPaths(c.du.Defs) + sortPaths(c.du.WeakDefs) + sortPaths(c.du.Uses) + return c.du +} + +type pathKind struct { + p Path + kind int +} + +const ( + kindDef = iota + kindWeak + kindUse +) + +type collector struct { + info *types.Info + k int + fn ast.Node + keyOf map[ast.Node]string + du DefUse + seen map[pathKind]bool + weakOnly bool // record every def as weak (a range with `=`) +} + +// fork returns an empty collector with c's settings. +func (c *collector) fork() *collector { + return &collector{info: c.info, k: c.k, fn: c.fn, keyOf: c.keyOf, seen: map[pathKind]bool{}} +} + +// mergeLoop adds a's facts to c, keeping a strong def strong only when other +// strong-defines the same path too. +func (c *collector) mergeLoop(a, other *collector) { + for _, p := range a.du.Defs { + if other.seen[pathKind{p, kindDef}] { + c.add(kindDef, p) + } else { + c.add(kindWeak, p) + } + } + for _, p := range a.du.WeakDefs { + c.add(kindWeak, p) + } + for _, p := range a.du.Uses { + c.add(kindUse, p) + } +} + +// sortPaths orders ps by spelling, then base position (shadowed variables +// share a spelling), so DefUse lists are sorted, not just deterministic. +func sortPaths(ps []Path) { + pos := func(p Path) token.Pos { + if p.Base == nil { + return token.NoPos + } + return p.Base.Pos() + } + sort.SliceStable(ps, func(i, j int) bool { + if ps[i].Spell != ps[j].Spell { + return ps[i].Spell < ps[j].Spell + } + return pos(ps[i]) < pos(ps[j]) + }) +} + +func (c *collector) add(kind int, p Path) { + if c.seen[pathKind{p, kind}] { + return + } + c.seen[pathKind{p, kind}] = true + switch kind { + case kindDef: + c.du.Defs = append(c.du.Defs, p) + case kindWeak: + c.du.WeakDefs = append(c.du.WeakDefs, p) + default: + c.du.Uses = append(c.du.Uses, p) + } +} + +func (c *collector) addUse(b pathB) { c.add(kindUse, b.limit(c.k)) } +func (c *collector) addWeak(b pathB) { c.add(kindWeak, b.limit(c.k)) } + +// stmt collects a node's statement (or clause head) — header fields only. +func (c *collector) stmt(s ast.Node) { + switch s := s.(type) { + case *ast.AssignStmt: + compound := s.Tok != token.ASSIGN && s.Tok != token.DEFINE + for _, r := range s.Rhs { + c.use(r) + } + for _, l := range s.Lhs { + c.assignTo(l, compound) + } + case *ast.IncDecStmt: + c.assignTo(s.X, true) + case *ast.DeclStmt: + gd, ok := s.Decl.(*ast.GenDecl) + if !ok || gd.Tok != token.VAR { + return + } + for _, sp := range gd.Specs { + vs := sp.(*ast.ValueSpec) + for _, v := range vs.Values { + c.use(v) + } + for _, name := range vs.Names { + c.assignTo(name, false) + } + } + case *ast.SendStmt: + c.use(s.Value) + if b, ok := pathOf(s.Chan, c.info); ok { + c.addUse(b) + c.indexReads(s.Chan) + c.addWeak(b.with("[*]")) + } else { + c.use(s.Chan) + } + case *ast.ExprStmt: + c.use(s.X) + case *ast.CallExpr: + c.call(s) // not produced by BuildNodes off a satellite; kept total + case *ast.GoStmt: + c.call(s.Call) + case *ast.DeferStmt: + c.callReads(s.Call, false) + if key, ok := c.keyOf[s.Call]; ok { + c.add(kindWeak, Path{Spell: DeferPrefix + key}) + } + case *ast.ReturnStmt: + for _, r := range s.Results { + c.use(r) + } + kind := kindDef + if len(s.Results) == 0 { + kind = kindUse + } + for _, r := range namedResults(c.fn, c.info) { + c.add(kind, Path{Base: r, Spell: r.Name()}) + } + case *ast.IfStmt: + c.stmt(s.Init) + c.use(s.Cond) + case *ast.ForStmt: + // The header node runs on loop entry (where Init runs) and on every + // back edge (where Post runs), so a strong def here would kill on the + // path where its statement does not run: only a variable both Init + // and Post define is strong; every other folded def is weak. + init, post := c.fork(), c.fork() + init.stmt(s.Init) + post.stmt(s.Post) + c.use(s.Cond) + c.mergeLoop(init, post) + c.mergeLoop(post, init) + case *ast.RangeStmt: + // `for k, v = range x` may run zero times, and then leaves k, v + // untouched: weak. `:=` declares fresh objects per loop: strong. + c.weakOnly = s.Tok == token.ASSIGN + for _, x := range []ast.Expr{s.Key, s.Value} { + if x != nil { + c.assignTo(x, false) + } + } + c.weakOnly = false + if isChan(c.info.TypeOf(s.X)) { + c.recv(s.X) + } else { + c.use(s.X) + } + case *ast.SwitchStmt: + c.stmt(s.Init) + c.use(s.Tag) + case *ast.TypeSwitchStmt: + c.stmt(s.Init) + c.stmt(s.Assign) // `t := v.(type)`: t has no object, only v is read + for _, cl := range s.Body.List { + if obj := c.info.Implicits[cl]; obj != nil && obj.Name() != "_" { + c.add(kindDef, Path{Base: obj, Spell: obj.Name()}) + } + } + case *ast.CaseClause: + for _, x := range s.List { + c.use(x) + } + case *ast.CommClause: + c.stmt(s.Comm) + } + // SelectStmt, BranchStmt, EmptyStmt, nil: nothing. +} + +// assignTo records lhs as an assignment target. A target is never use-only +// while it has a nameable root: one whose base is unresolved (type errors) +// weak-defines root.*, and one pathOf cannot walk to its root variable (x.(*T).f +// = v) weak-defines that root's x.*. Only `_` and a target rooted in a call or +// a literal (`*f() = v`), which name no variable of this callable, define +// nothing; their reads are still collected. +func (c *collector) assignTo(lhs ast.Expr, compound bool) { + b, ok := pathOf(lhs, c.info) + if !ok { + c.use(lhs) + if r, ok := c.rootOf(lhs); ok { + c.addWeak(r.with(".*")) + } + return + } + c.targetReads(ast.Unparen(lhs)) + if compound { + c.addUse(b) + } + if b.base == nil { + // An unresolved base: identity is the spelling alone, so the write + // is a may-write of the whole root and kills nothing. + c.addWeak(pathB{head: b.head}.with(".*")) + return + } + if c.weakOnly || b.weak(c.k) { + c.addWeak(b) + } else { + c.add(kindDef, b.limit(c.k)) + } +} + +// rootOf returns the base of the variable assignment target e is rooted in, +// walking selectors, indexes, slices, dereferences, type assertions and +// parentheses; false for `_` and a target rooted in a call or a literal. +func (c *collector) rootOf(e ast.Expr) (pathB, bool) { + for { + switch x := ast.Unparen(e).(type) { + case *ast.Ident: + return baseOf(x, c.info) + case *ast.SelectorExpr: + if b, ok, qualified := qualifiedBase(x, c.info); qualified { + return b, ok + } + e = x.X + case *ast.IndexExpr: + e = x.X + case *ast.SliceExpr: + e = x.X + case *ast.StarExpr: + e = x.X + case *ast.TypeAssertExpr: + e = x.X + default: + return pathB{}, false + } + } +} + +// targetReads collects what locating assignment target e (a path) reads: its +// index expressions, and the pointer / map / slice value it goes through. +func (c *collector) targetReads(e ast.Expr) { + switch e := e.(type) { + case *ast.SelectorExpr: + if isQualified(e, c.info) { + return + } + x := ast.Unparen(e.X) + if st, ok := x.(*ast.StarExpr); ok { + c.use(st.X) + } else if sel := c.info.Selections[e]; sel != nil && sel.Indirect() { + c.use(x) + } else { + c.targetReads(x) + } + case *ast.IndexExpr: + c.use(e.Index) + x := ast.Unparen(e.X) + if st, ok := x.(*ast.StarExpr); ok { + c.use(st.X) + } else if _, arr := underlying(c.info.TypeOf(x)).(*types.Array); arr { + c.targetReads(x) + } else { + c.use(x) + } + case *ast.StarExpr: + c.use(e.X) + case *ast.ParenExpr: + c.targetReads(ast.Unparen(e)) + } +} + +// use collects the reads of expression e. +func (c *collector) use(e ast.Expr) { + if e == nil { + return + } + e = ast.Unparen(e) + if b, ok := pathOf(e, c.info); ok { + c.addUse(b) + c.indexReads(e) + return + } + switch e := e.(type) { + case *ast.CallExpr: + c.call(e) + case *ast.FuncLit: + c.captures(e) + case *ast.UnaryExpr: + if e.Op == token.ARROW { + c.recv(e.X) + } else { + c.use(e.X) + } + case *ast.SelectorExpr: + if !isQualified(e, c.info) { + c.use(e.X) // a method value reads its receiver + } + default: + ast.Inspect(e, func(n ast.Node) bool { + if n == ast.Node(e) { + return true + } + if x, ok := n.(ast.Expr); ok { + c.use(x) + return false + } + return true + }) + } +} + +// indexReads collects the index expressions inside path expression e. +func (c *collector) indexReads(e ast.Expr) { + switch e := ast.Unparen(e).(type) { + case *ast.SelectorExpr: + if !isQualified(e, c.info) { + c.indexReads(e.X) + } + case *ast.IndexExpr: + c.use(e.Index) + c.indexReads(e.X) + case *ast.StarExpr: + c.indexReads(e.X) + } +} + +// recv collects a receive from channel expression ch: ch and ch[*]. +func (c *collector) recv(ch ast.Expr) { + b, ok := pathOf(ch, c.info) + if !ok { + c.use(ch) + return + } + c.addUse(b) + c.addUse(b.with("[*]")) + c.indexReads(ch) +} + +// captures uses every variable function literal lit captures. +func (c *collector) captures(lit *ast.FuncLit) { + for _, v := range freeVars(lit, c.info) { + c.add(kindUse, Path{Base: v, Spell: v.Name()}) + } +} + +// isQualified reports whether e is a qualified identifier pkg.Name. +func isQualified(e *ast.SelectorExpr, info *types.Info) bool { + x, ok := e.X.(*ast.Ident) + if !ok { + return false + } + _, ok = info.Uses[x].(*types.PkgName) + return ok +} + +func underlying(t types.Type) types.Type { + if t == nil { + return nil + } + return t.Underlying() +} + +func isChan(t types.Type) bool { + _, ok := underlying(t).(*types.Chan) + return ok +} diff --git a/internal/dataflow/defuse_calls.go b/internal/dataflow/defuse_calls.go new file mode 100644 index 0000000..2e223fa --- /dev/null +++ b/internal/dataflow/defuse_calls.go @@ -0,0 +1,161 @@ +package dataflow + +import ( + "go/ast" + "go/token" + "go/types" +) + +// The call rule of DefUseOf (spec §4.5), split into what a call reads and +// what it may write, because a `defer` evaluates the reads at registration +// and performs the writes in the exit chain. + +// call collects a call evaluated and run at the same node. +func (c *collector) call(call *ast.CallExpr) { + c.callReads(call, true) + c.callWrites(call) +} + +// callReads collects what evaluating call's operands reads: the function +// value (a method value's receiver, a func-typed variable, or — when +// captures is set — what a function literal captures) and the arguments, +// with the calls nested in them. A defer registration passes captures=false: +// a deferred closure reads its captures when it runs, at the exit. +func (c *collector) callReads(call *ast.CallExpr, captures bool) { + if _, lit := ast.Unparen(call.Fun).(*ast.FuncLit); !lit || captures { + c.use(call.Fun) + } + for _, a := range call.Args { + c.use(a) + } +} + +// callWrites collects the weak defs of call itself: a.* for each +// reference-typed argument (&x gives x.*), recv.* for a reference-typed +// receiver or a pointer-receiver method, and the element writes of append, +// copy, delete and clear. append(dst, …) may write into dst's backing array +// (spare capacity, or dst[:0]), so it weak-defines dst[*] besides its result +// flowing to the assignment. Conversions and other builtins write nothing. +func (c *collector) callWrites(call *ast.CallExpr) { + if isType(call.Fun, c.info) { + return + } + if name, ok := builtinName(call.Fun, c.info); ok { + switch name { + case "append", "copy", "delete", "clear": + if len(call.Args) > 0 { + x, _ := sliceBase(call.Args[0]) + if b, ok := pathOf(x, c.info); ok { + c.addWeak(b.with("[*]")) + } + } + } + return + } + if sel, ok := ast.Unparen(call.Fun).(*ast.SelectorExpr); ok { + if s := c.info.Selections[sel]; s != nil && s.Kind() == types.MethodVal { + if ptrRecv(s.Obj()) || refTyped(c.info.TypeOf(sel.X)) { + c.writeThrough(sel.X) + } + } + } + for _, a := range call.Args { + if refTyped(c.info.TypeOf(a)) { + c.writeThrough(a) + } + } +} + +// writeThrough weak-defines a.* (x.* for &x) when a is a path, and base[*] +// when a slices a path (buf[:n], arr[:], (*p)[i:]): the callee writes the +// elements the slice shares with its base. +func (c *collector) writeThrough(a ast.Expr) { + if base, sliced := sliceBase(a); sliced { + if b, ok := pathOf(base, c.info); ok { + c.addWeak(b.with("[*]")) + } + return + } + a = ast.Unparen(a) + if u, ok := a.(*ast.UnaryExpr); ok && u.Op == token.AND { + a = u.X + } + if b, ok := pathOf(a, c.info); ok { + c.addWeak(b.with(".*")) + } +} + +// sliceBase strips slice expressions (and the parentheses, & and * between +// them) from e down to the sliced operand: buf[2:][:n] gives buf, (&arr)[:] +// and (*p)[:] give arr and p. sliced reports whether any slice expression was +// stripped; when it is false e is returned unchanged. The slice's elements are +// its base's elements, whether the base is a slice, an array or a pointer to +// an array, so a write through the slice is a write to base[*]. +func sliceBase(e ast.Expr) (base ast.Expr, sliced bool) { + base = e + for { + x := ast.Unparen(base) + s, ok := x.(*ast.SliceExpr) + if !ok { + break + } + base, sliced = ast.Unparen(s.X), true + switch b := base.(type) { + case *ast.UnaryExpr: + if b.Op == token.AND { + base = b.X + } + case *ast.StarExpr: + base = b.X + } + } + if !sliced { + return e, false + } + return base, true +} + +// builtinName returns the name of the builtin fun calls (panic, len, copy, +// unsafe.Sizeof, …), resolved through info so a shadowing local is not one. +func builtinName(fun ast.Expr, info *types.Info) (string, bool) { + var id *ast.Ident + switch f := ast.Unparen(fun).(type) { + case *ast.Ident: + id = f + case *ast.SelectorExpr: + id = f.Sel + default: + return "", false + } + b, ok := info.Uses[id].(*types.Builtin) + if !ok { + return "", false + } + return b.Name(), true +} + +// ptrRecv reports whether method obj has a pointer receiver. +func ptrRecv(obj types.Object) bool { + f, ok := obj.(*types.Func) + if !ok { + return false + } + recv := f.Signature().Recv() + return recv != nil && isPointer(recv.Type()) +} + +func isPointer(t types.Type) bool { + _, ok := underlying(t).(*types.Pointer) + return ok +} + +// refTyped reports whether a value of type t lets a callee write state the +// caller sees: pointer, slice, map, chan, func or interface (a type +// parameter's underlying type is its constraint interface, so it counts). +func refTyped(t types.Type) bool { + switch underlying(t).(type) { + case *types.Pointer, *types.Slice, *types.Map, *types.Chan, *types.Signature, *types.Interface: + return true + } + return false +} diff --git a/internal/dataflow/defuse_entry.go b/internal/dataflow/defuse_entry.go new file mode 100644 index 0000000..a52543d --- /dev/null +++ b/internal/dataflow/defuse_entry.go @@ -0,0 +1,136 @@ +package dataflow + +import ( + "go/ast" + "go/types" + "sort" +) + +// What @entry defines (EntryBases) and the callable-level bases DefUseOf +// reads: named results and a function literal's captured free variables. + +// namedResults returns fn's named result variables ("_" excluded). +func namedResults(fn ast.Node, info *types.Info) []types.Object { + ft := funcType(fn) + if ft == nil || ft.Results == nil { + return nil + } + return fieldObjs(ft.Results, info) +} + +func funcType(fn ast.Node) *ast.FuncType { + switch fn := fn.(type) { + case *ast.FuncDecl: + return fn.Type + case *ast.FuncLit: + return fn.Type + } + return nil +} + +// fieldObjs returns the objects of fl's named fields, in order ("_" and +// unnamed fields excluded). +func fieldObjs(fl *ast.FieldList, info *types.Info) []types.Object { + if fl == nil { + return nil + } + var out []types.Object + for _, f := range fl.List { + for _, name := range f.Names { + if obj := info.Defs[name]; obj != nil && name.Name != "_" { + out = append(out, obj) + } + } + } + return out +} + +// EntryBases returns what @entry of callable fn (a *ast.FuncDecl or +// *ast.FuncLit) defines, in a deterministic order: +// +// 1. the receiver, the parameters and the named results in declaration +// order, then — for a function literal — every captured free variable +// (a variable declared outside the literal, not at package scope) by +// position, then name; +// 2. every package-level variable fn's body reads or writes (not those of +// nested function literals: those are their own callables), sorted by +// package path, then name — positions of other files depend on load +// order, so they are not used across files. +// +// "_" and unnamed parameters are skipped. A nil info yields nil. +func EntryBases(fn ast.Node, info *types.Info) []types.Object { + if info == nil { + return nil + } + var out []types.Object + var body *ast.BlockStmt + switch fn := fn.(type) { + case *ast.FuncDecl: + out = append(out, fieldObjs(fn.Recv, info)...) + body = fn.Body + case *ast.FuncLit: + body = fn.Body + default: + return nil + } + if ft := funcType(fn); ft != nil { + out = append(out, fieldObjs(ft.Params, info)...) + out = append(out, fieldObjs(ft.Results, info)...) + } + if lit, ok := fn.(*ast.FuncLit); ok { + out = append(out, freeVars(lit, info)...) + } + if body == nil { + return out + } + seen := map[types.Object]bool{} + var globals []types.Object + ast.Inspect(body, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncLit: + return false + case *ast.Ident: + if obj := info.Uses[n]; isPkgVar(obj) && !seen[obj] { + seen[obj] = true + globals = append(globals, obj) + } + } + return true + }) + sort.Slice(globals, func(i, j int) bool { + a, b := globals[i], globals[j] + if a.Pkg().Path() != b.Pkg().Path() { + return a.Pkg().Path() < b.Pkg().Path() + } + return a.Name() < b.Name() + }) + return append(out, globals...) +} + +// freeVars returns the variables function literal lit captures: every +// variable its body (nested literals included) uses that is declared outside +// lit and not at package scope, by position, then name. +func freeVars(lit *ast.FuncLit, info *types.Info) []types.Object { + seen := map[types.Object]bool{} + var out []types.Object + ast.Inspect(lit.Body, func(n ast.Node) bool { + id, ok := n.(*ast.Ident) + if !ok { + return true + } + v, ok := info.Uses[id].(*types.Var) + if !ok || v.IsField() || isPkgVar(v) || seen[v] || (v.Pos() >= lit.Pos() && v.Pos() < lit.End()) { + return true + } + seen[v] = true + out = append(out, v) + return true + }) + sort.Slice(out, func(i, j int) bool { + if out[i].Pos() != out[j].Pos() { + return out[i].Pos() < out[j].Pos() + } + return out[i].Name() < out[j].Name() + }) + return out +} diff --git a/internal/dataflow/defuse_test.go b/internal/dataflow/defuse_test.go new file mode 100644 index 0000000..d23d0ed --- /dev/null +++ b/internal/dataflow/defuse_test.go @@ -0,0 +1,421 @@ +package dataflow_test + +import ( + "go/ast" + "go/types" + "reflect" + "sort" + "strings" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" +) + +const kTest = 3 + +// spells is the sorted spellings of ps. +func spells(ps []dataflow.Path) []string { + out := []string{} + for _, p := range ps { + out = append(out, p.Spell) + } + sort.Strings(out) + return out +} + +func wantSet(t *testing.T, what string, got []dataflow.Path, want ...string) { + t.Helper() + sort.Strings(want) + if want == nil { + want = []string{} + } + if g := spells(got); !reflect.DeepEqual(g, want) { + t.Errorf("%s = %v, want %v", what, g, want) + } +} + +func wantDU(t *testing.T, what string, du dataflow.DefUse, defs, weak, uses []string) { + t.Helper() + wantSet(t, what+" Defs", du.Defs, defs...) + wantSet(t, what+" WeakDefs", du.WeakDefs, weak...) + wantSet(t, what+" Uses", du.Uses, uses...) +} + +func ss(s ...string) []string { return s } + +// nodes builds the body nodes of hermetic function fn with its call keys. +func (fx *duFixture) nodes(t *testing.T, fn string) (map[string]dataflow.Node, map[ast.Node]string, []types.Object) { + t.Helper() + d, ok := fx.decls[fn] + if !ok { + t.Fatalf("no function %s", fn) + } + nodes, keyOf := dataflow.BuildNodes(fx.fset, d.Body, gateCallKeys(fx.fset, d.Body)) + byKey := map[string]dataflow.Node{} + for _, n := range nodes { + byKey[n.Key] = n + } + return byKey, keyOf, dataflow.EntryBases(d, fx.info) +} + +// du is DefUseOf the node of fn keyed at the first occurrence of sub (plus +// delta bytes) in duSrc. +func (fx *duFixture) du(t *testing.T, fn, sub string, delta int) (dataflow.DefUse, string) { + t.Helper() + byKey, keyOf, bases := fx.nodes(t, fn) + key := fx.keyAt(t, sub, delta) + n, ok := byKey[key] + if !ok { + t.Fatalf("%s: no node %s (%q)", fn, key, sub) + } + return dataflow.DefUseOf(n, fx.info, kTest, bases, fx.decls[fn], keyOf), key +} + +// flowDU is DefUseOf the node keyed key of flow-fixture function fn. +func flowDU(t *testing.T, fn, key string) dataflow.DefUse { + t.Helper() + fx := loadFlow(t) + _, byKey, keyOf := build(t, fn) + n, ok := byKey[key] + if !ok { + t.Fatalf("%s: no node %s; have %v", fn, key, keysOf(byKey)) + } + d := fx.decl(t, fn) + return dataflow.DefUseOf(n, fx.info, kTest, dataflow.EntryBases(d, fx.info), d, keyOf) +} + +func TestDefUse_AssignStrongDef(t *testing.T) { + // Bump: y = y + 1. + wantDU(t, "Bump 148:3", flowDU(t, "Bump", "148:3"), ss("y"), nil, ss("y")) + // ForLoop header: init and post fold in. + wantDU(t, "ForLoop 27:2", flowDU(t, "ForLoop", "27:2"), ss("i"), nil, ss("i", "n")) + // s += i: compound assignment defines and uses. + wantDU(t, "ForLoop 28:3", flowDU(t, "ForLoop", "28:3"), ss("s"), nil, ss("i", "s")) + wantDU(t, "IfElse 16:2", flowDU(t, "IfElse", "16:2"), ss("b"), nil, ss("a", "b")) + wantDU(t, "Range 46:2", flowDU(t, "Range", "46:2"), ss("x"), nil, ss("xs")) + wantDU(t, "Switch 55:2", flowDU(t, "Switch", "55:2"), nil, nil, ss("a")) + wantDU(t, "Switch 56:2", flowDU(t, "Switch", "56:2"), nil, nil, nil) + wantDU(t, "GotoBack 112:2", flowDU(t, "GotoBack", "112:2"), ss("i"), nil, ss("i")) + wantDU(t, "Bump 150:2", flowDU(t, "Bump", "150:2"), nil, nil, ss("y")) + wantDU(t, "Labeled 97:5", flowDU(t, "Labeled", "97:5"), nil, nil, nil) + wantDU(t, "Bump @exit", flowDU(t, "Bump", dataflow.ExitKey), nil, nil, nil) + + fx := loadDU(t) + du, _ := fx.du(t, "Assign", "k++", 0) + wantDU(t, "k++", du, ss("k"), nil, ss("k")) + du, _ = fx.du(t, "Assign", "var z, w", 0) + wantDU(t, "var z, w", du, ss("w", "z"), nil, ss("k", "v")) + + // Mixed := : b is new, a is the parameter, re-assigned. + du, _ = fx.du(t, "Mixed", "b, a := 1, 2", 0) + wantDU(t, "b, a :=", du, ss("a", "b"), nil, nil) + param := fx.info.Defs[fx.decls["Mixed"].Type.Params.List[0].Names[0]] + for _, d := range du.Defs { + if d.Spell == "a" && d.Base != param { + t.Errorf("b, a := : def a Base = %v, want the parameter", d.Base) + } + } +} + +func TestDefUse_ShadowedBasesDiffer(t *testing.T) { + outer := flowDU(t, "Shadow", "155:2").Defs + inner := flowDU(t, "Shadow", "157:3").Defs + ret := flowDU(t, "Shadow", "160:2").Uses + if len(outer) != 1 || len(inner) != 1 || len(ret) != 1 { + t.Fatalf("Shadow: outer=%v inner=%v ret=%v", spells(outer), spells(inner), spells(ret)) + } + if outer[0].Spell != "x" || inner[0].Spell != "x" || ret[0].Spell != "x" { + t.Fatalf("Shadow: spellings %v %v %v", spells(outer), spells(inner), spells(ret)) + } + if outer[0].Base == inner[0].Base { + t.Errorf("Shadow: inner x has the outer x's base") + } + if ret[0].Base != outer[0].Base { + t.Errorf("Shadow: return x is not the outer x") + } + if inner[0].Covers(ret[0]) || ret[0].Covers(inner[0]) { + t.Errorf("Shadow: inner x interferes with the outer x") + } +} + +func TestDefUse_FieldDefIsPrefixDef(t *testing.T) { + fx := loadDU(t) + du, _ := fx.du(t, "Assign", "x.f = v", 0) + wantDU(t, "x.f = v", du, ss("x.f"), nil, ss("v")) + du, _ = fx.du(t, "Assign", "m[k] = v", 0) + wantDU(t, "m[k] = v", du, nil, ss("m[*]"), ss("k", "m", "v")) + du, _ = fx.du(t, "Assign", "*q = v", 0) + wantDU(t, "*q = v", du, nil, ss("q.*"), ss("q", "v")) + du, _ = fx.du(t, "Assign", "x.f += 1", 0) + wantDU(t, "x.f += 1", du, ss("x.f"), nil, ss("x.f")) + du, _ = fx.du(t, "Assign", "m[k]++", 0) + wantDU(t, "m[k]++", du, nil, ss("m[*]"), ss("k", "m", "m[*]")) + + // Fields: a write deeper than k is a weak def of the truncated path; the + // write goes through the pointer p (p.f is (*p).f), so it reads p. + wantDU(t, "Fields 176:2", flowDU(t, "Fields", "176:2"), nil, ss("p.f.g.*"), ss("p", "v")) + wantDU(t, "Fields 177:2", flowDU(t, "Fields", "177:2"), nil, nil, ss("p.f.g.x")) + // A statement whose LHS starts with a call defines no path; the call is + // its satellite, so the host carries the call's uses and weak defs. + wantDU(t, "CallLHS 211:2/2", flowDU(t, "CallLHS", "211:2/2"), nil, nil, ss("v")) +} + +func TestDefUse_CallWeakDefsRefArgs(t *testing.T) { + fx := loadDU(t) + du, _ := fx.du(t, "Calls", "mut(&req", 0) + wantDU(t, "mut(...)", du, nil, ss("m.*", "p.*", "req.*", "s.*"), ss("m", "p", "req", "s", "t")) + du, _ = fx.du(t, "Calls", "t.Ptr()", 0) + wantDU(t, "t.Ptr()", du, nil, ss("t.*"), ss("t")) + du, _ = fx.du(t, "Calls", "p.Ptr()", 0) + wantDU(t, "p.Ptr()", du, nil, ss("p.*"), ss("p")) + du, _ = fx.du(t, "Calls", "t.Val()", 0) + wantDU(t, "t.Val()", du, nil, nil, ss("t")) + du, _ = fx.du(t, "Calls", "n := len(s)", 0) + wantDU(t, "n := len(s)", du, ss("n"), nil, ss("s")) + du, _ = fx.du(t, "Calls", "s = append(s, n)", 0) + // append may write into s's spare capacity: s[*] weak, besides the s def. + wantDU(t, "s = append(s, n)", du, ss("s"), ss("s[*]"), ss("n", "s")) + du, _ = fx.du(t, "Calls", "copy(s, s)", 0) + wantDU(t, "copy(s, s)", du, nil, ss("s[*]"), ss("s")) + du, _ = fx.du(t, "Calls", "_ = T(t)", 0) + wantDU(t, "_ = T(t)", du, nil, nil, ss("t")) + du, _ = fx.du(t, "Calls", "r := pid(&req)", 0) + wantDU(t, "r := pid(&req)", du, ss("r"), ss("req.*"), ss("req")) + + // Satellites carry nothing; the host has the nested calls' flow. + for _, k := range []string{"196:7", "196:9"} { + wantDU(t, "Nested "+k, flowDU(t, "Nested", k), nil, nil, nil) + } + wantDU(t, "Nested 196:2", flowDU(t, "Nested", "196:2"), ss("x"), nil, ss("y")) + wantDU(t, "Chain 234:2/2", flowDU(t, "Chain", "234:2/2"), nil, nil, nil) + wantDU(t, "Chain 234:2", flowDU(t, "Chain", "234:2"), nil, ss("s.*"), ss("s")) + // A bare panic(i) is a call node of its own. + wantDU(t, "PanicLast 413:2", flowDU(t, "PanicLast", "413:2"), nil, nil, ss("i")) + wantDU(t, "PanicLast 412:2", flowDU(t, "PanicLast", "412:2"), nil, ss("xs[*]"), ss("i", "xs")) + // A closure expression uses what it captures, not its body's locals. + wantDU(t, "Closure 216:2", flowDU(t, "Closure", "216:2"), ss("h"), nil, ss("x")) + wantDU(t, "Closure 219:2", flowDU(t, "Closure", "219:2"), nil, nil, ss("h")) + + // go: the call's receiver, arguments and calls nested in them. + du, _ = fx.du(t, "Goes", "mut2(s, id(x))", 0) + wantDU(t, "go mut2(s, id(x))", du, nil, ss("s.*"), ss("s", "x")) + du, _ = fx.du(t, "Goes", "func() { sink(x) }()", 0) + wantDU(t, "go func(){sink(x)}()", du, nil, nil, ss("x")) + wantDU(t, "Spawn 202:5", flowDU(t, "Spawn", "202:5"), nil, nil, ss("x")) + // The calls in a go statement's arguments are satellites now (L1 records + // them): their uses stay on the go node, the satellites carry nothing. + wantDU(t, "Spawn 202:10", flowDU(t, "Spawn", "202:10"), nil, nil, nil) + wantDU(t, "GoArgs 534:5", flowDU(t, "GoArgs", "534:5"), nil, nil, ss("x")) + wantDU(t, "GoArgs 534:7", flowDU(t, "GoArgs", "534:7"), nil, nil, nil) +} + +func TestDefUse_ChanSendRecv(t *testing.T) { + wantDU(t, "Chan 182:2", flowDU(t, "Chan", "182:2"), nil, ss("ch[*]"), ss("ch", "v")) + wantDU(t, "Chan 183:2", flowDU(t, "Chan", "183:2"), ss("w"), nil, ss("ch", "ch[*]")) + wantDU(t, "Select2 82:2", flowDU(t, "Select2", "82:2"), nil, nil, nil) + wantDU(t, "Select2 83:2", flowDU(t, "Select2", "83:2"), ss("x"), nil, ss("a", "a[*]")) + wantDU(t, "Select2 85:2", flowDU(t, "Select2", "85:2"), nil, ss("b[*]"), ss("b")) + + fx := loadDU(t) + du, _ := fx.du(t, "Chans", "for x := range ch", 0) + wantDU(t, "range ch", du, ss("x"), nil, ss("ch", "ch[*]")) + du, _ = fx.du(t, "Chans", "out <- x", 0) + wantDU(t, "out <- x", du, nil, ss("out[*]"), ss("out", "x")) + du, _ = fx.du(t, "Chans", "case y, ok", 0) + wantDU(t, "case y, ok := <-ch", du, ss("ok", "y"), nil, ss("ch", "ch[*]")) + du, _ = fx.du(t, "Chans", "case <-ch", 0) + wantDU(t, "case <-ch", du, nil, nil, ss("ch", "ch[*]")) + du, _ = fx.du(t, "Chans", "sink(<-ch)", 0) + wantDU(t, "sink(<-ch)", du, nil, nil, ss("ch", "ch[*]")) +} + +func TestDefUse_TypeSwitchGuardDefinesClauseVars(t *testing.T) { + hdr := flowDU(t, "TypeSwitch", "69:2") + wantSet(t, "TypeSwitch 69:2 Uses", hdr.Uses, "v") + use := flowDU(t, "TypeSwitch", "71:3").Uses + if len(use) != 1 || use[0].Spell != "t" { + t.Fatalf("TypeSwitch 71:3 Uses = %v, want [t]", spells(use)) + } + found := false + for _, d := range hdr.Defs { + if d.Spell != "t" { + t.Errorf("TypeSwitch header def %q, want only t", d.Spell) + } + found = found || d.Base == use[0].Base + } + if !found { + t.Errorf("TypeSwitch: the case-int clause's t is not defined by the guard; defs %v", spells(hdr.Defs)) + } +} + +func TestDefUse_DeferRegistrationCarriesArgs(t *testing.T) { + // EarlyReturnDefer: defer work(g(a)). + wantDU(t, "EarlyReturnDefer 127:2", flowDU(t, "EarlyReturnDefer", "127:2"), nil, ss(":127:8"), ss("a")) + wantDU(t, "EarlyReturnDefer 127:8", flowDU(t, "EarlyReturnDefer", "127:8"), nil, nil, ss(":127:8")) + wantDU(t, "EarlyReturnDefer 127:13", flowDU(t, "EarlyReturnDefer", "127:13"), nil, nil, nil) + wantDU(t, "EarlyReturnDefer 126:2", flowDU(t, "EarlyReturnDefer", "126:2"), nil, ss(":126:8"), nil) + wantDU(t, "EarlyReturnDefer 126:8", flowDU(t, "EarlyReturnDefer", "126:8"), nil, nil, ss(":126:8")) + + fx := loadDU(t) + // defer mu.Unlock(): the receiver is evaluated at registration; the + // deferred call writes through it, reads the named results and (I7) + // reads the state behind the receiver at exit (mu.*). + reg, _ := fx.du(t, "Defers", "defer mu.Unlock()", 0) + call, ck := fx.du(t, "Defers", "mu.Unlock()", 0) + d := ":" + ck + wantDU(t, "defer mu.Unlock()", reg, nil, ss(d), ss("mu")) + wantDU(t, "mu.Unlock() (deferred)", call, nil, ss("mu.*"), ss(d, "err", "mu.*", "n")) + + // defer func(){…}(): nothing is evaluated at registration but the func + // value; the deferred call uses what the closure captures. + reg, _ = fx.du(t, "Defers", "defer func() {", 0) + call, ck = fx.du(t, "Defers", "defer func() {", len("defer ")) + d = ":" + ck + wantDU(t, "defer func(){…}", reg, nil, ss(d), nil) + wantDU(t, "func(){…}() (deferred)", call, nil, nil, ss(d, "err", "n", "x")) + + reg, _ = fx.du(t, "Defers", "defer mut2(s, x)", 0) + call, ck = fx.du(t, "Defers", "defer mut2(s, x)", len("defer ")) + d = ":" + ck + wantDU(t, "defer mut2(s, x)", reg, nil, ss(d), ss("s", "x")) + wantDU(t, "mut2(s, x) (deferred)", call, nil, ss("s.*"), ss(d, "err", "n", "s.*")) + + // A bare return reads the named results; a return with operands assigns + // them (the deferred calls run after that assignment). + ret, _ := fx.du(t, "Defers", "return\n", 0) + wantDU(t, "return", ret, nil, nil, ss("err", "n")) + ret, _ = fx.du(t, "NamedRet", "\treturn x\n", 1) + wantDU(t, "return x", ret, ss("n"), nil, ss("x")) +} + +func TestDefUse_EntryDefinesParamsReceiverNamedResults(t *testing.T) { + fx := loadDU(t) + entry := fx.decls["Entry"] + bases := dataflow.EntryBases(entry, fx.info) + var names []string + for _, b := range bases { + names = append(names, b.Name()) + } + // The signature by position (one file), then package-level vars by + // (pkgpath, name) — FileSet bases of other files are load-order dependent. + if want := ss("r", "a", "b", "n", "err", "V", "W", "G"); !reflect.DeepEqual(names, want) { + t.Errorf("EntryBases(Entry) = %v, want %v", names, want) + } + byKey, keyOf, _ := fx.nodes(t, "Entry") + du := dataflow.DefUseOf(byKey[dataflow.EntryKey], fx.info, kTest, bases, entry, keyOf) + wantDU(t, "Entry @entry", du, + ss("example.com/lib::V", "example.com/lib::W", "example.com/p::G", "r", "a", "b", "n", "err"), nil, nil) + + // G = a defines the package-level path. + g, _ := fx.du(t, "Entry", "G = a", 0) + wantDU(t, "G = a", g, ss("example.com/p::G"), nil, ss("a")) + l, _ := fx.du(t, "Entry", "_ = lib.V.F + lib.W", 0) + wantDU(t, "_ = lib.V.F + lib.W", l, nil, nil, ss("example.com/lib::V.F", "example.com/lib::W")) + + // The closure's captures are its free variables: a and r, not its own k + // (which the inner closure captures). + var lit *ast.FuncLit + ast.Inspect(entry.Body, func(n ast.Node) bool { + if f, ok := n.(*ast.FuncLit); ok && lit == nil { + lit = f + } + return lit == nil + }) + names = nil + for _, b := range dataflow.EntryBases(lit, fx.info) { + names = append(names, b.Name()) + } + if want := ss("r", "a"); !reflect.DeepEqual(names, want) { + t.Errorf("EntryBases(closure) = %v, want %v", names, want) + } + h, _ := fx.du(t, "Entry", "h := func", 0) + wantDU(t, "h := func…", h, ss("h"), nil, ss("a", "r")) + + // A flow-fixture function with no package vars: params only. + wantDU(t, "Bump @entry", flowDU(t, "Bump", dataflow.EntryKey), ss("n"), nil, nil) +} + +// TestDefUse_Invariants runs DefUseOf over every node of every flow-fixture +// function: no panic, satellites and @exit are empty, strong defs carry no +// wildcard, and @entry has only defs. +func TestDefUse_Invariants(t *testing.T) { + fx := loadFlow(t) + for fn, d := range fx.decls { + nodes, keyOf := dataflow.BuildNodes(fx.fset, d.Body, fx.callKeys(t, fn)) + bases := dataflow.EntryBases(d, fx.info) + for _, n := range nodes { + du := dataflow.DefUseOf(n, fx.info, kTest, bases, d, keyOf) + empty := len(du.Defs)+len(du.WeakDefs)+len(du.Uses) == 0 + if (n.Parent != "" || n.Kind == dataflow.KindExit) && !empty { + t.Errorf("%s %s: satellite / @exit has def/use %+v", fn, n.Key, du) + } + if n.Kind == dataflow.KindEntry && len(du.WeakDefs)+len(du.Uses) != 0 { + t.Errorf("%s @entry: weak defs / uses %+v", fn, du) + } + if n.Deferred && len(du.Defs) != 0 { + t.Errorf("%s %s: deferred call strong-defines %v (R11)", fn, n.Key, spells(du.Defs)) + } + for what, ps := range map[string][]dataflow.Path{"Defs": du.Defs, "WeakDefs": du.WeakDefs, "Uses": du.Uses} { + if !sort.SliceIsSorted(ps, func(i, j int) bool { return ps[i].Spell < ps[j].Spell }) { + t.Errorf("%s %s: %s not sorted by spelling: %v", fn, n.Key, what, ps) + } + } + for _, p := range du.Defs { + if strings.Contains(p.Spell, "*") { + t.Errorf("%s %s: strong def %q has a wildcard", fn, n.Key, p.Spell) + } + } + } + } +} + +// TestDefUse_FoldedLoopDefsAreWeak locks the for/range header rule: the +// header node runs on loop entry and on every back edge, so only a variable +// defined by both Init and Post is a strong def there; a range with `=` may +// run zero times, so its Key/Value are weak. +func TestDefUse_FoldedLoopDefsAreWeak(t *testing.T) { + fx := loadDU(t) + du, _ := fx.du(t, "ForPost", "for i2 := 0", 0) + wantDU(t, "ForPost header", du, ss("i2"), ss("x"), ss("i2")) + du, _ = fx.du(t, "ForInit", "for i, v := 0, 0", 0) + wantDU(t, "ForInit header", du, ss("i"), ss("v"), ss("i", "n")) + du, _ = fx.du(t, "RangeAssign", "for _, last = range a", 0) + wantDU(t, "RangeAssign header", du, nil, ss("last"), ss("a")) + // Both Init and Post define i: strong (flow fixture). + wantDU(t, "Bump 147:2", flowDU(t, "Bump", "147:2"), ss("i"), nil, ss("i", "n")) +} + +func TestDefUse_ReferenceTypedReceiverIsWeakDefined(t *testing.T) { + fx := loadDU(t) + du, _ := fx.du(t, "Recv", "w.Write(data)", 0) + wantDU(t, "w.Write(data)", du, nil, ss("data.*", "w.*"), ss("data", "w")) + du, _ = fx.du(t, "Recv", "h.Set(", 0) + wantDU(t, "h.Set(...)", du, nil, ss("h.*"), ss("h")) + du, _ = fx.du(t, "Recv", "\tt.Val()\n}\n\nfunc IfShadow", 1) + wantDU(t, "t.Val() value struct", du, nil, nil, ss("t")) +} + +func TestDefUse_IfInitShadows(t *testing.T) { + fx := loadDU(t) + du, _ := fx.du(t, "IfShadow", "if x := id(x)", 0) + wantDU(t, "IfShadow header", du, ss("x"), nil, ss("x", "x")) + param := fx.info.Defs[fx.decls["IfShadow"].Type.Params.List[0].Names[0]] + if du.Defs[0].Base == param { + t.Errorf("if-init x has the parameter's base") + } + // Uses are sorted by (spelling, base position): the parameter first. + if du.Uses[0].Base != param || du.Uses[1].Base != du.Defs[0].Base { + t.Errorf("IfShadow header uses: want [param x, init x], got bases %v, %v", du.Uses[0].Base, du.Uses[1].Base) + } +} + +// TestDefUse_SubsliceArgs (C1): a slice expression passed to a callee (or as +// copy's/clear's destination) weak-defines its base's elements, base[*]; the +// call still reads the base and the slice bounds. +func TestDefUse_SubsliceArgs(t *testing.T) { + wantDU(t, "r.Read(buf[:n])", flowDU(t, "SubsliceRead", "487:2"), nil, ss("buf[*]", "r.*"), ss("buf", "n", "r")) + wantDU(t, "readFull(r, arr[:])", flowDU(t, "ArraySliceRead", "494:2"), nil, ss("arr[*]", "r.*"), ss("arr", "r")) + wantDU(t, "copy(dst[i:], src)", flowDU(t, "CopyInto", "501:2"), nil, ss("dst[*]"), ss("dst", "i", "src")) + wantDU(t, "clear(m)", flowDU(t, "ClearAll", "507:2"), nil, ss("m[*]"), ss("m")) + wantDU(t, "clear(s[:n])", flowDU(t, "ClearAll", "508:2"), nil, ss("s[*]"), ss("n", "s")) +} diff --git a/internal/dataflow/export_test.go b/internal/dataflow/export_test.go new file mode 100644 index 0000000..0b9aab0 --- /dev/null +++ b/internal/dataflow/export_test.go @@ -0,0 +1,8 @@ +package dataflow + +// CanPanic exposes canPanic to the external test package. +var CanPanic = canPanic + +// PostDominators exposes the immediate post-dominator map to the external +// test package. +var PostDominators = postDominators diff --git a/internal/dataflow/ir.go b/internal/dataflow/ir.go new file mode 100644 index 0000000..41b2fa2 --- /dev/null +++ b/internal/dataflow/ir.go @@ -0,0 +1,83 @@ +// Package dataflow builds the L3 intraprocedural graphs (CFG, CDG, DDG) of one +// callable body from go/ast. Its IR is keyed on the same local body keys the L1 +// emitter writes ("l:c", "l:c/2", "@entry", "@exit"), so the L3 overlay joins +// the L1 call nodes on one string and never recomposes ids. +// +// The package must not import the v1 schema (internal/schema) or the v2 emitter +// (internal/syntactic_analysis/v2emit): the emitter imports this package. +package dataflow + +import ( + "go/ast" + "go/token" +) + +// NodeKind is a body node's kind. Shared vocabulary only — Go adds no kinds +// (spec §4.2): `go` and `defer` are carried by the call node's flags. +type NodeKind string + +const ( + KindCall NodeKind = "call" + KindStatement NodeKind = "statement" + KindBranch NodeKind = "branch" + KindLoop NodeKind = "loop" + KindSwitch NodeKind = "switch" + KindReturn NodeKind = "return" + KindEntry NodeKind = "entry" + KindExit NodeKind = "exit" +) + +// Synthetic node keys. They start with "@", so v2.GlobalOrdinal concatenates +// them to the callable id as-is. +const ( + EntryKey = "@entry" + ExitKey = "@exit" +) + +// Node is one body node of a callable. +// +// - Key is the local key: "l:c" (v2.LocalID of the node's anchor position), +// "l:c/2", … for co-positioned nodes, or EntryKey / ExitKey. +// - Pos/End is the span of Stmt; NoPos for synthetic nodes. StartLC/EndLC +// are their [line, col] (fset.Position, so //line directives apply, as +// they do to the L1 call-site positions); zero for synthetic nodes. +// - Stmt is the AST node the body node stands for: +// a statement (*ast.ExprStmt for a bare call, *ast.GoStmt for a `go` call), +// a clause head (*ast.CaseClause / *ast.CommClause), +// or an *ast.CallExpr for a satellite call node or the deferred call node. +// nil for synthetic nodes. +// - Parent is the innermost enclosing statement's key for a satellite call +// node (a call nested in a statement, header or clause head; for a call in +// a `defer`'s arguments, the defer statement). "" otherwise. +// - Deferred marks the deferred call node of a `defer` statement (spec +// §4.3.1): it runs only in the exit chain; its registration is the +// separate `statement` node keyed at the `defer` keyword. +type Node struct { + Key string + Kind NodeKind + Pos, End token.Pos + StartLC [2]int + EndLC [2]int + Stmt ast.Node + Parent string + Deferred bool +} + +// CfgEdge is a control-flow edge between two local keys. Kind is a shared CFG +// edge kind ("fallthrough", "true", "false", …) or the Go add "defer_resume". +type CfgEdge struct{ Src, Dst, Kind string } + +// CdgEdge is a control-dependence edge between two local keys. +type CdgEdge struct{ Src, Dst string } + +// DdgEdge is a data-dependence edge on access path Var. Its prov is always +// ["ssa"] at L3, so it is not stored. +type DdgEdge struct{ Src, Dst, Var string } + +// Result is the L3 overlay of one callable body. +type Result struct { + Nodes []Node + Cfg []CfgEdge + Cdg []CdgEdge + Ddg []DdgEdge +} diff --git a/internal/dataflow/postdom.go b/internal/dataflow/postdom.go new file mode 100644 index 0000000..2c27453 --- /dev/null +++ b/internal/dataflow/postdom.go @@ -0,0 +1,226 @@ +package dataflow + +import "sort" + +// BuildCDG computes the control-dependence graph of one callable body (spec +// §4.4) from BuildCFG's output: the kept nodes and the final CFG. +// +// Post-dominators come from the iterative Cooper-Harvey-Kennedy algorithm on +// the reverse CFG rooted at @exit (postDominators). Control dependence +// follows Ferrante-Ottenstein-Warren: for each CFG edge a→b with b ≠ +// ipdom(a), every node x on the post-dominator-tree path from b up to +// ipdom(a) (excluded) gets a→x. When b post-dominates a the walk is empty. +// +// The computation is purely structural (R10): edge kinds are ignored, and +// parallel edges between one pair (defer_resume and its exception twin, …) +// are one edge. Loops are not identified by loop_back; a back edge is just an +// edge. Self-loops are skipped: x→x is never emitted. +// +// Anchoring: every non-synthetic CFG node that has no controller gets +// @entry→x. That includes the deferred call nodes of the exit chain, which +// post-dominate every statement and so are never controlled. @entry and +// @exit are never anchored: no @entry→@entry, no @entry→@exit. This matches +// Python (codeanalyzer-python dataflow/dominance.py control_dependence, +// which skips entry and exit when anchoring). When nodes holds no @entry, +// nothing is anchored. +// +// The CFG nodes are the nodes with no Parent; satellite call nodes are not +// CFG steps and get no CDG edge. Edges with an endpoint outside that set are +// ignored. +// +// Precondition (guaranteed by BuildCFG, asserted by the tests, not checked +// here): every node reaches @exit. Outside it, BuildCDG never panics: a node +// that cannot reach @exit has no post-dominator, controls nothing and is +// controlled by nothing, so its only CDG edge is the @entry anchor. +// +// The output is sorted by (Src, Dst), deduplicated and non-nil. +func BuildCDG(nodes []Node, cfg []CfgEdge) []CdgEdge { + g := newPdomGraph(nodes, cfg) + ipdom := g.ipdoms() + + deps := map[[2]int]bool{} + for a, succs := range g.succ { + if ipdom[a] < 0 { + continue + } + stop := ipdom[a] + for _, b := range succs { + if b == a || ipdom[b] < 0 { + continue + } + // ipdom(a) post-dominates b, so the walk reaches stop. The root + // guard only protects against a malformed tree. + for x := b; x != stop; x = ipdom[x] { + if x != a { + deps[[2]int{a, x}] = true + } + if x == g.exit { + break + } + } + } + } + + if entry, ok := g.index[EntryKey]; ok { + controlled := make([]bool, len(g.keys)) + for d := range deps { + controlled[d[1]] = true + } + for x := range g.keys { + if x != entry && x != g.exit && !controlled[x] { + deps[[2]int{entry, x}] = true + } + } + } + + out := make([]CdgEdge, 0, len(deps)) + for d := range deps { + out = append(out, CdgEdge{Src: g.keys[d[0]], Dst: g.keys[d[1]]}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Src != out[j].Src { + return out[i].Src < out[j].Src + } + return out[i].Dst < out[j].Dst + }) + return out +} + +// postDominators returns the immediate post-dominator of every CFG node that +// reaches @exit, keyed by local key. @exit maps to itself (the tree root). +// A node that cannot reach @exit is absent. +func postDominators(nodes []Node, cfg []CfgEdge) map[string]string { + g := newPdomGraph(nodes, cfg) + out := map[string]string{} + for x, d := range g.ipdoms() { + if d >= 0 { + out[g.keys[x]] = g.keys[d] + } + } + return out +} + +// pdomGraph is the CFG over integer vertices: keys sorted, edges deduplicated +// and kind-blind, adjacency sorted. Every iteration order derives from it, +// never from map order. +type pdomGraph struct { + keys []string + index map[string]int + succ, pred [][]int + exit int // -1 when there is no @exit node +} + +func newPdomGraph(nodes []Node, cfg []CfgEdge) *pdomGraph { + index := map[string]int{} + var keys []string + for _, n := range nodes { + if n.Parent != "" { + continue + } + if _, dup := index[n.Key]; !dup { + index[n.Key] = -1 + keys = append(keys, n.Key) + } + } + sort.Strings(keys) + for i, k := range keys { + index[k] = i + } + g := &pdomGraph{keys: keys, index: index, succ: make([][]int, len(keys)), pred: make([][]int, len(keys)), exit: -1} + if x, ok := index[ExitKey]; ok { + g.exit = x + } + seen := map[[2]int]bool{} + for _, e := range cfg { + a, okA := index[e.Src] + b, okB := index[e.Dst] + if !okA || !okB || seen[[2]int{a, b}] { + continue + } + seen[[2]int{a, b}] = true + g.succ[a] = append(g.succ[a], b) + g.pred[b] = append(g.pred[b], a) + } + for i := range keys { + sort.Ints(g.succ[i]) + sort.Ints(g.pred[i]) + } + return g +} + +// ipdoms runs Cooper-Harvey-Kennedy ("A Simple, Fast Dominance Algorithm") on +// the reverse CFG rooted at @exit: a node's reverse-graph predecessors are its +// CFG successors. It returns ipdom per vertex, -1 for a vertex that cannot +// reach @exit (or for all of them when there is no @exit). +func (g *pdomGraph) ipdoms() []int { + ipdom := make([]int, len(g.keys)) + for i := range ipdom { + ipdom[i] = -1 + } + if g.exit < 0 { + return ipdom + } + + // Postorder of the reverse CFG from @exit (iterative DFS over sorted + // CFG-predecessor lists). + po := make([]int, len(g.keys)) // postorder number; valid when ipdom set + visited := make([]bool, len(g.keys)) + var order []int + type frame struct{ v, i int } + stack := []frame{{g.exit, 0}} + visited[g.exit] = true + for len(stack) > 0 { + top := &stack[len(stack)-1] + if top.i < len(g.pred[top.v]) { + w := g.pred[top.v][top.i] + top.i++ + if !visited[w] { + visited[w] = true + stack = append(stack, frame{w, 0}) + } + continue + } + po[top.v] = len(order) + order = append(order, top.v) + stack = stack[:len(stack)-1] + } + + intersect := func(a, b int) int { + for a != b { + for po[a] < po[b] { + a = ipdom[a] + } + for po[b] < po[a] { + b = ipdom[b] + } + } + return a + } + + ipdom[g.exit] = g.exit + for changed := true; changed; { + changed = false + for i := len(order) - 1; i >= 0; i-- { // reverse postorder + v := order[i] + if v == g.exit { + continue + } + nd := -1 + for _, s := range g.succ[v] { + if s == v || ipdom[s] < 0 { + continue + } + if nd < 0 { + nd = s + } else { + nd = intersect(nd, s) + } + } + if nd >= 0 && ipdom[v] != nd { + ipdom[v] = nd + changed = true + } + } + } + return ipdom +} diff --git a/internal/dataflow/postdom_ref_test.go b/internal/dataflow/postdom_ref_test.go new file mode 100644 index 0000000..e889225 --- /dev/null +++ b/internal/dataflow/postdom_ref_test.go @@ -0,0 +1,142 @@ +package dataflow_test + +import ( + "path/filepath" + "reflect" + "runtime" + "sort" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" +) + +// refCDG is control dependence computed by definition, independent of +// BuildCDG's Cooper-Harvey-Kennedy / Ferrante-Ottenstein-Warren code: +// +// - x post-dominates y iff every CFG path from y to @exit goes through x, +// i.e. x == y, or y cannot reach @exit once x is removed; +// - x is control-dependent on a iff x post-dominates some CFG successor of +// a and x does not strictly post-dominate a; +// - x→x is never emitted; every non-synthetic node with no controller gets +// @entry→x; @entry→@exit is never emitted. +// +// The CFG nodes are the non-satellite nodes; edge kinds and parallel edges +// are ignored. ok is false when a node cannot reach @exit (BuildCDG's +// precondition), where "post-dominates" would hold vacuously. +func refCDG(nodes []dataflow.Node, cfg []dataflow.CfgEdge) (out []dataflow.CdgEdge, ok bool) { + var keys []string + in := map[string]bool{} + for _, n := range nodes { + if n.Parent == "" && !in[n.Key] { + in[n.Key] = true + keys = append(keys, n.Key) + } + } + sort.Strings(keys) + succ := map[string][]string{} + pred := map[string][]string{} + for _, e := range cfg { + if in[e.Src] && in[e.Dst] { + succ[e.Src] = append(succ[e.Src], e.Dst) + pred[e.Dst] = append(pred[e.Dst], e.Src) + } + } + + // reachesExit[x] is the set of nodes that reach @exit with x removed + // ("" = nothing removed), by backward search from @exit. + reachesExit := func(removed string) map[string]bool { + seen := map[string]bool{} + if removed == exit || !in[exit] { + return seen + } + seen[exit] = true + work := []string{exit} + for len(work) > 0 { + n := work[len(work)-1] + work = work[:len(work)-1] + for _, p := range pred[n] { + if p != removed && !seen[p] { + seen[p] = true + work = append(work, p) + } + } + } + return seen + } + all := reachesExit("") + for _, k := range keys { + if !all[k] { + return nil, false + } + } + without := map[string]map[string]bool{} + for _, x := range keys { + without[x] = reachesExit(x) + } + pdom := func(x, y string) bool { return x == y || !without[x][y] } + + deps := map[dataflow.CdgEdge]bool{} + controlled := map[string]bool{} + for _, a := range keys { + for _, x := range keys { + if x == a || pdom(x, a) { // x == a is a self-loop; pdom(x, a) with x ≠ a is strict + continue + } + for _, b := range succ[a] { + if pdom(x, b) { + deps[dataflow.CdgEdge{Src: a, Dst: x}] = true + controlled[x] = true + break + } + } + } + } + if in[entry] { + for _, x := range keys { + if x != entry && x != exit && !controlled[x] { + deps[dataflow.CdgEdge{Src: entry, Dst: x}] = true + } + } + } + out = []dataflow.CdgEdge{} + for d := range deps { + out = append(out, d) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Src != out[j].Src { + return out[i].Src < out[j].Src + } + return out[i].Dst < out[j].Dst + }) + return out, true +} + +// TestCDG_BruteForceReference (R12) asserts BuildCDG equals refCDG on every +// function and function literal body in testdata/dataflow and testdata/chi. +func TestCDG_BruteForceReference(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata") + total, edges := 0, 0 + for _, dir := range []string{"dataflow", "chi"} { + fset, cases := loadDiffCases(t, filepath.Join(root, dir)) + for _, c := range cases { + total++ + nodes, keyOf := dataflow.BuildNodes(fset, c.body, gateCallKeys(fset, c.body)) + kept, es := dataflow.BuildCFG(nodes, keyOf, c.body, c.info) + want, ok := refCDG(kept, es) + if !ok { + t.Errorf("%s: a CFG node cannot reach @exit (BuildCDG precondition)", c.name) + continue + } + got := dataflow.BuildCDG(kept, es) + edges += len(got) + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: BuildCDG differs from the by-definition reference\n got %v\nwant %v", c.name, got, want) + } + } + } + if total < 100 { + t.Errorf("reference covered only %d bodies", total) + } + t.Logf("cdg reference: %d bodies, %d edges equal", total, edges) +} diff --git a/internal/dataflow/postdom_test.go b/internal/dataflow/postdom_test.go new file mode 100644 index 0000000..6223c75 --- /dev/null +++ b/internal/dataflow/postdom_test.go @@ -0,0 +1,326 @@ +package dataflow_test + +import ( + "path/filepath" + "reflect" + "runtime" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" +) + +func c(src, dst string) dataflow.CdgEdge { return dataflow.CdgEdge{Src: src, Dst: dst} } + +// cdgOf builds fn's CFG (Tasks 5–7), asserts BuildCDG's input precondition — +// every CFG node reaches @exit — and returns the CDG with the CFG. +func cdgOf(t *testing.T, fn string) ([]dataflow.Node, []dataflow.CfgEdge, []dataflow.CdgEdge) { + t.Helper() + nodes, es := cfgOf(t, fn) + requireReachesExit(t, fn, nodes, es) + return nodes, es, dataflow.BuildCDG(nodes, es) +} + +// requireReachesExit is BuildCDG's precondition, asserted by the tests (not +// by production code): every CFG node (non-satellite) reaches @exit. +func requireReachesExit(t *testing.T, fn string, nodes []dataflow.Node, es []dataflow.CfgEdge) { + t.Helper() + rev := make([]dataflow.CfgEdge, len(es)) + for i, g := range es { + rev[i] = dataflow.CfgEdge{Src: g.Dst, Dst: g.Src, Kind: g.Kind} + } + bwd := closure(rev)[exit] + for _, n := range nodes { + if n.Parent == "" && n.Key != exit && !bwd[n.Key] { + t.Fatalf("%s: precondition: node %s cannot reach @exit", fn, n.Key) + } + } +} + +// wantCDG asserts fn's CDG is exactly want, in (src, dst) order. +func wantCDG(t *testing.T, fn string, want ...dataflow.CdgEdge) { + t.Helper() + _, _, got := cdgOf(t, fn) + have := map[dataflow.CdgEdge]bool{} + for _, g := range got { + have[g] = true + } + exp := map[dataflow.CdgEdge]bool{} + for _, w := range want { + exp[w] = true + if !have[w] { + t.Errorf("%s: missing cdg %s→%s", fn, w.Src, w.Dst) + } + } + for _, g := range got { + if !exp[g] { + t.Errorf("%s: unexpected cdg %s→%s", fn, g.Src, g.Dst) + } + } + checkCDGOrder(t, fn, got) +} + +func checkCDGOrder(t *testing.T, fn string, got []dataflow.CdgEdge) { + t.Helper() + for i := 1; i < len(got); i++ { + a, b := got[i-1], got[i] + if a.Src > b.Src || a.Src == b.Src && a.Dst >= b.Dst { + t.Errorf("%s: cdg out of order or duplicated: %v before %v", fn, a, b) + } + } +} + +// Derivations below use the CFGs pinned in cfg_test.go. ipdom(n) is n's +// immediate post-dominator; FOW adds a→x for each CFG edge a→b with b ≠ +// ipdom(a), for x on the post-dominator-tree path b … ipdom(a) (excluded). +// Anchoring: every non-synthetic CFG node with no controller gets @entry→x; +// @entry and @exit are never anchored (Python parity). + +func TestCDG_IfElse(t *testing.T) { + // ipdom: 17:3, 21:2 → @exit; 19:3 → 21:2; 16:2 → @exit; @entry → 16:2. + // 16:2→17:3: walk 17:3 (stop @exit). 16:2→19:3: walk 19:3, 21:2. + // The then-arm returns, so the statement after the if (21:2) runs only + // on the false path: it depends on the if, not on @entry. + // Anchored: 16:2. + wantCDG(t, "IfElse", + c(ieIf, ieThenRet), + c(ieIf, ieElseAsg), + c(ieIf, ieRet), + c(entry, ieIf), + ) +} + +func TestCDG_Loop(t *testing.T) { + // ForLoop. ipdom: 30:2 → @exit; 27:2 → 30:2; 28:3 → 27:2; 26:2 → 27:2. + // 27:2→28:3: walk 28:3, 27:2 (stop 30:2); 27:2→27:2 is a self-loop, + // skipped. 28:3→27:2 is ipdom(28:3): nothing. The header has no other + // controller, so it is anchored with 26:2 and 30:2. + wantCDG(t, "ForLoop", + c(flFor, flBody), + c(entry, flInit), + c(entry, flFor), + c(entry, flRet), + ) + // Spin: two empty-body loops, i.e. CFG self-loops 294:2→294:2 and + // 296:2→296:2. ipdom: 294:2 → 296:2 → @exit. The only non-ipdom edges + // are the self-loops, whose walk visits only the loop itself (skipped): + // everything is anchored. + wantCDG(t, "Spin", + c(entry, "294:2"), + c(entry, "296:2"), + ) +} + +func TestCDG_Infinite(t *testing.T) { + // for {} with the synthetic 35:2→@exit exception edge. ipdom: 35:2 → + // @exit (its successors 36:3 and @exit meet at @exit); 36:3 → 37:3; + // 37:3 → @exit (35:2 and 38:4 meet at @exit); 38:4 → @exit. + // 35:2→36:3: walk 36:3, 37:3. 37:3→35:2: walk 35:2. 37:3→38:4: walk + // 38:4. The header is controlled by the if (the only way back into it), + // so it is not anchored: nothing is. + wantCDG(t, "Infinite", + c(infFor, infRecv), + c(infFor, infIf), + c(infIf, infFor), + c(infIf, infRet), + ) +} + +func TestCDG_DeferChain(t *testing.T) { + // EarlyReturnDefer. Every exit (129:3, 132:2, the exception edges of + // 127:2 and 131:2) enters D1 = 127:8, then D2 = 126:8, then @exit, so + // the chain post-dominates every statement. ipdom: 126:8 → @exit; + // 127:8 → 126:8; 127:2, 128:2, 129:3, 131:2, 132:2 → 127:8; + // 126:2 → 127:2. Parallel defer_resume/exception edges are one edge. + // 127:2→128:2 (exception twin to 127:8): walk 128:2. 128:2→129:3, + // 128:2→131:2, 131:2→132:2: one node each. Anchored: 126:2, 127:2, both + // deferred calls. The satellite g(a) at 127:13 is not a CFG + // node and gets no CDG edge. + wantCDG(t, "EarlyReturnDefer", + c(erdReg2, erdIf), + c(erdIf, erdRetNeg), + c(erdIf, erdCall), + c(erdCall, erdRet), + c(entry, erdReg1), + c(entry, erdD2), + c(entry, erdReg2), + c(entry, erdD1), + ) +} + +func TestCDG_Switch(t *testing.T) { + // Switch with fallthrough. ipdom: 64:2 → @exit; 60:3, 62:3 → 64:2; + // 58:3, 59:2 → 60:3; 57:3 → 58:3; 56:2 → 57:3; 61:2 → 62:3; 55:2 → 64:2. + // 55:2→56:2: walk 56:2, 57:3, 58:3, 60:3. 55:2→59:2: walk 59:2, 60:3. + // 55:2→61:2: walk 61:2, 62:3. 60:3 (reached by case 1's fallthrough and + // case 2, not default) depends on the header. Anchored: 54:2, 55:2, + // 64:2. + wantCDG(t, "Switch", + c(swHdr, swCase1), + c(swHdr, swAsg1), + c(swHdr, swFall), + c(swHdr, swCase2), + c(swHdr, swAsg2), + c(swHdr, swDefault), + c(swHdr, swAsgD), + c(entry, swInit), + c(entry, swHdr), + c(entry, swRet), + ) +} + +func TestCDG_Select(t *testing.T) { + // Select2. ipdom: 83:2 → 84:3; 82:2, 84:3, 85:2, 86:3 → @exit. + // 82:2→83:2: walk 83:2, 84:3. 82:2→85:2: walk 85:2. 85:2→86:3 (the + // send's exception edge goes to @exit, ipdom(85:2)): walk 86:3. + // Anchored: 82:2. + wantCDG(t, "Select2", + c(selHdr, selRecv), + c(selHdr, selRetX), + c(selHdr, selSend), + c(selSend, selRet0), + c(entry, selHdr), + ) +} + +func TestPostDominators_Infinite(t *testing.T) { + nodes, es := cfgOf(t, "Infinite") + got := dataflow.PostDominators(nodes, es) + want := map[string]string{ + entry: infFor, + infFor: exit, + infRecv: infIf, + infIf: exit, + infRet: exit, + exit: exit, // the tree root maps to itself + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Infinite ipdom = %v, want %v", got, want) + } +} + +// TestCDG_NoExitPath: outside the precondition, a node that cannot reach +// @exit has no post-dominator and no controller but @entry; BuildCDG never +// panics. Edges with an endpoint that is not a node are ignored. +func TestCDG_NoExitPath(t *testing.T) { + n := func(k string) dataflow.Node { return dataflow.Node{Key: k} } + nodes := []dataflow.Node{n(entry), n("a"), n("b"), n("c"), n(exit)} + es := []dataflow.CfgEdge{ + e(entry, "a", "fallthrough"), + e("a", "b", "true"), // b, c cycle forever + e("a", exit, "false"), + e("b", "c", "fallthrough"), + e("c", "b", "loop_back"), + e("c", "ghost", "fallthrough"), + } + ipdom := dataflow.PostDominators(nodes, es) + if want := map[string]string{entry: "a", "a": exit, exit: exit}; !reflect.DeepEqual(ipdom, want) { + t.Errorf("ipdom = %v, want %v", ipdom, want) + } + got := dataflow.BuildCDG(nodes, es) + want := []dataflow.CdgEdge{c(entry, "a"), c(entry, "b"), c(entry, "c")} + if !reflect.DeepEqual(got, want) { + t.Errorf("cdg = %v, want %v", got, want) + } + + // No @exit at all: nothing has a post-dominator; everything is anchored. + got = dataflow.BuildCDG([]dataflow.Node{n(entry), n("a")}, []dataflow.CfgEdge{e(entry, "a", "fallthrough")}) + if want := []dataflow.CdgEdge{c(entry, "a")}; !reflect.DeepEqual(got, want) { + t.Errorf("no-exit cdg = %v, want %v", got, want) + } +} + +func TestCDG_EmptyBody(t *testing.T) { + if got := dataflow.BuildCDG(nil, nil); got == nil || len(got) != 0 { + t.Errorf("BuildCDG(nil, nil) = %#v, want empty non-nil", got) + } + // Empty body: @entry → @exit only. Neither synthetic node is anchored, + // so the CDG is empty: no @entry→@exit, no @entry→@entry. + got := dataflow.BuildCDG( + []dataflow.Node{{Key: entry, Kind: dataflow.KindEntry}, {Key: exit, Kind: dataflow.KindExit}}, + []dataflow.CfgEdge{e(entry, exit, "return")}, + ) + if got == nil || len(got) != 0 { + t.Errorf("empty-body cdg = %#v, want empty non-nil", got) + } +} + +// TestCDG_Gate runs BuildCDG over every function and function literal body in +// testdata/dataflow and testdata/chi. Gates: two builds deep-equal; the CDG is +// independent of edge kinds (R10: every kind rewritten to one, plus a parallel +// twin of every edge with another kind) and of input edge order; every +// non-synthetic CFG node has an incoming CDG edge; every endpoint is a CFG +// node key (never a satellite); no self-loop, no edge into @entry or @exit; +// sorted and deduped; every node has a post-dominator chain ending at @exit. +func TestCDG_Gate(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata") + total, edgesN := 0, 0 + for _, dir := range []string{"dataflow", "chi"} { + fset, cases := loadDiffCases(t, filepath.Join(root, dir)) + for _, cs := range cases { + total++ + nodes, keyOf := dataflow.BuildNodes(fset, cs.body, gateCallKeys(fset, cs.body)) + kept, es := dataflow.BuildCFG(nodes, keyOf, cs.body, cs.info) + requireReachesExit(t, cs.name, kept, es) + got := dataflow.BuildCDG(kept, es) + edgesN += len(got) + + if again := dataflow.BuildCDG(kept, es); !reflect.DeepEqual(got, again) { + t.Errorf("%s: two builds differ", cs.name) + } + blind := make([]dataflow.CfgEdge, 0, 2*len(es)) + for i := len(es) - 1; i >= 0; i-- { + blind = append(blind, e(es[i].Src, es[i].Dst, "k"), e(es[i].Src, es[i].Dst, "k2")) + } + if other := dataflow.BuildCDG(kept, blind); !reflect.DeepEqual(got, other) { + t.Errorf("%s: CDG depends on edge kinds or order:\n got %v\nwant %v", cs.name, other, got) + } + gateCDG(t, cs.name, kept, es, got) + } + } + if total < 100 || edgesN == 0 { + t.Errorf("cdg gate covered only %d bodies / %d edges", total, edgesN) + } + t.Logf("cdg gate: %d bodies, %d cdg edges", total, edgesN) +} + +func gateCDG(t *testing.T, name string, nodes []dataflow.Node, es []dataflow.CfgEdge, cdg []dataflow.CdgEdge) { + t.Helper() + cfgNode := map[string]bool{} + for _, n := range nodes { + if n.Parent == "" { + cfgNode[n.Key] = true + } + } + in := map[string]bool{} + for _, g := range cdg { + if !cfgNode[g.Src] || !cfgNode[g.Dst] { + t.Errorf("%s: cdg %v endpoint is not a CFG node", name, g) + } + if g.Src == g.Dst { + t.Errorf("%s: cdg self-loop %v", name, g) + } + if g.Dst == entry || g.Dst == exit { + t.Errorf("%s: cdg edge into a synthetic node %v", name, g) + } + in[g.Dst] = true + } + for k := range cfgNode { + if k != entry && k != exit && !in[k] { + t.Errorf("%s: node %s has no incoming cdg edge", name, k) + } + } + checkCDGOrder(t, name, cdg) + + ipdom := dataflow.PostDominators(nodes, es) + for k := range cfgNode { + seen := map[string]bool{} + for cur := k; cur != exit; cur = ipdom[cur] { + if _, ok := ipdom[cur]; !ok || seen[cur] { + t.Errorf("%s: post-dominator chain of %s broken at %s", name, k, cur) + break + } + seen[cur] = true + } + } +} diff --git a/internal/dataflow/reachdefs.go b/internal/dataflow/reachdefs.go new file mode 100644 index 0000000..97dd592 --- /dev/null +++ b/internal/dataflow/reachdefs.go @@ -0,0 +1,153 @@ +package dataflow + +import ( + "math/bits" + "sort" +) + +// BuildDDG computes the syntactic data-dependence graph of one callable body +// (spec §4.5) by monotone forward may-reaching-definitions over k-limited +// access paths. nodes and cfg are BuildCFG's output; du is DefUseOf of every +// node, keyed by Key (a node absent from du has an empty DefUse). +// +// A definition is a (node, path) pair from the node's Defs or WeakDefs. Per +// node, OUT = GEN ∪ (IN − KILL), IN = ∪ OUT(pred): +// +// - GEN is every def and weak def of the node. +// - KILL is every def d, anywhere in the body, with p.Covers(d.path) for a +// strong def p of the node (`x = …` kills x, x.f, x[*]; `x.f = …` kills +// x.f… but not x). Weak defs kill nothing. Strong defs carry no wildcard +// (DefUseOf), so a kill never over-reaches. A deferred call node only +// weak-defines (R11), so nothing is killed through the exit chain. +// +// A node's uses read IN — the defs reaching it before its own defs apply — +// so `y = y + 1` in a loop links to itself across the back edge, never +// within one visit. Each use u at n links from every def d in IN(n) whose +// path interferes with it, d.path.Covers(u) || u.Covers(d.path), as edge +// d.node → n with Var = u.Spell: one edge per distinct use spelling. +// +// The analysis is structural (R10): edge kinds are ignored and parallel +// edges are one. The CFG nodes are the nodes with no Parent; satellite call +// nodes carry an empty DefUse (DefUseOf) and are never endpoints (R13). +// Edges with an endpoint outside that set are ignored. +// +// Every iteration order derives from sorted keys and sorted adjacency, and +// def-sets are bitsets, so the result does not depend on the order of nodes, +// cfg or the DefUse lists. The output is deduplicated, sorted by (Src, Dst, +// Var) and non-nil. Its prov (["ssa"] at L3) is added at emission. +func BuildDDG(nodes []Node, cfg []CfgEdge, du map[string]DefUse) []DdgEdge { + g := newPdomGraph(nodes, cfg) + n := len(g.keys) + + // Definitions, numbered in (node key, list) order; deduplicated per node. + type def struct { + node int + path Path + } + var defs []def + genOf := make([][]int, n) + for v, k := range g.keys { + d := du[k] + seen := map[Path]bool{} + for _, list := range [][]Path{d.Defs, d.WeakDefs} { + for _, p := range list { + if !seen[p] { + seen[p] = true + genOf[v] = append(genOf[v], len(defs)) + defs = append(defs, def{v, p}) + } + } + } + } + + words := (len(defs) + 63) / 64 + newSet := func() []uint64 { return make([]uint64, words) } + gen := make([][]uint64, n) + kill := make([][]uint64, n) + for v, k := range g.keys { + gen[v], kill[v] = newSet(), newSet() + for _, i := range genOf[v] { + gen[v][i/64] |= 1 << (i % 64) + } + for _, p := range du[k].Defs { + for i, d := range defs { + if p.Covers(d.path) { + kill[v][i/64] |= 1 << (i % 64) + } + } + } + } + + in := make([][]uint64, n) + out := make([][]uint64, n) + for v := range in { + in[v], out[v] = newSet(), newSet() + copy(out[v], gen[v]) + } + queued := make([]bool, n) + work := make([]int, n) + for v := range work { + work[v], queued[v] = v, true + } + for len(work) > 0 { + v := work[0] + work = work[1:] + queued[v] = false + for _, p := range g.pred[v] { + for w, x := range out[p] { + in[v][w] |= x + } + } + changed := false + for w := range out[v] { + x := gen[v][w] | in[v][w]&^kill[v][w] + if x != out[v][w] { + out[v][w], changed = x, true + } + } + if changed { + for _, s := range g.succ[v] { + if !queued[s] { + queued[s] = true + work = append(work, s) + } + } + } + } + + edges := map[DdgEdge]bool{} + for v, k := range g.keys { + uses := du[k].Uses + if len(uses) == 0 { + continue + } + for w, x := range in[v] { + for x != 0 { + b := bits.TrailingZeros64(x) + x &^= 1 << b + d := defs[w*64+b] + for _, u := range uses { + if d.path.Covers(u) || u.Covers(d.path) { + edges[DdgEdge{Src: g.keys[d.node], Dst: k, Var: u.Spell}] = true + } + } + } + } + } + + res := make([]DdgEdge, 0, len(edges)) + for e := range edges { + res = append(res, e) + } + sort.Slice(res, func(i, j int) bool { + a, b := res[i], res[j] + if a.Src != b.Src { + return a.Src < b.Src + } + if a.Dst != b.Dst { + return a.Dst < b.Dst + } + return a.Var < b.Var + }) + return res +} diff --git a/internal/dataflow/reachdefs_test.go b/internal/dataflow/reachdefs_test.go new file mode 100644 index 0000000..2868f0b --- /dev/null +++ b/internal/dataflow/reachdefs_test.go @@ -0,0 +1,445 @@ +package dataflow_test + +import ( + "go/ast" + "go/token" + "go/types" + "path/filepath" + "reflect" + "runtime" + "sort" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" +) + +func dd(src, dst, v string) dataflow.DdgEdge { return dataflow.DdgEdge{Src: src, Dst: dst, Var: v} } + +// duMap is DefUseOf for every node of the pruned set, keyed by node Key — +// BuildDDG's input, computed the way the analyzer computes it. +func duMap(nodes []dataflow.Node, info *types.Info, fn ast.Node, keyOf map[ast.Node]string) map[string]dataflow.DefUse { + bases := dataflow.EntryBases(fn, info) + out := map[string]dataflow.DefUse{} + for _, n := range nodes { + out[n.Key] = dataflow.DefUseOf(n, info, kTest, bases, fn, keyOf) + } + return out +} + +// ddgOf builds fn's pruned nodes, CFG, CDG and DDG from the flow fixture with +// its real L1 call keys. +func ddgOf(t *testing.T, fn string) (kept []dataflow.Node, cdg []dataflow.CdgEdge, ddg []dataflow.DdgEdge) { + t.Helper() + fx := loadFlow(t) + d := fx.decl(t, fn) + nodes, keyOf := dataflow.BuildNodes(fx.fset, d.Body, fx.callKeys(t, fn)) + kept, es := dataflow.BuildCFG(nodes, keyOf, d.Body, fx.info) + checkContract(t, fn, kept, es) + ddg = dataflow.BuildDDG(kept, es, duMap(kept, fx.info, d, keyOf)) + checkDDG(t, fn, kept, ddg) + return kept, dataflow.BuildCDG(kept, es), ddg +} + +// checkDDG is the output contract: every endpoint is a pruned node and never +// a satellite call node (R13); sorted by (Src, Dst, Var), no duplicates; +// non-nil. +func checkDDG(t *testing.T, fn string, nodes []dataflow.Node, ddg []dataflow.DdgEdge) { + t.Helper() + if ddg == nil { + t.Errorf("%s: BuildDDG returned nil", fn) + } + byKey := map[string]dataflow.Node{} + for _, n := range nodes { + byKey[n.Key] = n + } + for _, g := range ddg { + for _, k := range []string{g.Src, g.Dst} { + n, ok := byKey[k] + if !ok { + t.Errorf("%s: ddg %v endpoint %s is not a pruned node", fn, g, k) + } else if n.Parent != "" { + t.Errorf("%s: ddg %v links satellite call %s (R13)", fn, g, k) + } + } + if g.Var == "" { + t.Errorf("%s: ddg %v has no var", fn, g) + } + } + for i := 1; i < len(ddg); i++ { + a, b := ddg[i-1], ddg[i] + if !ddgLess(a, b) { + t.Errorf("%s: ddg out of order or duplicated: %v before %v", fn, a, b) + } + } +} + +func ddgLess(a, b dataflow.DdgEdge) bool { + if a.Src != b.Src { + return a.Src < b.Src + } + if a.Dst != b.Dst { + return a.Dst < b.Dst + } + return a.Var < b.Var +} + +// wantDDG asserts fn's DDG is exactly want. +func wantDDG(t *testing.T, fn string, want ...dataflow.DdgEdge) []dataflow.DdgEdge { + t.Helper() + _, _, got := ddgOf(t, fn) + have := map[dataflow.DdgEdge]bool{} + for _, g := range got { + have[g] = true + } + exp := map[dataflow.DdgEdge]bool{} + for _, w := range want { + exp[w] = true + if !have[w] { + t.Errorf("%s: missing ddg %s→%s on %q", fn, w.Src, w.Dst, w.Var) + } + } + for _, g := range got { + if !exp[g] { + t.Errorf("%s: unexpected ddg %s→%s on %q", fn, g.Src, g.Dst, g.Var) + } + } + return got +} + +// Derivations below use the CFGs pinned in cfg_test.go and the DefUse sets +// pinned in defuse_test.go. IN(n) = ∪ OUT(pred); a node's uses read IN(n) +// (before its own defs); OUT(n) = GEN(n) ∪ (IN(n) − KILL(n)), KILL by the +// node's strong defs only. + +func TestDDG_LoopCarried(t *testing.T) { + // Bump. @entry D n; 146:2 D y; 147:2 (for header, Init and Post both + // define i: strong) D i U i,n; 148:3 D y U y; 150:2 U y. + // CFG: @entry→146:2→147:2; 147:2→148:3, 147:2→150:2; 148:3→147:2. + // IN(147:2) = OUT(146:2) ∪ OUT(148:3) = {n@entry, y@146, i@147, y@148}. + // i ← 147:2 (the post's i++ across the back edge), n ← @entry. + // IN(148:3) = OUT(147:2) = same set: y ← 146:2 (first trip) and 148:3 + // itself (loop-carried: its use reads IN, before its own def). + // IN(150:2) = OUT(147:2): y ← 146:2 (zero trips), 148:3. + wantDDG(t, "Bump", + dd(entry, "147:2", "n"), + dd("146:2", "148:3", "y"), + dd("146:2", "150:2", "y"), + dd("147:2", "147:2", "i"), + dd("148:3", "148:3", "y"), + dd("148:3", "150:2", "y"), + ) +} + +func TestDDG_ShadowedVariableDoesNotLeak(t *testing.T) { + // Shadow. 155:2 x(outer) := a; 157:3 x(inner) := a*2; 158:3 _ = x(inner); + // 160:2 return x(outer). The two x are distinct bases: the inner def + // reaches 160:2 (157:3→158:3→160:2) but does not interfere with the + // outer use, and the outer def never feeds 158:3. + got := wantDDG(t, "Shadow", + dd(entry, "155:2", "a"), + dd(entry, "156:2", "a"), + dd(entry, "157:3", "a"), + dd("155:2", "160:2", "x"), + dd("157:3", "158:3", "x"), + ) + for _, g := range got { + if g.Src == "157:3" && g.Dst == "160:2" { + t.Errorf("Shadow: inner x leaks to the outer use: %v", g) + } + } +} + +func TestDDG_FieldPrefix(t *testing.T) { + // Fields. @entry D p, v; 176:2 (p.f.g.h.i = v, past k=3) W p.f.g.* U p, v; + // 177:2 U p.f.g.x. The weak p.f.g.* covers p.f.g.x; the @entry def p is a + // prefix of it and survives (a weak def kills nothing). + wantDDG(t, "Fields", + dd(entry, "176:2", "p"), + dd(entry, "176:2", "v"), + dd(entry, "177:2", "p.f.g.x"), + dd("176:2", "177:2", "p.f.g.x"), + ) +} + +func TestDDG_DeferArgsFlowToExit(t *testing.T) { + // EarlyReturnDefer. Each registration weak-defines ":" of its + // deferred call node, which uses it; it reaches the chain along every + // exit path. a's args evaluate at registration (127:2 U a); the nested + // satellite g(a) (127:13) carries nothing and is never an endpoint. + // The pseudo paths compare by full spelling: :126:8 does not feed + // 127:8. + got := wantDDG(t, "EarlyReturnDefer", + dd(entry, "127:2", "a"), + dd(entry, "128:2", "a"), + dd(entry, "132:2", "a"), + dd("126:2", "126:8", ":126:8"), + dd("127:2", "127:8", ":127:8"), + ) + for _, g := range got { + if g.Src == "127:13" || g.Dst == "127:13" { + t.Errorf("EarlyReturnDefer: satellite 127:13 is a ddg endpoint: %v", g) + } + } +} + +func TestDDG_Chan(t *testing.T) { + // Chan. @entry D ch, v; 182:2 (ch <- v) W ch[*] U ch, v; 183:2 (w := <-ch) + // D w U ch, ch[*]; 184:2 U w. + // Use ch[*] at 183:2: 182:2's ch[*] (equal) and @entry's ch (a prefix). + // Use ch at 183:2: @entry's ch (equal) and 182:2's ch[*] (ch is a prefix + // of it — interference is symmetric). One edge per use spelling. + wantDDG(t, "Chan", + dd(entry, "182:2", "ch"), + dd(entry, "182:2", "v"), + dd(entry, "183:2", "ch"), + dd(entry, "183:2", "ch[*]"), + dd("182:2", "183:2", "ch"), + dd("182:2", "183:2", "ch[*]"), + dd("183:2", "184:2", "w"), + ) +} + +// TestDDG_KillRules locks gen/kill on a hand-built straight line, with no +// fixture in between: +// +// @entry D x.f, x.g +// 1:1 D x.f kills @entry's x.f only +// 1:2 W x.f, x.h.* kills nothing (1:1's x.f survives) +// 1:3 U x ← @entry x.g, 1:1 x.f, 1:2 x.f / x.h.* +// 1:4 D x U x uses read IN first: same three; then +// x kills every x… def (x covers them) +// 1:5 U x.f ← 1:4 only (x is a prefix of x.f) +// 1:6 D x.f.g U x.f a field def does not kill x: 1:4 only +// 1:7 U x ← 1:4 x, 1:6 x.f.g +func TestDDG_KillRules(t *testing.T) { + x := types.NewVar(token.NoPos, nil, "x", types.Typ[types.Int]) + P := func(s ...string) []dataflow.Path { + var out []dataflow.Path + for _, sp := range s { + out = append(out, dataflow.Path{Base: x, Spell: sp}) + } + return out + } + keys := []string{entry, "1:1", "1:2", "1:3", "1:4", "1:5", "1:6", "1:7", exit} + var nodes []dataflow.Node + var cfg []dataflow.CfgEdge + for i, k := range keys { + nodes = append(nodes, dataflow.Node{Key: k}) + if i > 0 { + cfg = append(cfg, dataflow.CfgEdge{Src: keys[i-1], Dst: k, Kind: "fallthrough"}) + } + } + du := map[string]dataflow.DefUse{ + entry: {Defs: P("x.f", "x.g")}, + "1:1": {Defs: P("x.f")}, + "1:2": {WeakDefs: P("x.f", "x.h.*")}, + "1:3": {Uses: P("x")}, + "1:4": {Defs: P("x"), Uses: P("x")}, + "1:5": {Uses: P("x.f")}, + "1:6": {Defs: P("x.f.g"), Uses: P("x.f")}, + "1:7": {Uses: P("x")}, + } + got := dataflow.BuildDDG(nodes, cfg, du) + want := []dataflow.DdgEdge{ + dd("1:1", "1:3", "x"), + dd("1:1", "1:4", "x"), + dd("1:2", "1:3", "x"), + dd("1:2", "1:4", "x"), + dd("1:4", "1:5", "x.f"), + dd("1:4", "1:6", "x.f"), + dd("1:4", "1:7", "x"), + dd("1:6", "1:7", "x"), + dd(entry, "1:3", "x"), + dd(entry, "1:4", "x"), + } + if !reflect.DeepEqual(got, want) { + t.Errorf("kill rules:\n got %v\nwant %v", got, want) + } +} + +// sliceOf is the backward intraprocedural slice from criterion: reverse +// reachability over cdg ∪ ddg, criterion and @entry included, @exit excluded. +func sliceOf(cdg []dataflow.CdgEdge, ddg []dataflow.DdgEdge, criterion string) []string { + pred := map[string][]string{} + for _, e := range cdg { + pred[e.Dst] = append(pred[e.Dst], e.Src) + } + for _, e := range ddg { + pred[e.Dst] = append(pred[e.Dst], e.Src) + } + seen := map[string]bool{criterion: true} + work := []string{criterion} + for len(work) > 0 { + n := work[len(work)-1] + work = work[:len(work)-1] + for _, p := range pred[n] { + if !seen[p] { + seen[p] = true + work = append(work, p) + } + } + } + delete(seen, exit) + out := []string{} + for k := range seen { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// bumpSlice is the backward slice of Bump from its `return y` (150:2) on y, +// derived by hand before BuildDDG existed: +// +// ddg (TestDDG_LoopCarried): @entry→147:2 n; 146:2→148:3 y; 146:2→150:2 y; +// 147:2→147:2 i; 148:3→148:3 y; 148:3→150:2 y. +// cdg (postdom_test): 147:2→148:3; @entry→146:2, 147:2, 150:2. +// 150:2 ← 146:2, 148:3 (ddg y), @entry (cdg). +// 148:3 ← 146:2, 148:3 (ddg y), 147:2 (cdg: the loop header controls it). +// 147:2 ← @entry (ddg n — @entry defines the param n; and cdg), itself. +// 146:2 ← @entry (cdg). +// +// So every node but @exit. @entry IS included: it defines the parameter n +// that the loop condition reads, and it anchors the uncontrolled nodes. +var bumpSlice = []string{"146:2", "147:2", "148:3", "150:2", entry} + +// guardSlice is the backward slice of Guard (appended to flow.go) from its +// `return r` (476:2) on r. DefUse: @entry D a, c; 470:2 r := 0 D r; +// 471:2 d := a + 1 D d U a; 472:2 if c > 0 U c; 473:3 r = a D r U a; +// 475:2 _ = d U d; 476:2 return r U r. +// +// ddg: @entry→471:2 a; @entry→472:2 c; @entry→473:3 a; 471:2→475:2 d; +// 470:2→476:2 r; 473:3→476:2 r. +// cdg: 472:2→473:3; @entry→470:2, 471:2, 472:2, 475:2, 476:2 (the if has +// no else and rejoins at 475:2, which post-dominates it). +// 476:2 ← 470:2, 473:3 (ddg r), @entry (cdg). +// 473:3 ← @entry (ddg a), 472:2 (cdg). +// 472:2 ← @entry (ddg c, cdg). 470:2 ← @entry (cdg). +// +// 472:2 is reached only through the CDG: the DDG alone from 476:2 gives +// {476:2, 470:2, 473:3, @entry}. 471:2 and 475:2 (the d chain) are out. +// @entry is included. +var guardSlice = []string{"470:2", "472:2", "473:3", "476:2", entry} + +func TestSlice_BackwardExact(t *testing.T) { + _, cdg, ddg := ddgOf(t, "Bump") + if got := sliceOf(cdg, ddg, "150:2"); !reflect.DeepEqual(got, bumpSlice) { + t.Errorf("Bump slice from 150:2 = %v, want %v", got, bumpSlice) + } + + _, cdg, ddg = ddgOf(t, "Guard") + if got := sliceOf(cdg, ddg, "476:2"); !reflect.DeepEqual(got, guardSlice) { + t.Errorf("Guard slice from 476:2 = %v, want %v", got, guardSlice) + } + // The CDG contributes: the DDG alone does not reach the if header. + if got := sliceOf(nil, ddg, "476:2"); reflect.DeepEqual(got, guardSlice) { + t.Errorf("Guard: DDG-only slice %v already equals the PDG slice; the gate does not exercise the CDG", got) + } +} + +// TestDDG_CorpusGate runs BuildDDG over every function and function literal +// body in testdata/dataflow and testdata/chi: no panic, the output contract +// (checkDDG: endpoints are pruned non-satellite nodes, sorted, deduped), two +// builds deep-equal, and the result independent of input order (nodes, CFG +// edges and every DefUse list reversed). +func TestDDG_CorpusGate(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata") + total, edges := 0, 0 + for _, dir := range []string{"dataflow", "chi"} { + fset, cases := loadDiffCases(t, filepath.Join(root, dir)) + for _, c := range cases { + total++ + nodes, keyOf := dataflow.BuildNodes(fset, c.body, gateCallKeys(fset, c.body)) + kept, es := dataflow.BuildCFG(nodes, keyOf, c.body, c.info) + du := duMap(kept, c.info, c.fn, keyOf) + got := dataflow.BuildDDG(kept, es, du) + checkDDG(t, c.name, kept, got) + edges += len(got) + + if again := dataflow.BuildDDG(kept, es, duMap(kept, c.info, c.fn, keyOf)); !reflect.DeepEqual(got, again) { + t.Errorf("%s: two builds differ", c.name) + } + rn := append([]dataflow.Node(nil), kept...) + reverse(rn) + re := append([]dataflow.CfgEdge(nil), es...) + reverse(re) + rdu := map[string]dataflow.DefUse{} + for k, v := range du { + rdu[k] = dataflow.DefUse{Defs: reversed(v.Defs), WeakDefs: reversed(v.WeakDefs), Uses: reversed(v.Uses)} + } + if perm := dataflow.BuildDDG(rn, re, rdu); !reflect.DeepEqual(got, perm) { + t.Errorf("%s: DDG depends on input order", c.name) + } + } + } + if total < 100 || edges < 1000 { + t.Errorf("ddg gate covered only %d bodies / %d edges", total, edges) + } + t.Logf("ddg gate: %d bodies, %d edges", total, edges) +} + +func reverse[T any](s []T) { + for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } +} + +func reversed(ps []dataflow.Path) []dataflow.Path { + out := append([]dataflow.Path(nil), ps...) + reverse(out) + return out +} + +// hasDDG asserts fn's DDG contains each of want (not necessarily only them). +func hasDDG(t *testing.T, fn string, want ...dataflow.DdgEdge) { + t.Helper() + _, _, got := ddgOf(t, fn) + have := map[dataflow.DdgEdge]bool{} + for _, g := range got { + have[g] = true + } + for _, w := range want { + if !have[w] { + t.Errorf("%s: missing ddg %s→%s on %q; have %v", fn, w.Src, w.Dst, w.Var, got) + } + } +} + +// TestDDG_SubsliceArgWritesBase (C1): a callee writing through a subslice +// (buf[:n], arr[:], dst[i:]) writes the base's elements: the call +// weak-defines base[*], which reaches a later read of the base. +func TestDDG_SubsliceArgWritesBase(t *testing.T) { + hasDDG(t, "SubsliceRead", dd("487:2", "488:2", "buf[*]")) + hasDDG(t, "ArraySliceRead", dd("494:2", "495:2", "arr[*]")) + hasDDG(t, "CopyInto", dd("501:2", "502:2", "dst[*]")) +} + +// TestDDG_DeferredCallReadsStateAtExit (I7): a deferred call runs at exit and +// reads through its receiver and reference-typed arguments then, so a write +// after its registration reaches it. +func TestDDG_DeferredCallReadsStateAtExit(t *testing.T) { + hasDDG(t, "DeferFlush", + dd("524:2", "522:8", "w.*"), // w.buf = nil → deferred w.Flush() + dd("525:2", "523:8", "b[*]"), // b[0] = 1 → deferred fill(b[1:]) + ) +} + +// TestDDG_AppendWritesDstBackingArray: append(dst[:0], x) writes x into dst's +// backing array, so it weak-defines dst[*], which reaches a later dst[0]. +func TestDDG_AppendWritesDstBackingArray(t *testing.T) { + hasDDG(t, "AppendReuse", dd("541:2", "542:2", "dst[*]")) +} + +// TestDDG_DeferredNamedResultWriteNeverKills (R11): in DeferWrap the exit +// chain is return 558:2 → 557:8 (err = wrapErr(err)) → 556:8 (observe(err)) +// → @exit. A deferred call node never strong-defines, so err = wrapErr(nil) +// (555:2) reaches the return, the rewriting closure and, through it, the +// observer at the end of the chain. +func TestDDG_DeferredNamedResultWriteNeverKills(t *testing.T) { + hasDDG(t, "DeferWrap", + dd("555:2", "558:2", "err"), // the bare return reads the named result + dd("555:2", "557:8", "err"), // the rewriting closure reads it at exit + dd("555:2", "556:8", "err"), // and the def survives past it + ) +} diff --git a/internal/dataflow/run.go b/internal/dataflow/run.go new file mode 100644 index 0000000..fb3e01e --- /dev/null +++ b/internal/dataflow/run.go @@ -0,0 +1,122 @@ +package dataflow + +import ( + "go/ast" + "go/token" + "go/types" + "runtime" + "runtime/debug" + "sync" + + "github.com/codellm-devkit/codeanalyzer-go/internal/utils" +) + +// L3Key names one callable of the symbol table: its declaring file (a +// symbol_table key; for a closure, its enclosing callable's file) and its +// signature (for a closure, ".closure_N"). It is the join key between +// Run's side table and the emitted tree — the emitter looks each callable up +// by (declaring file, signature), so a method declared apart from its +// receiver type still finds its result. +type L3Key struct{ Path, Signature string } + +// Unit is the input for one callable. The caller (the symbol-table builder, +// which owns the AST walk) builds one per callable that has a body: +// +// - Fn is the *ast.FuncDecl or *ast.FuncLit; a closure is its own unit. +// - Info is the type info of Fn's package (nil degrades: no def/use). +// - Calls is the callable's L1 call-node keys in L1 recording order +// (v2.CallBodyKeys over its call sites). BuildNodes pairs them with the +// body's calls by index (ruling R5); Run checks the counts first. +type Unit struct { + Key L3Key + Fn ast.Node + Info *types.Info + Calls []string +} + +// Run builds the L3 overlay of every unit: the pruned body nodes, CFG, CDG +// and DDG (k = the access-path limit, --graph-field-depth). Units fan out +// over jobs workers (jobs < 1 means GOMAXPROCS); each worker writes only its +// own result slot and no id is minted here, so the map is the same for any +// jobs value and its iteration order never reaches the output. +// +// A unit that panics in any stage is skipped the same way (runOne). +// +// A unit whose L1 call-key count differs from its body's recorded call count +// gets no result and a warning on stderr: pairing by index would mis-key its +// call nodes, so its body stays at L1 rather than being silently wrong. +func Run(fset *token.FileSet, units []Unit, k, jobs int) map[L3Key]Result { + if jobs < 1 { + jobs = runtime.GOMAXPROCS(0) + } + results := make([]*Result, len(units)) + next := make(chan int) + var wg sync.WaitGroup + for w := 0; w < jobs; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range next { + results[i] = runOne(fset, units[i], k) + } + }() + } + for i := range units { + next <- i + } + close(next) + wg.Wait() + + out := make(map[L3Key]Result, len(units)) + for i, r := range results { + if r != nil { + out[units[i].Key] = *r + } + } + return out +} + +// runOne builds one unit's result. A panic in any stage (a latent bug in the +// CFG, def-use or DDG builders) is recovered: that callable stays at L1 with a +// warning (and, at -vv, the panic's stack), and the rest of the run — and the +// L1/L2 output — is unaffected. +func runOne(fset *token.FileSet, u Unit, k int) (res *Result) { + defer func() { + if r := recover(); r != nil { + utils.Warn("L3 skipped for %s (%s): %v", u.Key.Signature, u.Key.Path, r) + utils.Debug("L3 panic in %s (%s):\n%s", u.Key.Signature, u.Key.Path, debug.Stack()) + res = nil + } + }() + body := funcBody(u.Fn) + if body == nil { + return nil + } + if n := RecordedCallCount(body); n != len(u.Calls) { + utils.Warn("L3 skipped for %s (%s): %d L1 call keys for %d calls in the body", u.Key.Signature, u.Key.Path, len(u.Calls), n) + return nil + } + nodes, keyOf := BuildNodes(fset, body, u.Calls) + kept, cfg := BuildCFG(nodes, keyOf, body, u.Info) + bases := EntryBases(u.Fn, u.Info) + du := make(map[string]DefUse, len(kept)) + for _, n := range kept { + du[n.Key] = DefUseOf(n, u.Info, k, bases, u.Fn, keyOf) + } + return &Result{ + Nodes: kept, + Cfg: cfg, + Cdg: BuildCDG(kept, cfg), + Ddg: BuildDDG(kept, cfg, du), + } +} + +func funcBody(fn ast.Node) *ast.BlockStmt { + switch fn := fn.(type) { + case *ast.FuncDecl: + return fn.Body + case *ast.FuncLit: + return fn.Body + } + return nil +} diff --git a/internal/dataflow/run_test.go b/internal/dataflow/run_test.go new file mode 100644 index 0000000..0e723c8 --- /dev/null +++ b/internal/dataflow/run_test.go @@ -0,0 +1,349 @@ +package dataflow_test + +import ( + "bytes" + "encoding/json" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" + "github.com/codellm-devkit/codeanalyzer-go/internal/schema" + "github.com/codellm-devkit/codeanalyzer-go/internal/semantic_analysis" + "github.com/codellm-devkit/codeanalyzer-go/internal/syntactic_analysis" + "github.com/codellm-devkit/codeanalyzer-go/internal/syntactic_analysis/v2emit" + "github.com/codellm-devkit/codeanalyzer-go/internal/utils" +) + +// TestRun_JobsDeterminism locks spec §4.12: the L3 output of testdata/chi is +// byte-identical across -j 1 and -j 8, and across two -j 8 runs. +func TestRun_JobsDeterminism(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + dir, err := filepath.Abs(filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata", "chi")) + if err != nil { + t.Fatal(err) + } + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("symbol table build failed: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()} + units := b.L3Units(st) + if len(units) < 100 { + t.Fatalf("chi yields only %d L3 units", len(units)) + } + + emit := func(jobs int) []byte { + l3 := dataflow.Run(b.Fset(), units, 3, jobs) + if len(l3) != len(units) { + t.Errorf("-j %d: %d results for %d units", jobs, len(l3), len(units)) + } + data, err := json.Marshal(v2emit.Emit(app, "chi", dir, 3, "det", 3, l3)) + if err != nil { + t.Fatal(err) + } + return data + } + one, eight, again := emit(1), emit(8), emit(8) + if !bytes.Equal(one, eight) { + t.Error("-j 1 and -j 8 outputs differ") + } + if !bytes.Equal(eight, again) { + t.Error("two -j 8 runs differ") + } + if !bytes.Contains(one, []byte(`"cfg":[`)) || !bytes.Contains(one, []byte(`"ddg":[`)) { + t.Error("L3 output carries no cfg/ddg edge lists") + } +} + +// lineDirectiveSrc maps its body to a different (virtual) file and line with +// a //line directive. L1 records call positions through fset.Position, which +// honours the directive, so the call keys name virtual lines; pairing them +// with the AST calls by recording index must still work (ruling R5). +const lineDirectiveSrc = `package p + +func f(x int) int { return x } + +func g(x int) int { +//line virtual.go:900:1 + y := f(f(x)) + if y > 0 { + f(y) + } + return y +} +` + +func parseCheck(t *testing.T, src string) (*token.FileSet, *ast.File, *types.Info) { + t.Helper() + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "p.go", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + info := &types.Info{ + Types: map[ast.Expr]types.TypeAndValue{}, Defs: map[*ast.Ident]types.Object{}, + Uses: map[*ast.Ident]types.Object{}, Implicits: map[ast.Node]types.Object{}, + Selections: map[*ast.SelectorExpr]*types.Selection{}, Scopes: map[ast.Node]*types.Scope{}, + } + if _, err := (&types.Config{Importer: importer.Default()}).Check("p", fset, []*ast.File{f}, info); err != nil { + t.Fatal(err) + } + return fset, f, info +} + +func funcDecl(t *testing.T, f *ast.File, name string) *ast.FuncDecl { + t.Helper() + for _, d := range f.Decls { + if fd, ok := d.(*ast.FuncDecl); ok && fd.Name.Name == name { + return fd + } + } + t.Fatalf("no func %s", name) + return nil +} + +// TestRun_LineDirective: the call keys of a body under a //line directive +// land on the call nodes, and spans carry the directive-adjusted positions +// the keys name. +func TestRun_LineDirective(t *testing.T) { + fset, f, info := parseCheck(t, lineDirectiveSrc) + g := funcDecl(t, f, "g") + keys := gateCallKeys(fset, g.Body) + // Virtual positions; the file has only 12 real lines, so rebuilding a + // token.Pos from them (File.LineStart(900)) would panic. + if strings.Join(keys, ",") != "900:7,900:9,902:3" { + t.Fatalf("call keys = %v, want the virtual 900:7, 900:9, 902:3", keys) + } + key := dataflow.L3Key{Path: "p.go", Signature: "p.g"} + out := dataflow.Run(fset, []dataflow.Unit{{Key: key, Fn: g, Info: info, Calls: keys}}, 3, 1) + res, ok := out[key] + if !ok { + t.Fatal("no result for g") + } + byKey := map[string]dataflow.Node{} + for _, n := range res.Nodes { + byKey[n.Key] = n + } + for _, k := range keys { + if n, ok := byKey[k]; !ok || n.Kind != dataflow.KindCall { + t.Errorf("call key %s has no call node", k) + } + } + if n := byKey["901:2"]; n.Kind != dataflow.KindBranch || n.StartLC != [2]int{901, 2} || n.EndLC != [2]int{903, 3} { + t.Errorf("if node = %+v, want branch spanning 901:2–903:3", n) + } +} + +// TestRun_CallCountMismatchSkips: a unit whose L1 key list does not match +// its body's calls gets no result (never a silently mis-keyed one); other +// units are unaffected. +func TestRun_CallCountMismatchSkips(t *testing.T) { + fset, f, info := parseCheck(t, lineDirectiveSrc) + g, fn := funcDecl(t, f, "g"), funcDecl(t, f, "f") + bad := dataflow.L3Key{Path: "p.go", Signature: "p.g"} + good := dataflow.L3Key{Path: "p.go", Signature: "p.f"} + out := dataflow.Run(fset, []dataflow.Unit{ + {Key: bad, Fn: g, Info: info, Calls: []string{"900:7"}}, // 1 key, 3 calls + {Key: good, Fn: fn, Info: info, Calls: nil}, + }, 3, 2) + if _, ok := out[bad]; ok { + t.Error("mismatched unit produced a result") + } + if r, ok := out[good]; !ok || len(r.Cfg) == 0 { + t.Errorf("good unit result = %+v, %v", r, ok) + } +} + +// TestRun_ResultContract: on the flow fixture every result has @entry and +// @exit, a non-empty CFG and edges only between its nodes; DDG edges never +// touch a satellite (R13). +func TestRun_ResultContract(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata", "dataflow") + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatal(err) + } + units := b.L3Units(st) + out := dataflow.Run(b.Fset(), units, 3, 4) + if len(out) != len(units) || len(out) < 50 { + t.Fatalf("%d results for %d units", len(out), len(units)) + } + closure := dataflow.L3Key{Path: filepath.Join("flow", "flow.go"), Signature: "example.com/dataflow/flow.Closure.closure_1"} + if _, ok := out[closure]; !ok { + t.Errorf("no result for %v", closure) + } + for key, r := range out { + nodes := map[string]dataflow.Node{} + for _, n := range r.Nodes { + nodes[n.Key] = n + } + if _, ok := nodes[dataflow.EntryKey]; !ok { + t.Errorf("%v: no @entry", key) + } + if _, ok := nodes[dataflow.ExitKey]; !ok { + t.Errorf("%v: no @exit", key) + } + if len(r.Cfg) == 0 { + t.Errorf("%v: empty cfg", key) + } + for _, e := range r.Cfg { + if _, ok := nodes[e.Src]; !ok { + t.Errorf("%v: cfg src %s dangling", key, e.Src) + } + if _, ok := nodes[e.Dst]; !ok { + t.Errorf("%v: cfg dst %s dangling", key, e.Dst) + } + } + for _, e := range r.Ddg { + if nodes[e.Src].Parent != "" || nodes[e.Dst].Parent != "" { + t.Errorf("%v: ddg edge %v touches a satellite", key, e) + } + } + for _, n := range r.Nodes { + if n.Kind != dataflow.KindEntry && n.Kind != dataflow.KindExit && n.StartLC[0] == 0 { + t.Errorf("%v: node %s has no position", key, n.Key) + } + } + } +} + +// TestRun_PanicInUnitIsContained: a unit whose body makes a builder panic +// (a switch body holding a non-case statement, which BuildNodes asserts +// against) gets no result; the run completes and the other units are built. +func TestRun_PanicInUnitIsContained(t *testing.T) { + fset, f, info := parseCheck(t, lineDirectiveSrc) + bad := &ast.FuncDecl{ + Name: ast.NewIdent("bad"), + Type: &ast.FuncType{Params: &ast.FieldList{}}, + Body: &ast.BlockStmt{List: []ast.Stmt{ + &ast.SwitchStmt{Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: ast.NewIdent("x")}}}}, + }}, + } + badKey := dataflow.L3Key{Path: "p.go", Signature: "p.bad"} + goodF := dataflow.L3Key{Path: "p.go", Signature: "p.f"} + goodG := dataflow.L3Key{Path: "p.go", Signature: "p.g"} + g := funcDecl(t, f, "g") + for _, jobs := range []int{1, 4} { + out := dataflow.Run(fset, []dataflow.Unit{ + {Key: goodF, Fn: funcDecl(t, f, "f"), Info: info}, + {Key: badKey, Fn: bad, Info: info}, + {Key: goodG, Fn: g, Info: info, Calls: gateCallKeys(fset, g.Body)}, + }, 3, jobs) + if _, ok := out[badKey]; ok { + t.Errorf("-j %d: panicking unit produced a result", jobs) + } + for _, k := range []dataflow.L3Key{goodF, goodG} { + if r, ok := out[k]; !ok || len(r.Cfg) == 0 { + t.Errorf("-j %d: %v result = %+v, %v; want a CFG", jobs, k, r, ok) + } + } + } + + // The warning is always printed; at -vv the panic's stack follows it, so + // the failing builder can be found. + run := func() { + dataflow.Run(fset, []dataflow.Unit{{Key: badKey, Fn: bad, Info: info}}, 3, 1) + } + if quiet := captureStderr(t, run); !strings.Contains(quiet, "L3 skipped for p.bad") || strings.Contains(quiet, "goroutine ") { + t.Errorf("default verbosity: want the warning and no stack; got:\n%s", quiet) + } + utils.SetVerbosity(2) + defer utils.SetVerbosity(0) + if loud := captureStderr(t, run); !strings.Contains(loud, "L3 skipped for p.bad") || + !strings.Contains(loud, "DEBUG") || !strings.Contains(loud, "dataflow.BuildNodes") { + t.Errorf("-vv: want the warning and a stack naming the panicking builder; got:\n%s", loud) + } +} + +// TestRun_TypeErrorsKeepDefs (I2): in a package that does not type-check +// (testdata/typeerr imports a missing package), go/types records no field +// selection on a value of the unresolved type and no object for an undeclared +// name. The field write must still be a def that reaches the later read. +func TestRun_TypeErrorsKeepDefs(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata", "typeerr") + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatal(err) + } + out := dataflow.Run(b.Fset(), b.L3Units(st), 3, 1) + for _, c := range []struct{ fn, def, use, v string }{ + {"Forward", "11:2", "12:2", "req.Body"}, // unresolved type, resolved base + {"Undeclared", "18:2", "19:2", "undeclared.Body"}, + {"Qualified", "24:2", "25:2", "example.com/typeerr/missing::Cfg.Body"}, + {"Asserted", "30:2", "31:2", "x"}, // x.(*missing.Req).Body = src weak-defines x.* + } { + key := dataflow.L3Key{Path: "typeerr.go", Signature: "example.com/typeerr." + c.fn} + r, ok := out[key] + if !ok { + t.Errorf("no result for %v", key) + continue + } + found := false + for _, e := range r.Ddg { + found = found || (e.Src == c.def && e.Dst == c.use && e.Var == c.v) + } + if !found { + t.Errorf("%s: no ddg %s→%s on %q; have %v", c.fn, c.def, c.use, c.v, r.Ddg) + } + } +} + +// captureStderr returns what fn writes to os.Stderr. +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stderr + os.Stderr = w + done := make(chan string) + go func() { + var buf bytes.Buffer + _, _ = io.Copy(&buf, r) + done <- buf.String() + }() + defer func() { os.Stderr = old }() + fn() + _ = w.Close() + os.Stderr = old + return <-done +} + +// TestRun_FlowFixtureHasNoCallCountMismatch: L1's call walk (buildCallSites) +// and L3's (recordedCalls) agree on every flow-fixture body — including calls +// in a `go` statement's arguments — so no unit is skipped with a warning. +func TestRun_FlowFixtureHasNoCallCountMismatch(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata", "dataflow") + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatal(err) + } + units := b.L3Units(st) + var out map[dataflow.L3Key]dataflow.Result + stderr := captureStderr(t, func() { out = dataflow.Run(b.Fset(), units, 3, 4) }) + if strings.Contains(stderr, "L1 call keys for") { + t.Errorf("call-count mismatch warnings:\n%s", stderr) + } + if len(out) != len(units) { + t.Errorf("%d results for %d units", len(out), len(units)) + } +} diff --git a/internal/neo4j/bolt.go b/internal/neo4j/bolt.go index 1a26181..6dc6c08 100644 --- a/internal/neo4j/bolt.go +++ b/internal/neo4j/bolt.go @@ -89,7 +89,7 @@ func (w *boltWriter) run() error { } root, scoped := applicationPrefix(w.appID) - if w.cfg.Eager && scoped { + if w.shouldEagerPurge(scoped) { if err := w.eagerPurge(root); err != nil { return err } @@ -198,12 +198,20 @@ func (w *boltWriter) schemaVersionMismatch() (bool, error) { // ─── destructive steps (--eager only) ───────────────────────────────────────── +// eagerPurge runs as an implicit (auto-commit) transaction: Neo4j rejects +// CALL { ... } IN TRANSACTIONS inside an explicit one, which is what +// ExecuteWrite opens. +// shouldEagerPurge reports whether to delete the app's whole subgraph before +// pushing. The purge is app-wide, so it runs only on a full run: with +// --target-files the push carries only the targeted packages, and purging +// first would leave the rest of the app missing from the graph. +func (w *boltWriter) shouldEagerPurge(scoped bool) bool { + return w.cfg.Eager && w.cfg.FullRun && scoped +} + func (w *boltWriter) eagerPurge(root string) error { q, params := eagerPurgeQuery(root) - return w.write(func(tx neo4j.ManagedTransaction) error { - _, err := tx.Run(w.ctx, q, params) - return err - }) + return w.autoCommit(q, params) } // purgeModule deletes the edges and declarations a module owned, scoped to its @@ -355,6 +363,17 @@ func (w *boltWriter) write(fn func(neo4j.ManagedTransaction) error) error { return err } +// autoCommit runs one statement as an implicit transaction and consumes its +// result, so a server-side failure is returned here rather than dropped. +func (w *boltWriter) autoCommit(q string, params map[string]any) error { + res, err := w.session.Run(w.ctx, q, params) + if err != nil { + return err + } + _, err = res.Consume(w.ctx) + return err +} + func (w *boltWriter) read(fn func(neo4j.ManagedTransaction) error) error { _, err := w.session.ExecuteRead(w.ctx, func(tx neo4j.ManagedTransaction) (any, error) { return nil, fn(tx) diff --git a/internal/neo4j/bolt_session_test.go b/internal/neo4j/bolt_session_test.go new file mode 100644 index 0000000..ce13d83 --- /dev/null +++ b/internal/neo4j/bolt_session_test.go @@ -0,0 +1,74 @@ +package neo4j + +import ( + "context" + "strings" + "testing" + + "github.com/neo4j/neo4j-go-driver/v5/neo4j" +) + +// recordingSession is a DB-free SessionWithContext: it records which path a +// statement takes (implicit Run vs an explicit ExecuteWrite transaction). +// Methods it does not override panic through the nil embedded interface. +type recordingSession struct { + neo4j.SessionWithContext + autoCommit []string + explicit int +} + +func (s *recordingSession) Run(_ context.Context, q string, _ map[string]any, _ ...func(*neo4j.TransactionConfig)) (neo4j.ResultWithContext, error) { + s.autoCommit = append(s.autoCommit, q) + return consumedResult{}, nil +} + +func (s *recordingSession) ExecuteWrite(context.Context, neo4j.ManagedTransactionWork, ...func(*neo4j.TransactionConfig)) (any, error) { + s.explicit++ + return nil, nil +} + +type consumedResult struct{ neo4j.ResultWithContext } + +func (consumedResult) Consume(context.Context) (neo4j.ResultSummary, error) { return nil, nil } + +// TestEagerPurge_RunsAsImplicitTransaction: Neo4j 5 rejects CALL { … } IN +// TRANSACTIONS inside an explicit transaction (what ExecuteWrite opens), so +// the batched eager purge must go through the session's implicit Run path. +func TestEagerPurge_RunsAsImplicitTransaction(t *testing.T) { + if q, _ := eagerPurgeQuery("can://app"); !strings.Contains(q, "IN TRANSACTIONS") { + t.Fatalf("eagerPurgeQuery is not batched: %q", q) + } + s := &recordingSession{} + w := &boltWriter{ctx: context.Background(), session: s} + if err := w.eagerPurge("can://app"); err != nil { + t.Fatal(err) + } + if s.explicit != 0 { + t.Errorf("eagerPurge opened %d explicit transaction(s)", s.explicit) + } + if len(s.autoCommit) != 1 || !strings.Contains(s.autoCommit[0], "IN TRANSACTIONS") { + t.Errorf("eagerPurge auto-commit statements = %q, want the one batched purge", s.autoCommit) + } +} + +// TestEagerPurge_OnlyOnFullRun: the app-wide purge deletes every node under the +// app prefix, so it must not run when --target-files limited the analysis to a +// few packages. Otherwise `--eager -t file.go` would wipe the app and push back +// only the targeted part. +func TestEagerPurge_OnlyOnFullRun(t *testing.T) { + cases := []struct { + eager, fullRun, scoped, want bool + }{ + {eager: true, fullRun: true, scoped: true, want: true}, + {eager: true, fullRun: false, scoped: true, want: false}, + {eager: false, fullRun: true, scoped: true, want: false}, + {eager: true, fullRun: true, scoped: false, want: false}, + } + for _, c := range cases { + w := &boltWriter{cfg: BoltConfig{Eager: c.eager, FullRun: c.fullRun}} + if got := w.shouldEagerPurge(c.scoped); got != c.want { + t.Errorf("eager=%v fullRun=%v scoped=%v: shouldEagerPurge = %v, want %v", + c.eager, c.fullRun, c.scoped, got, c.want) + } + } +} diff --git a/internal/neo4j/bolt_test.go b/internal/neo4j/bolt_test.go index 49a8e41..bb3723a 100644 --- a/internal/neo4j/bolt_test.go +++ b/internal/neo4j/bolt_test.go @@ -14,9 +14,9 @@ import ( func TestPartitionNodes_SharedHaveNoModule(t *testing.T) { b := NewRowBuilder() - b.Node([]string{LabelModule, MarkerLabel}, "id", "can://go/app/x.go", Props{"content_hash": "h1"}, "x.go") - b.Node([]string{LabelCallable, SymbolLabel, MarkerLabel}, "id", "can://go/app/x.go/f", nil, "x.go") - b.Node([]string{SymbolLabel, LabelExternal, MarkerLabel}, "id", "can://go/app/@external/fmt", nil, "") // shared + b.Node([]string{LabelModule, MarkerLabel}, "id", "can://app/go/x.go", Props{"content_hash": "h1"}, "x.go") + b.Node([]string{LabelCallable, SymbolLabel, MarkerLabel}, "id", "can://app/go/x.go/f", nil, "x.go") + b.Node([]string{SymbolLabel, LabelExternal, MarkerLabel}, "id", "can://app/@external/fmt", nil, "") // shared rows := b.Finish() byModule, shared := partitionNodes(rows.Nodes) @@ -30,8 +30,8 @@ func TestPartitionNodes_SharedHaveNoModule(t *testing.T) { func TestModuleHash_ReadsModuleNode(t *testing.T) { nodes := []NodeRow{ - {Labels: []string{LabelCallable}, Value: "can://go/app/x.go/f"}, - {Labels: []string{LabelModule}, Value: "can://go/app/x.go", Props: Props{"content_hash": "abc123"}}, + {Labels: []string{LabelCallable}, Value: "can://app/go/x.go/f"}, + {Labels: []string{LabelModule}, Value: "can://app/go/x.go", Props: Props{"content_hash": "abc123"}}, } if got := moduleHash(nodes); got != "abc123" { t.Errorf("moduleHash = %q, want abc123", got) @@ -44,15 +44,15 @@ func TestModuleHash_ReadsModuleNode(t *testing.T) { func TestDestructiveQueries_AreScopedAndAnchored(t *testing.T) { // Both destructive queries MUST filter on :GoCanNode and bind the prefix as a // parameter — never an unqualified DETACH DELETE. - eq, ep := eagerPurgeQuery("can://go/app") + eq, ep := eagerPurgeQuery("can://app") assertScoped(t, "eagerPurge", eq, ep) - if ep["p"] != "can://go/app/" || ep["root"] != "can://go/app" { + if ep["p"] != "can://app/" || ep["root"] != "can://app" { t.Errorf("eagerPurge params not the app prefix+root: %v", ep) } - pq, pp := purgeModuleQuery("can://go/app/x.go") + pq, pp := purgeModuleQuery("can://app/go/x.go") assertScoped(t, "purgeModule", pq, pp) - if pp["p"] != "can://go/app/x.go/" || pp["m"] != "can://go/app/x.go" { + if pp["p"] != "can://app/go/x.go/" || pp["m"] != "can://app/go/x.go" { t.Errorf("purgeModule params not the module prefix+id: %v", pp) } } diff --git a/internal/neo4j/cypher_test.go b/internal/neo4j/cypher_test.go index 7d3f8fb..74140de 100644 --- a/internal/neo4j/cypher_test.go +++ b/internal/neo4j/cypher_test.go @@ -45,10 +45,10 @@ func TestCypher_WipeIsPrefixScopedAndAnchored(t *testing.T) { script := renderFixture(t) // The wipe must match on the :GoCanNode marker and the can:/// prefix, // never a bare label like (:GoModule) or the whole store. - if !strings.Contains(script, "MATCH (n:GoCanNode) WHERE n.id STARTS WITH 'can://go/multipackage/'") { + if !strings.Contains(script, "MATCH (n:GoCanNode) WHERE n.id STARTS WITH 'can://multipackage/'") { t.Errorf("wipe not anchored on :GoCanNode + descendant prefix:\n%s", wipeBlock(script)) } - if !strings.Contains(script, "MATCH (n:GoCanNode) WHERE n.id = 'can://go/multipackage'") { + if !strings.Contains(script, "MATCH (n:GoCanNode) WHERE n.id = 'can://multipackage'") { t.Errorf("wipe does not match the root id separately:\n%s", wipeBlock(script)) } // A destructive statement must never be unscoped. @@ -63,8 +63,8 @@ func TestCypher_NoRootEmitsNoWipe(t *testing.T) { // A hand-built bag with no bare application id must emit NO destructive // statement — the applicationPrefix refusal in action. b := NewRowBuilder() - b.Node([]string{LabelCallable, SymbolLabel}, "id", "can://go/app/x.go/f", nil, "x.go") - script := RenderCypher(b.Finish(), "can://go/app/x.go/f") // not a bare app id + b.Node([]string{LabelCallable, SymbolLabel}, "id", "can://app/go/x.go/f", nil, "x.go") + script := RenderCypher(b.Finish(), "can://app/go/x.go/f") // not a bare app id if strings.Contains(script, "DETACH DELETE") { t.Errorf("a bag with no bare application root must emit no wipe:\n%s", script) } diff --git a/internal/neo4j/project.go b/internal/neo4j/project.go index 68bc936..8c74346 100644 --- a/internal/neo4j/project.go +++ b/internal/neo4j/project.go @@ -2,6 +2,7 @@ package neo4j import ( "encoding/json" + "strings" v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" ) @@ -22,7 +23,7 @@ import ( // resolves to a declared node or an id-keyed :GoExternal ghost — nothing dangles. // Project is the pure (tree) → GraphRows function. appID is analysis.Application.ID -// (the can:/// root); it is read from the envelope, never recomposed. +// (the can:// root); it is read from the envelope, never recomposed. func Project(a *v2.Analysis) GraphRows { p := &projector{ b: NewRowBuilder(), @@ -163,6 +164,7 @@ func (p *projector) projectCallable(parentRef NodeRef, module string, c v2.Calla for local, bn := range c.Body { p.projectBodyNode(cRef, module, c.ID, local, bn) } + p.projectFlow(c) // Nested closures → GO_DECLARES (callable → closure). for _, inner := range c.Callables { @@ -173,7 +175,17 @@ func (p *projector) projectCallable(parentRef NodeRef, module string, c v2.Calla } func (p *projector) projectBodyNode(callableRef NodeRef, module, callableID, local string, bn v2.BodyNode) { - id := callableID + "@" + local + // GlobalOrdinal, not callableID+"@"+local: synthetic keys ("@entry") already + // carry the "@", and the JSON body node id uses the same rule. + id := v2.GlobalOrdinal(callableID, local) + // call_node is the parent statement's LOCAL key (JSON `parent` verbatim), + // resolved against the owning callable — Python parity: codeanalyzer-python + // projects node.parent unchanged and python-sdk's reconstruct maps call_node + // straight back to parent (I6). + var callNode Value + if bn.Parent != "" { + callNode = bn.Parent + } bRef := p.b.Node( []string{LabelBodyNode, MarkerLabel}, "id", id, @@ -181,19 +193,46 @@ func (p *projector) projectBodyNode(callableRef NodeRef, module, callableID, loc "kind": bn.Kind, "is_goroutine": boolOrNil(bn.IsGoroutine), "is_deferred": boolOrNil(bn.IsDeferred), - "span_json": spanJSON(bn.Span), + "span_json": bodySpanJSON(bn.Span), + "call_node": callNode, + // "var" (JSON `of`) is L4: the v2 BodyNode carries no such field yet, + // and a nil prop is pruned, so L3 emits none. }, module, ) p.b.Edge(RelHasBodyNode, callableRef, bRef, nil, "") // A resolved callee (backfilled at L2) → GO_RESOLVES_TO. The callee is - // already a resolved can:// id; route through the plain edge only when it - // names a declared callable (an external callee stays null in JSON and - // emits no edge here). - if bn.Callee != nil && p.declared[*bn.Callee] { - p.b.Edge(RelResolvesTo, bRef, - NodeRef{Label: SymbolLabel, KeyProp: "id", Value: *bn.Callee}, nil, "") + // already a resolved can:// id: a declared callable, or a library callee's + // @external id, which routes to its ghost via callEndpoint. + if bn.Callee != nil { + _, external := p.analysis.Application.ExternalSymbols[*bn.Callee] + if p.declared[*bn.Callee] || external { + p.b.Edge(RelResolvesTo, bRef, p.callEndpoint(*bn.Callee), nil, "") + } + } +} + +// projectFlow emits a callable's L3 intra-callable overlay. JSON edge endpoints +// are LOCAL body keys; the graph addresses :GoBodyNode by its global id, so each +// endpoint goes through v2.GlobalOrdinal. Parallel edges survive via _k: +// GO_CFG_NEXT per kind, GO_DDG per (var, prov). +func (p *projector) projectFlow(c v2.Callable) { + ref := func(local string) NodeRef { + return NodeRef{Label: LabelBodyNode, KeyProp: "id", Value: v2.GlobalOrdinal(c.ID, local)} + } + for _, e := range c.Cfg { + p.b.Edge(RelCFGNext, ref(e.Src), ref(e.Dst), Props{"kind": e.Kind}, e.Kind) + } + for _, e := range c.Cdg { + p.b.Edge(RelCDG, ref(e.Src), ref(e.Dst), nil, "") + } + for _, e := range c.Ddg { + // Own the slice: the JSON tree stays untouched by later prop mutation. + prov := append([]string(nil), e.Prov...) + p.b.Edge(RelDDG, ref(e.Src), ref(e.Dst), + Props{"var": e.Var, "prov": stringsOrNil(prov)}, + e.Var+"|"+strings.Join(prov, ",")) } } @@ -217,7 +256,7 @@ func (p *projector) projectImports(modRef NodeRef, m v2.Module) { continue } // External/stdlib → :GoExternal ghost keyed by import path. - ghostID := appID + "/@external/" + imp.Path + ghostID := v2.ExternalID(appID, imp.Path, "") ghostRef := p.b.Node( []string{SymbolLabel, LabelExternal, MarkerLabel}, "id", ghostID, @@ -253,9 +292,15 @@ func (p *projector) callEndpoint(id string) NodeRef { if p.declared[id] { return ref } - // Emit the ghost so the endpoint is a real node, not a dangling MATCH. - p.b.Node([]string{SymbolLabel, LabelExternal, MarkerLabel}, "id", id, - Props{"name": lastSegment(id)}, "") + // Emit the ghost so the endpoint is a real node, not a dangling MATCH. A + // library callee recorded in external_symbols carries its signature/module. + props := Props{"name": lastSegment(id)} + if ext, ok := p.analysis.Application.ExternalSymbols[id]; ok { + props["name"] = ext.Name + props["signature"] = ext.Signature + props["module"] = ext.Module + } + p.b.Node([]string{SymbolLabel, LabelExternal, MarkerLabel}, "id", id, props, "") return ref } @@ -354,3 +399,12 @@ func lastSegment(id string) string { } return last } + +// bodySpanJSON is spanJSON for a body node, nil for the span-less synthetic +// @entry / @exit (L3), whose JSON node has no span either. +func bodySpanJSON(s v2.Span) Value { + if s == (v2.Span{}) { + return nil + } + return spanJSON(s) +} diff --git a/internal/neo4j/project_test.go b/internal/neo4j/project_test.go index 971e219..a2d545e 100644 --- a/internal/neo4j/project_test.go +++ b/internal/neo4j/project_test.go @@ -3,8 +3,10 @@ package neo4j import ( "path/filepath" "runtime" + "strings" "testing" + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" "github.com/codellm-devkit/codeanalyzer-go/internal/schema" v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" "github.com/codellm-devkit/codeanalyzer-go/internal/semantic_analysis" @@ -33,8 +35,8 @@ func buildMultipackageL2(t *testing.T) *v2.Analysis { } cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) edges := cg.Build(st) - app := &schema.GoApplication{SymbolTable: st, CallGraph: edges} - return v2emit.Emit(app, "multipackage", dir, 2, "neo4j-gate") + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()} + return v2emit.Emit(app, "multipackage", dir, 2, "neo4j-gate", 0, nil) } func TestProject_NoDanglingEndpoints(t *testing.T) { @@ -156,3 +158,343 @@ func hasLabel(n NodeRow, label string) bool { } return false } + +// TestProject_ExternalGhostCarriesSignature locks that a library callee becomes a +// :GoExternal ghost carrying signature/module, is the target of a GO_CALLS edge, +// and is reached by a GO_RESOLVES_TO edge from the call body node. +func TestProject_ExternalGhostCarriesSignature(t *testing.T) { + a := buildMultipackageL2(t) + rows := Project(a) + ext := v2.ExternalID(a.Application.ID, "fmt", "Println") + + var ghost *NodeRow + for i, n := range rows.Nodes { + if n.Value == ext && hasLabel(n, LabelExternal) { + ghost = &rows.Nodes[i] + } + } + if ghost == nil { + t.Fatalf("no :GoExternal ghost for %s", ext) + } + if ghost.Props["signature"] != "Println" || ghost.Props["module"] != "fmt" { + t.Errorf("ghost props = %v, want signature=Println module=fmt", ghost.Props) + } + var calls, resolves bool + for _, e := range rows.Edges { + if e.To.Value != ext { + continue + } + switch e.Type { + case RelCalls: + calls = true + case RelResolvesTo: + resolves = true + } + } + if !calls || !resolves { + t.Errorf("edges to external: GO_CALLS=%v GO_RESOLVES_TO=%v, want both", calls, resolves) + } +} + +// ─── L3 overlay ────────────────────────────────────────────────────────────── + +// buildDataflowL3 builds testdata/dataflow through the real -a 3 wiring (symbol +// table, resolver, dataflow driver) and emits the v2 L3 analysis. +func buildDataflowL3(t *testing.T) *v2.Analysis { + t.Helper() + _, thisFile, _, _ := runtime.Caller(0) + dir, err := filepath.Abs(filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata", "dataflow")) + if err != nil { + t.Fatalf("resolving fixture dir: %v", err) + } + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("symbol table build failed: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()} + res := dataflow.Run(b.Fset(), b.L3Units(st), 3, 4) + return v2emit.Emit(app, "dataflow", dir, 3, "neo4j-gate", 3, res) +} + +// walkCallables visits every callable (methods and nested closures included). +func walkCallables(a *v2.Analysis, visit func(c v2.Callable)) { + var walk func(c v2.Callable) + walk = func(c v2.Callable) { + visit(c) + for _, in := range c.Callables { + walk(in) + } + } + for _, m := range a.Application.SymbolTable { + for _, ty := range m.Types { + for _, c := range ty.Callables { + walk(c) + } + } + for _, f := range m.Functions { + walk(f) + } + } +} + +func countEdges(rows GraphRows, typ string) int { + n := 0 + for _, e := range rows.Edges { + if e.Type == typ { + n++ + } + } + return n +} + +// TestProject_L3CountsMatchJSON is the cross-projection gate: the graph carries +// exactly the facts the JSON carries. +func TestProject_L3CountsMatchJSON(t *testing.T) { + a := buildDataflowL3(t) + rows := Project(a) + + var cfg, cdg, ddg, body int + walkCallables(a, func(c v2.Callable) { + cfg += len(c.Cfg) + cdg += len(c.Cdg) + ddg += len(c.Ddg) + body += len(c.Body) + }) + if cfg == 0 || cdg == 0 || ddg == 0 { + t.Fatalf("fixture produced no overlay edges: cfg=%d cdg=%d ddg=%d", cfg, cdg, ddg) + } + t.Logf("JSON: cfg=%d cdg=%d ddg=%d body=%d", cfg, cdg, ddg, body) + + for _, c := range []struct { + name string + got int + want int + }{ + {RelCFGNext, countEdges(rows, RelCFGNext), cfg}, + {RelCDG, countEdges(rows, RelCDG), cdg}, + {RelDDG, countEdges(rows, RelDDG), ddg}, + {RelHasBodyNode, countEdges(rows, RelHasBodyNode), body}, + } { + if c.got != c.want { + t.Errorf("%s rows = %d, JSON = %d", c.name, c.got, c.want) + } + } + nodes := 0 + for _, n := range rows.Nodes { + if n.Labels[0] == LabelBodyNode { + nodes++ + } + } + if nodes != body { + t.Errorf(":GoBodyNode rows = %d, Σ len(body) = %d", nodes, body) + } +} + +// TestProject_L3OverlayEndpointsAreBodyNodes: every overlay endpoint addresses a +// projected :GoBodyNode by its global id. +func TestProject_L3OverlayEndpointsAreBodyNodes(t *testing.T) { + rows := Project(buildDataflowL3(t)) + have := map[string]bool{} + for _, n := range rows.Nodes { + if n.Labels[0] == LabelBodyNode { + have[n.Value] = true + } + } + for _, e := range rows.Edges { + switch e.Type { + case RelCFGNext, RelCDG, RelDDG: + for _, r := range []NodeRef{e.From, e.To} { + if r.Label != LabelBodyNode || r.KeyProp != "id" || !have[r.Value] { + t.Fatalf("%s endpoint %+v is not a projected :GoBodyNode", e.Type, r) + } + } + } + } +} + +// TestProject_SyntheticBodyNodes locks the Task 11 gap: @entry/@exit are +// projected with the single-@ id, no span, and a GO_HAS_BODY_NODE edge. +func TestProject_SyntheticBodyNodes(t *testing.T) { + a := buildDataflowL3(t) + rows := Project(a) + nodes := map[string]NodeRow{} + for _, n := range rows.Nodes { + if n.Labels[0] == LabelBodyNode { + nodes[n.Value] = n + } + } + hasEdge := map[string]string{} // body id -> owning callable id + for _, e := range rows.Edges { + if e.Type == RelHasBodyNode { + hasEdge[e.To.Value] = e.From.Value + } + } + checked := 0 + walkCallables(a, func(c v2.Callable) { + for _, tag := range []string{"@entry", "@exit"} { + if _, ok := c.Body[tag]; !ok { + continue + } + id := c.ID + tag + n, ok := nodes[id] + if !ok { + t.Errorf("synthetic node %s not projected", id) + continue + } + if strings.Contains(id, "@@") { + t.Errorf("doubled @ in %s", id) + } + if _, has := n.Props["span_json"]; has { + t.Errorf("%s carries span_json", id) + } + if hasEdge[id] != c.ID { + t.Errorf("%s: GO_HAS_BODY_NODE from %q, want %q", id, hasEdge[id], c.ID) + } + checked++ + } + }) + if checked == 0 { + t.Fatal("no synthetic nodes in the fixture") + } +} + +func overlayFixture() *v2.Analysis { + const app = "can://go/x" + const mod = app + "/a.go" + const fn = mod + "/f()" + return &v2.Analysis{ + SchemaVersion: v2.SchemaVersion, Language: v2.Language, MaxLevel: 3, + Application: v2.Application{ + ID: app, + SymbolTable: map[string]v2.Module{"a.go": { + ID: mod, + Functions: map[string]v2.Callable{"f()": { + ID: fn, + Body: map[string]v2.BodyNode{ + "@entry": {Kind: "entry", ID: fn + "@entry"}, + "3:2": {Kind: "branch", ID: fn + "@3:2", Span: v2.Span{Start: [2]int{3, 2}, End: [2]int{3, 9}, Bytes: [2]int{10, 17}}}, + "4:3": {Kind: "call", ID: fn + "@4:3", Parent: "3:2", Span: v2.Span{Start: [2]int{4, 3}, End: [2]int{4, 9}, Bytes: [2]int{20, 26}}}, + "@exit": {Kind: "exit", ID: fn + "@exit"}, + }, + Cfg: []v2.CfgEdge{ + {Src: "3:2", Dst: "4:3", Kind: "true"}, + {Src: "3:2", Dst: "4:3", Kind: "exception"}, + }, + Cdg: []v2.CdgEdge{{Src: "3:2", Dst: "4:3"}}, + Ddg: []v2.DdgEdge{ + {Src: "@entry", Dst: "4:3", Var: "x", Prov: []string{"ssa"}}, + {Src: "@entry", Dst: "4:3", Var: "x", Prov: []string{"ssa", "points-to"}}, + {Src: "@entry", Dst: "4:3", Var: "y", Prov: []string{"ssa"}}, + }, + }}, + }}, + }, + } +} + +func TestProject_ParallelCfgEdgesSurviveMerge(t *testing.T) { + rows := Project(overlayFixture()) + keys := map[string]bool{} + for _, e := range rows.Edges { + if e.Type == RelCFGNext { + if e.Key != e.Props["kind"] { + t.Errorf("_k = %q, want kind %v", e.Key, e.Props["kind"]) + } + keys[e.Key] = true + } + } + if len(keys) != 2 || !keys["true"] || !keys["exception"] { + t.Errorf("want parallel cfg rows true+exception, got %v", keys) + } +} + +func TestProject_DdgKeyIsVarAndProv(t *testing.T) { + rows := Project(overlayFixture()) + got := map[string]bool{} + for _, e := range rows.Edges { + if e.Type != RelDDG { + continue + } + prov, ok := e.Props["prov"].([]string) + if !ok { + t.Fatalf("prov = %T, want []string", e.Props["prov"]) + } + if want := e.Props["var"].(string) + "|" + strings.Join(prov, ","); e.Key != want { + t.Errorf("_k = %q, want %q", e.Key, want) + } + got[e.Key] = true + } + for _, k := range []string{"x|ssa", "x|ssa,points-to", "y|ssa"} { + if !got[k] { + t.Errorf("missing ddg key %q (got %v)", k, got) + } + } + if len(got) != 3 { + t.Errorf("want 3 distinct ddg keys, got %v", got) + } +} + +func TestProject_BodyNodeCallNodeAndVar(t *testing.T) { + rows := Project(overlayFixture()) + const fn = "can://go/x/a.go/f()" + for _, n := range rows.Nodes { + if n.Labels[0] != LabelBodyNode { + continue + } + _, hasVar := n.Props["var"] + if hasVar { + t.Errorf("%s carries var at L3", n.Value) + } + // call_node is the LOCAL key of the parent statement (JSON `parent` + // verbatim), as codeanalyzer-python projects it and python-sdk's + // reconstruct maps it back (I6). + cn, hasCall := n.Props["call_node"] + if n.Value == fn+"@4:3" { + if cn != "3:2" { + t.Errorf("call_node = %v, want the local key 3:2", cn) + } + } else if hasCall { + t.Errorf("%s has unexpected call_node", n.Value) + } + } +} + +// TestProject_CallNodeResolvesWithinOwner (I6): on the real -a 3 fixture every +// call_node is a local key that, resolved against the owning callable (the +// GO_HAS_BODY_NODE source) with v2.GlobalOrdinal, names a projected body node +// of that same callable — no dangling call_node. +func TestProject_CallNodeResolvesWithinOwner(t *testing.T) { + rows := Project(buildDataflowL3(t)) + owner := map[string]string{} // body id -> owning callable id + for _, e := range rows.Edges { + if e.Type == RelHasBodyNode { + owner[e.To.Value] = e.From.Value + } + } + checked := 0 + for _, n := range rows.Nodes { + if n.Labels[0] != LabelBodyNode { + continue + } + cn, ok := n.Props["call_node"] + if !ok { + continue + } + local, isStr := cn.(string) + if !isStr || strings.HasPrefix(local, "can://") { + t.Errorf("%s: call_node %v is not a local key", n.Value, cn) + continue + } + parent := v2.GlobalOrdinal(owner[n.Value], local) + if owner[parent] != owner[n.Value] || owner[n.Value] == "" { + t.Errorf("%s: call_node %q resolves to %s, not a body node of %s", n.Value, local, parent, owner[n.Value]) + } + checked++ + } + if checked < 10 { + t.Errorf("only %d call_node props checked", checked) + } +} diff --git a/internal/neo4j/rows.go b/internal/neo4j/rows.go index b917550..5bdf384 100644 --- a/internal/neo4j/rows.go +++ b/internal/neo4j/rows.go @@ -175,7 +175,7 @@ func edgeSortKey(e EdgeRow) string { // ─── scoping helpers ────────────────────────────────────────────────────────── // applicationPrefix returns the bare application id used to scope destructive -// statements. It REFUSES anything that is not a bare can:/// id (an +// statements. It REFUSES anything that is not a bare can:// id (an // empty or deeper id would silently widen or narrow the blast radius): it // returns ("", false) so a hand-built GraphRows with no proper application root // gets NO destructive statement. The app segment is the outermost, so one @@ -186,9 +186,8 @@ func applicationPrefix(appID string) (string, bool) { return "", false } rest := strings.TrimPrefix(appID, scheme) - // A bare app id is exactly / — two non-empty segments, no deeper. - parts := strings.Split(rest, "/") - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + // A bare app id is exactly — one non-empty segment, no deeper. + if rest == "" || strings.Contains(rest, "/") { return "", false } return appID, true @@ -196,8 +195,8 @@ func applicationPrefix(appID string) (string, bool) { // descendantPrefix appends "/" to a can:// id so a STARTS WITH predicate matches // strict descendants only — "foo.go" is not treated as a prefix of "foo.go2", -// and the app root id itself (can://go/app) is outside its own descendant -// prefix (can://go/app/), so a wipe must match the root separately. +// and the app root id itself (can://app) is outside its own descendant +// prefix (can://app/), so a wipe must match the root separately. func descendantPrefix(canID string) string { return canID + "/" } diff --git a/internal/neo4j/rows_test.go b/internal/neo4j/rows_test.go index aad83e0..707ea15 100644 --- a/internal/neo4j/rows_test.go +++ b/internal/neo4j/rows_test.go @@ -11,7 +11,7 @@ import ( func TestRowBuilder_NodeMergeDedupsAndUnionsLabels(t *testing.T) { b := NewRowBuilder() - id := "can://go/app/x.go/T" + id := "can://app/go/x.go/T" b.Node([]string{SymbolLabel, LabelType}, "id", id, Props{"kind": "struct"}, "x.go") // Re-see the same identity with an extra label + an extra prop. b.Node([]string{SymbolLabel, MarkerLabel}, "id", id, Props{"signature": "T"}, "x.go") @@ -34,7 +34,7 @@ func TestRowBuilder_NodeMergeDedupsAndUnionsLabels(t *testing.T) { func TestRowBuilder_PropLastWriteWins(t *testing.T) { b := NewRowBuilder() - id := "can://go/app/x.go/T" + id := "can://app/go/x.go/T" b.Node([]string{LabelType}, "id", id, Props{"kind": "struct"}, "") b.Node([]string{LabelType}, "id", id, Props{"kind": "interface"}, "") rows := b.Finish() @@ -45,9 +45,9 @@ func TestRowBuilder_PropLastWriteWins(t *testing.T) { func TestRowBuilder_DeferredEdgeDroppedWhenTargetAbsent(t *testing.T) { b := NewRowBuilder() - src := b.Node([]string{SymbolLabel, LabelType}, "id", "can://go/app/x.go/T", nil, "x.go") + src := b.Node([]string{SymbolLabel, LabelType}, "id", "can://app/go/x.go/T", nil, "x.go") // Target never emitted as a node. - b.EdgeToSymbol(RelEmbeds, src, SymbolLabel, "id", "can://go/app/y.go/Missing", nil, "") + b.EdgeToSymbol(RelEmbeds, src, SymbolLabel, "id", "can://app/go/y.go/Missing", nil, "") rows := b.Finish() if len(rows.Edges) != 0 { t.Errorf("deferred edge to an unemitted target must be dropped; got %d edges", len(rows.Edges)) @@ -56,8 +56,8 @@ func TestRowBuilder_DeferredEdgeDroppedWhenTargetAbsent(t *testing.T) { func TestRowBuilder_DeferredEdgeKeptWhenTargetEmitted(t *testing.T) { b := NewRowBuilder() - src := b.Node([]string{SymbolLabel, LabelType}, "id", "can://go/app/x.go/T", nil, "x.go") - target := "can://go/app/y.go/Base" + src := b.Node([]string{SymbolLabel, LabelType}, "id", "can://app/go/x.go/T", nil, "x.go") + target := "can://app/go/y.go/Base" b.Node([]string{SymbolLabel, LabelType}, "id", target, nil, "y.go") b.EdgeToSymbol(RelEmbeds, src, SymbolLabel, "id", target, nil, "") rows := b.Finish() @@ -100,14 +100,15 @@ func TestPrune_AllEmptyReturnsNil(t *testing.T) { func TestApplicationPrefix_RefusesNonBareID(t *testing.T) { cases := map[string]bool{ - "can://go/app": true, // bare — accepted - "can://go/my-app": true, // + "can://app": true, // bare — accepted + "can://my-app": true, // "": false, // empty - "can://go": false, // missing app segment - "can://go/app/x.go": false, // too deep (a module id) - "can://go/app/x.go/T": false, // deeper still + "can://": false, // missing app segment + "can://app/go/x.go": false, // too deep (a module id) + "can://app/go/x.go/T": false, // deeper still "go/app": false, // no scheme - "can://go//": false, // empty segment + "can://app/": false, // trailing empty segment + "can://app//": false, // empty segments } for id, wantOK := range cases { _, ok := applicationPrefix(id) @@ -118,8 +119,8 @@ func TestApplicationPrefix_RefusesNonBareID(t *testing.T) { } func TestDescendantPrefix_AppendsSlash(t *testing.T) { - if got := descendantPrefix("can://go/app"); got != "can://go/app/" { - t.Errorf("descendantPrefix = %q, want can://go/app/", got) + if got := descendantPrefix("can://app"); got != "can://app/" { + t.Errorf("descendantPrefix = %q, want can://app/", got) } } @@ -130,7 +131,7 @@ func TestFinish_IsOrderStable(t *testing.T) { ref := func(b *RowBuilder, id string) NodeRef { return b.Node([]string{SymbolLabel, LabelCallable}, "id", id, nil, "") } - const A, B, C = "can://go/app/a.go/A", "can://go/app/b.go/B", "can://go/app/c.go/C" + const A, B, C = "can://app/go/a.go/A", "can://app/go/b.go/B", "can://app/go/c.go/C" build := func(reverse bool) GraphRows { b := NewRowBuilder() @@ -190,3 +191,15 @@ func edgeKeys(g GraphRows) []string { } return out } + +func TestApplicationPrefix_AcceptsBareAppOnly(t *testing.T) { + if _, ok := applicationPrefix("can://shop"); !ok { + t.Error("bare can://shop should be accepted") + } + if _, ok := applicationPrefix("can://shop/go/a.go"); ok { + t.Error("module id should be rejected") + } + if _, ok := applicationPrefix("can://"); ok { + t.Error("can:// should be rejected") + } +} diff --git a/internal/neo4j/schema.go b/internal/neo4j/schema.go index 271837d..6fae733 100644 --- a/internal/neo4j/schema.go +++ b/internal/neo4j/schema.go @@ -119,13 +119,14 @@ const ( RelProvides = "GO_PROVIDES" RelUnresolvedImport = "GO_UNRESOLVED_IMPORT" - // Reserved L3/L4 overlay — declared NOW, emits ZERO rows until the levels - // land (CLAUDE.md L1). When L3/L4 arrive the projector fills these with no - // schema-contract change. --emit neo4j is "full-depth = max implemented" - // (L2 today); the graph grows additively. - RelCFGNext = "GO_CFG_NEXT" // _k=kind - RelCDG = "GO_CDG" - RelDDG = "GO_DDG" // _k=var,prov + // L3 overlay (LIVE at -a 3): intra-callable edges between :GoBodyNode ids. + // GO_CFG_NEXT _k=kind; GO_DDG _k=var|prov. + RelCFGNext = "GO_CFG_NEXT" + RelCDG = "GO_CDG" + RelDDG = "GO_DDG" + + // Reserved L4 overlay — declared NOW, emits ZERO rows until L4 lands + // (CLAUDE.md). The projector fills these with no schema-contract change. RelParamIn = "GO_PARAM_IN" RelParamOut = "GO_PARAM_OUT" RelSummary = "GO_SUMMARY" @@ -166,14 +167,16 @@ type RelSpec struct { // discriminant, which is Discriminant). Props []string // Discriminant names the _k key when parallel edges per (endpoint pair) are - // legitimate (e.g. "kind" for GO_CFG_NEXT). Empty = plain endpoint-pair MERGE. + // legitimate (e.g. "kind" for GO_CFG_NEXT). A composite is written + // "a|b" and means _k = a + "|" + strings.Join(b, ","), b being an array + // prop (GO_DDG: "var|prov"). Empty = plain endpoint-pair MERGE. Discriminant string // Deferred marks an edge whose target may be library/external code not // present in the graph: it is emitted via the RowBuilder's edge-only-when- // resolved path and kept at finish() only if the target was emitted. Deferred bool - // Reserved marks the L3/L4 overlay edges: declared now, emits zero rows at - // the current max level. + // Reserved marks the L4 overlay edges: declared now, emits zero rows at + // the current max level (L3). Reserved bool } @@ -204,12 +207,16 @@ var Nodes = []NodeSpec{ }, { Label: LabelBodyNode, MergeLabel: LabelBodyNode, Key: "id", CanNode: true, - // kind "call" at L1; is_goroutine/is_deferred pruned when false. - Props: []string{"kind", "is_goroutine", "is_deferred", "span_json"}, + // kind "call" at L1, statement|branch|loop|switch|return|entry|exit at L3; + // is_goroutine/is_deferred pruned when false. call_node = the enclosing + // statement's LOCAL key (JSON `parent` verbatim, resolved against the + // owning callable; Python parity); var = JSON `of` (L4 — absent at L3). + // span_json absent on the synthetic entry/exit nodes. + Props: []string{"kind", "is_goroutine", "is_deferred", "span_json", "call_node", "var"}, }, { Label: LabelExternal, MergeLabel: SymbolLabel, Key: "id", CanNode: true, - Props: []string{"name", "path"}, + Props: []string{"name", "path", "signature", "module"}, }, // Neutral layer. Keyed by own id (not a can:// id), so NO marker label and @@ -235,7 +242,7 @@ var Rels = []RelSpec{ {Type: RelDeclares, From: []string{LabelModule, LabelCallable}, To: []string{LabelType, LabelCallable}}, {Type: RelHasMethod, From: []string{LabelType}, To: []string{LabelCallable}}, {Type: RelHasField, From: []string{LabelType}, To: []string{LabelField}}, - {Type: RelHasBodyNode, From: []string{LabelCallable}, To: []string{LabelBodyNode}}, + {Type: RelHasBodyNode, From: []string{LabelCallable, LabelExternal}, To: []string{LabelBodyNode}}, // Call graph. {Type: RelCalls, From: []string{LabelCallable, LabelExternal}, To: []string{LabelCallable, LabelExternal}, Props: []string{"weight", "prov"}}, @@ -256,10 +263,12 @@ var Rels = []RelSpec{ {Type: RelProvides, From: []string{LabelPackage}, To: []string{LabelExternal}}, {Type: RelUnresolvedImport, From: []string{LabelApplication}, To: []string{LabelExternal}}, - // Reserved L3/L4 overlay (zero rows until the levels land). - {Type: RelCFGNext, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}, Discriminant: "kind", Reserved: true}, - {Type: RelCDG, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}, Reserved: true}, - {Type: RelDDG, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}, Props: []string{"var", "prov"}, Discriminant: "var", Reserved: true}, + // L3 overlay (live at -a 3). + {Type: RelCFGNext, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}, Props: []string{"kind"}, Discriminant: "kind"}, + {Type: RelCDG, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}}, + {Type: RelDDG, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}, Props: []string{"var", "prov"}, Discriminant: "var|prov"}, + + // Reserved L4 overlay (zero rows until L4 lands). {Type: RelParamIn, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}, Reserved: true}, {Type: RelParamOut, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}, Reserved: true}, {Type: RelSummary, From: []string{LabelBodyNode}, To: []string{LabelBodyNode}, Reserved: true}, diff --git a/internal/neo4j/schema_test.go b/internal/neo4j/schema_test.go index 8634009..212cd61 100644 --- a/internal/neo4j/schema_test.go +++ b/internal/neo4j/schema_test.go @@ -2,6 +2,7 @@ package neo4j import ( "encoding/json" + "strings" "testing" ) @@ -135,7 +136,12 @@ func TestSchemaDocument_RoundTrips(t *testing.T) { // never produces a label, relationship type, or property that schema.go does // not declare — so the contract cannot silently drift from the projector. func TestConformance_ProjectorEmitsOnlyDeclared(t *testing.T) { - rows := Project(buildMultipackageL2(t)) + t.Run("L2", func(t *testing.T) { checkConformance(t, Project(buildMultipackageL2(t))) }) + t.Run("L3", func(t *testing.T) { checkConformance(t, Project(buildDataflowL3(t))) }) +} + +func checkConformance(t *testing.T, rows GraphRows) { + t.Helper() // Declared label set + per-label allowed property set. A node's labels[0] is // the MERGE label, which for the symbol family is the SHARED SymbolLabel; the @@ -208,3 +214,41 @@ func TestConformance_ProjectorEmitsOnlyDeclared(t *testing.T) { } } } + +func TestContract_L3OverlayNotReserved(t *testing.T) { + want := map[string]bool{ + RelCFGNext: false, RelCDG: false, RelDDG: false, + RelParamIn: true, RelParamOut: true, RelSummary: true, + } + for _, r := range Rels { + if res, ok := want[r.Type]; ok { + if r.Reserved != res { + t.Errorf("%s Reserved = %v, want %v", r.Type, r.Reserved, res) + } + delete(want, r.Type) + } + if r.Type == RelDDG && r.Discriminant != "var|prov" { + t.Errorf("GO_DDG discriminant = %q, want var|prov", r.Discriminant) + } + if r.Type == RelHasBodyNode { + got := strings.Join(r.From, ",") + if got != LabelCallable+","+LabelExternal { + t.Errorf("GO_HAS_BODY_NODE From = %v", r.From) + } + } + } + if len(want) != 0 { + t.Errorf("rels not declared: %v", want) + } + for _, n := range Nodes { + if n.Label == LabelBodyNode { + have := map[string]bool{} + for _, p := range n.Props { + have[p] = true + } + if !have["var"] || !have["call_node"] { + t.Errorf(":GoBodyNode props %v lack var/call_node", n.Props) + } + } + } +} diff --git a/internal/options/options.go b/internal/options/options.go index c4d50f4..1006679 100644 --- a/internal/options/options.go +++ b/internal/options/options.go @@ -9,6 +9,8 @@ const ( LevelSymbolTable AnalysisLevel = 1 // LevelCallGraph produces symbol table + resolver-based call graph (still cheap). LevelCallGraph AnalysisLevel = 2 + // LevelDataflow adds intraprocedural CFG/CDG/DDG on top of the call graph. + LevelDataflow AnalysisLevel = 3 ) // EmitTarget selects the analyzer's output projection (cli-contract.md § Neo4j). @@ -33,10 +35,11 @@ type AnalysisOptions struct { Format string // Emit selects the output projection: json (default), neo4j, or schema. Emit EmitTarget - // AppName anchors application identity (can://go/ and the Neo4j + // AppName anchors application identity (can:// and the Neo4j // :Application node). Defaults to the input directory's base name. AppName string - // AnalysisLevel controls symbol-table-only (1) vs + call graph (2). + // AnalysisLevel controls symbol-table-only (1), + call graph (2), or + // + intraprocedural dataflow (3). Level AnalysisLevel // AnalyzerVersion is the analyzer's own version, recorded in the v2 // manifest's analyzer{} tag. Set by the CLI from its build-stamped value. @@ -54,6 +57,9 @@ type AnalysisOptions struct { Jobs int // Verbose enables verbose logging. Verbose bool + // FieldDepth is the k limit on field-path depth for L3 dataflow nodes + // (--graph-field-depth). Default 3. Recorded in the payload as k_limit. + FieldDepth int // Neo4j projection targets (consumed by the Neo4j emitter; validated now). // Precedence for each: explicit flag > env var > default. diff --git a/internal/schema/schema.go b/internal/schema/schema.go index a61f255..0161769 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -17,12 +17,12 @@ type GoImport struct { // GoComment represents a comment (line, block, or doc comment). type GoComment struct { - Content string `json:"content"` - StartLine int `json:"start_line"` - EndLine int `json:"end_line"` - StartColumn int `json:"start_column"` - EndColumn int `json:"end_column"` - IsDocComment bool `json:"is_doc_comment"` + Content string `json:"content"` + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` + StartColumn int `json:"start_column"` + EndColumn int `json:"end_column"` + IsDocComment bool `json:"is_doc_comment"` } // GoParameter represents a single parameter of a callable. @@ -82,12 +82,16 @@ type GoCallsite struct { ArgumentTypes []string `json:"argument_types"` ReturnType string `json:"return_type,omitempty"` CalleeSignature *string `json:"callee_signature"` // null until resolved + // CalleeImportPath is the callee's package import path, set only when the + // resolved callee is an external (library) function; see GoApplication.Externals. + CalleeImportPath string `json:"callee_import_path,omitempty"` IsConstructorCall bool `json:"is_constructor_call"` - IsGoroutine bool `json:"is_goroutine"` // true when preceded by `go` keyword - StartLine int `json:"start_line"` - StartColumn int `json:"start_column"` - EndLine int `json:"end_line"` - EndColumn int `json:"end_column"` + IsGoroutine bool `json:"is_goroutine"` // true when preceded by `go` keyword + IsDeferred bool `json:"is_deferred,omitempty"` // true when preceded by `defer` keyword + StartLine int `json:"start_line"` + StartColumn int `json:"start_column"` + EndLine int `json:"end_line"` + EndColumn int `json:"end_column"` } // ─── Callable ───────────────────────────────────────────────────────────────── @@ -95,27 +99,27 @@ type GoCallsite struct { // GoCallable represents a function, method, or function literal in Go. // receiver_type / receiver_name are non-empty for methods; empty for functions. type GoCallable struct { - Name string `json:"name"` - Path string `json:"path"` - Signature string `json:"signature"` // signatureOf() output — edge id - Comments []GoComment `json:"comments"` - Parameters []GoParameter `json:"parameters"` - ReturnType string `json:"return_type"` // joined, e.g. "(int, error)" - ReturnTypes []string `json:"return_types"` // Go extension: individual return types - Code string `json:"code,omitempty"` - StartLine int `json:"start_line"` - EndLine int `json:"end_line"` - CodeStartLine int `json:"code_start_line"` - AccessedSymbols []GoSymbol `json:"accessed_symbols"` - CallSites []GoCallsite `json:"call_sites"` - InnerCallables map[string]GoCallable `json:"inner_callables"` - LocalVariables []GoVariableDeclaration `json:"local_variables"` - CyclomaticComplexity int `json:"cyclomatic_complexity"` - IsEntrypoint bool `json:"is_entrypoint"` - EntrypointFramework string `json:"entrypoint_framework,omitempty"` - ReceiverType string `json:"receiver_type,omitempty"` // e.g. "*MyStruct" - ReceiverName string `json:"receiver_name,omitempty"` // e.g. "r" - IsExported bool `json:"is_exported"` + Name string `json:"name"` + Path string `json:"path"` + Signature string `json:"signature"` // signatureOf() output — edge id + Comments []GoComment `json:"comments"` + Parameters []GoParameter `json:"parameters"` + ReturnType string `json:"return_type"` // joined, e.g. "(int, error)" + ReturnTypes []string `json:"return_types"` // Go extension: individual return types + Code string `json:"code,omitempty"` + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` + CodeStartLine int `json:"code_start_line"` + AccessedSymbols []GoSymbol `json:"accessed_symbols"` + CallSites []GoCallsite `json:"call_sites"` + InnerCallables map[string]GoCallable `json:"inner_callables"` + LocalVariables []GoVariableDeclaration `json:"local_variables"` + CyclomaticComplexity int `json:"cyclomatic_complexity"` + IsEntrypoint bool `json:"is_entrypoint"` + EntrypointFramework string `json:"entrypoint_framework,omitempty"` + ReceiverType string `json:"receiver_type,omitempty"` // e.g. "*MyStruct" + ReceiverName string `json:"receiver_name,omitempty"` // e.g. "r" + IsExported bool `json:"is_exported"` } // ─── Type (struct or interface) ─────────────────────────────────────────────── @@ -127,18 +131,19 @@ type GoCallable struct { // base_types carries: embedded struct types (for structs) and the method-set // signatures of satisfied interfaces (for both) — the Go analog of base_classes. type GoType struct { - Name string `json:"name"` - Signature string `json:"signature"` // signatureOf() output - Comments []GoComment `json:"comments"` - Code string `json:"code,omitempty"` - IsInterface bool `json:"is_interface"` - IsExported bool `json:"is_exported"` - Fields []GoField `json:"fields"` // empty for interfaces - Methods map[string]GoCallable `json:"methods"` // sig → callable - BaseTypes []string `json:"base_types"` // embedded types + satisfied interface sigs - InnerTypes map[string]GoType `json:"inner_types"` // Go doesn't nest, but preserves spine - StartLine int `json:"start_line"` - EndLine int `json:"end_line"` + Name string `json:"name"` + Signature string `json:"signature"` // signatureOf() output + Comments []GoComment `json:"comments"` + Code string `json:"code,omitempty"` + IsInterface bool `json:"is_interface"` + TypeKind string `json:"type_kind,omitempty"` // struct | interface | alias | defined + IsExported bool `json:"is_exported"` + Fields []GoField `json:"fields"` // empty for interfaces + Methods map[string]GoCallable `json:"methods"` // sig → callable + BaseTypes []string `json:"base_types"` // embedded types + satisfied interface sigs + InnerTypes map[string]GoType `json:"inner_types"` // Go doesn't nest, but preserves spine + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` } // ─── File (Module analog) ───────────────────────────────────────────────────── @@ -146,13 +151,13 @@ type GoType struct { // GoFile is the Module analog for Go: one compilation unit (source file). // symbol_table is keyed by file path relative to the project root. type GoFile struct { - FilePath string `json:"file_path"` - PackageName string `json:"module_name"` // JSON key = module_name for spine compat - Imports []GoImport `json:"imports"` - Comments []GoComment `json:"comments"` - Types map[string]GoType `json:"classes"` // JSON key = classes for spine compat - Functions map[string]GoCallable `json:"functions"` - Variables []GoVariableDeclaration `json:"variables"` + FilePath string `json:"file_path"` + PackageName string `json:"module_name"` // JSON key = module_name for spine compat + Imports []GoImport `json:"imports"` + Comments []GoComment `json:"comments"` + Types map[string]GoType `json:"classes"` // JSON key = classes for spine compat + Functions map[string]GoCallable `json:"functions"` + Variables []GoVariableDeclaration `json:"variables"` // Caching metadata ContentHash *string `json:"content_hash"` LastModified *float64 `json:"last_modified"` @@ -166,8 +171,8 @@ type GoFile struct { type GoCallEdge struct { Source string `json:"source"` Target string `json:"target"` - Type string `json:"type"` // always "CALL_DEP" - Weight int `json:"weight"` // accumulated when merging backends + Type string `json:"type"` // always "CALL_DEP" + Weight int `json:"weight"` // accumulated when merging backends Provenance []string `json:"provenance"` // e.g. ["go/types"] Tags map[string]string `json:"tags"` @@ -187,9 +192,22 @@ type GoCallEdge struct { // GoApplication is the root of analysis.json. The SDK facade deserializes this. type GoApplication struct { - SymbolTable map[string]GoFile `json:"symbol_table"` // file_path → GoFile - CallGraph []GoCallEdge `json:"call_graph"` // identity-only edges - Entrypoints map[string][]GoEntrypoint `json:"entrypoints"` // optional, default {} + SymbolTable map[string]GoFile `json:"symbol_table"` // file_path → GoFile + CallGraph []GoCallEdge `json:"call_graph"` // identity-only edges + Entrypoints map[string][]GoEntrypoint `json:"entrypoints"` // optional, default {} + // Externals holds the library (out-of-project) functions the resolver saw + // called, keyed by the full callee signature (".[Type.]Name"). + Externals map[string]GoExternalCallee `json:"externals,omitempty"` +} + +// GoExternalCallee describes a library function or method called from the +// project. Params exclude the receiver; unnamed parameters are named "_". +type GoExternalCallee struct { + ImportPath string `json:"import_path"` + Sig string `json:"signature"` // callee signature without the "." prefix + Name string `json:"name"` + Params []GoParameter `json:"parameters"` + ReturnType string `json:"return_type,omitempty"` } // GoEntrypoint marks a callable as a framework entry point. diff --git a/internal/schema/v2/ids.go b/internal/schema/v2/ids.go index d8d192d..70e6ac5 100644 --- a/internal/schema/v2/ids.go +++ b/internal/schema/v2/ids.go @@ -7,11 +7,15 @@ import "strings" // helpers are pure so they can be tested in isolation and reused verbatim by // the deferred L3/L4 train (which must not re-decide id or span shape). // -// Durable id grammar (a containment path with an app segment so multiple apps -// in one language don't collide): +// Durable id grammar (the app segment is outermost so a polyglot app shares one +// `can://` prefix across languages; the language follows it): // -// can:////// -// can://go/myapp/src/util.go/Hasher/Hash(string)uint64 +// can:////// +// can://myapp/go/src/util.go/Hasher/Hash(string)uint64 +// +// External (out-of-project) symbols are language-neutral: +// +// can:///@external// // // Ordinal id grammar (statements and synthetic vertices, addressed WITHIN their // callable): @@ -28,16 +32,38 @@ const ( ordinalSep = "@" ) -// AppID builds the application id: can:///. lang is fixed to the -// package Language constant so callers cannot drift it. +// AppID builds the application id: can://. The language is not part of it +// (it follows the app in module ids) so sibling analyzers share the prefix. func AppID(app string) string { - return scheme + Language + idSep + app + return scheme + app } // ModuleID builds a module id from the application id and the module's file // path relative to the input root (forward-slashed, no leading slash, no ".."). +// The language segment is fixed to the package Language constant so callers +// cannot drift it. func ModuleID(appID, relPath string) string { - return appID + idSep + normalizePath(relPath) + return appID + idSep + Language + idSep + normalizePath(relPath) +} + +// ExternalID builds the id of an out-of-project symbol: +// /@external//. It is language-neutral. An empty sig +// (an import-level ghost, no symbol) yields /@external/. +func ExternalID(appID, importPath, sig string) string { + if sig == "" { + return appID + idSep + "@external" + idSep + importPath + } + return appID + idSep + "@external" + idSep + importPath + idSep + sig +} + +// GlobalOrdinal joins a callable id and a local ordinal: callableID + local +// when local already starts with "@" (a tag), else callableID + "@" + local +// (a ":" position). +func GlobalOrdinal(callableID, local string) string { + if strings.HasPrefix(local, ordinalSep) { + return callableID + local + } + return callableID + ordinalSep + local } // TypeID builds a type id under its module. sig is the type's signatureOf() @@ -72,6 +98,26 @@ func LocalID(line, col int) string { return itoa(line) + ":" + itoa(col) } +// CallBodyKeys returns the body-map key of each call site, given every site's +// [line, col] in recording order (ast.Inspect pre-order, so a call precedes the +// calls nested in it). The first site at a position gets the bare LocalID +// "l:c"; later sites at the same position (a chained call such as s.A().B() +// starts where s.A() does) get "l:c/2", "l:c/3", … This is the single source of +// call-node keys, shared by the L1 emitter and the L3 body builder so both +// produce byte-identical keys. +func CallBodyKeys(positions [][2]int) []string { + keys := make([]string, len(positions)) + seen := make(map[[2]int]int, len(positions)) + for i, p := range positions { + seen[p]++ + keys[i] = LocalID(p[0], p[1]) + if n := seen[p]; n > 1 { + keys[i] += "/" + itoa(n) + } + } + return keys +} + // NewSpan builds a Span from 1-based line/col positions and the [from, to) UTF-8 // byte offsets into module.source. Byte offsets are what make source slicing // O(1); line:col is what addresses and displays. diff --git a/internal/schema/v2/ids_test.go b/internal/schema/v2/ids_test.go index f7ec330..4b23e72 100644 --- a/internal/schema/v2/ids_test.go +++ b/internal/schema/v2/ids_test.go @@ -3,7 +3,7 @@ package v2 import "testing" func TestAppID(t *testing.T) { - if got, want := AppID("myapp"), "can://go/myapp"; got != want { + if got, want := AppID("myapp"), "can://myapp"; got != want { t.Errorf("AppID = %q, want %q", got, want) } } @@ -16,9 +16,9 @@ func TestDurableIDChain(t *testing.T) { call := CallableID(typ, "Hash(string)uint64") cases := []struct{ got, want string }{ - {mod, "can://go/myapp/src/util.go"}, - {typ, "can://go/myapp/src/util.go/Hasher"}, - {call, "can://go/myapp/src/util.go/Hasher/Hash(string)uint64"}, + {mod, "can://myapp/go/src/util.go"}, + {typ, "can://myapp/go/src/util.go/Hasher"}, + {call, "can://myapp/go/src/util.go/Hasher/Hash(string)uint64"}, } for _, c := range cases { if c.got != c.want { @@ -30,10 +30,10 @@ func TestDurableIDChain(t *testing.T) { func TestModuleIDNormalizesPath(t *testing.T) { app := AppID("myapp") // Backslashes become forward slashes; a leading slash is trimmed. - if got, want := ModuleID(app, "\\pkg\\a.go"), "can://go/myapp/pkg/a.go"; got != want { + if got, want := ModuleID(app, "\\pkg\\a.go"), "can://myapp/go/pkg/a.go"; got != want { t.Errorf("ModuleID = %q, want %q", got, want) } - if got, want := ModuleID(app, "/pkg/a.go"), "can://go/myapp/pkg/a.go"; got != want { + if got, want := ModuleID(app, "/pkg/a.go"), "can://myapp/go/pkg/a.go"; got != want { t.Errorf("ModuleID = %q, want %q", got, want) } } @@ -42,13 +42,13 @@ func TestModuleLevelFunctionParent(t *testing.T) { // A module-level function hangs off the module id, not a type. mod := ModuleID(AppID("myapp"), "src/util.go") fn := CallableID(mod, "New64()") - if want := "can://go/myapp/src/util.go/New64()"; fn != want { + if want := "can://myapp/go/src/util.go/New64()"; fn != want { t.Errorf("CallableID = %q, want %q", fn, want) } } func TestOrdinalAndTagIDs(t *testing.T) { - call := "can://go/myapp/src/util.go/Hasher/Hash(string)uint64" + call := "can://myapp/go/src/util.go/Hasher/Hash(string)uint64" if got, want := OrdinalID(call, 15, 2), call+"@15:2"; got != want { t.Errorf("OrdinalID = %q, want %q", got, want) } @@ -91,3 +91,35 @@ func TestItoa(t *testing.T) { } } } + +func TestIDs_AppOutermost(t *testing.T) { + app := AppID("shop") + cases := []struct{ got, want string }{ + {app, "can://shop"}, + {ModuleID(app, "pkg/a.go"), "can://shop/go/pkg/a.go"}, + {ExternalID(app, "database/sql", "DB.Query"), "can://shop/@external/database/sql/DB.Query"}, + {GlobalOrdinal("X/F", "@entry"), "X/F@entry"}, + {GlobalOrdinal("X/F", "12:3"), "X/F@12:3"}, + } + for _, c := range cases { + if c.got != c.want { + t.Errorf("id = %q, want %q", c.got, c.want) + } + } +} + +func TestCallBodyKeys(t *testing.T) { + got := CallBodyKeys([][2]int{{5, 2}, {5, 2}, {6, 1}, {5, 2}, {6, 1}}) + want := []string{"5:2", "5:2/2", "6:1", "5:2/3", "6:1/2"} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v, want %v", got, want) + } + } + if len(CallBodyKeys(nil)) != 0 { + t.Fatal("nil input must yield no keys") + } +} diff --git a/internal/schema/v2/schema.go b/internal/schema/v2/schema.go index e3a83f7..2293c09 100644 --- a/internal/schema/v2/schema.go +++ b/internal/schema/v2/schema.go @@ -5,8 +5,9 @@ // // The model is ONE additive containment tree // (application → module → type → callable → body) with typed edge overlays -// laid over it (a Code Property Graph). This package declares the L1 tree plus -// the L2 refinement/edge slots; L3/L4 fields (cfg/cdg/ddg/param_*) are not +// laid over it (a Code Property Graph). This package declares the L1 tree, the +// L2 refinement/edge slots and the L3 body completion (statement nodes, body +// node id/parent, the callable's cfg/cdg/ddg). L4 fields (param_*) are not // declared here — they arrive in their own train and reuse these ids/spans // unchanged. // @@ -19,10 +20,12 @@ // - open-vocabulary fields (prov, tags) are plain strings. package v2 +import "encoding/json" + // SchemaVersion is the canonical schema major this package targets. const SchemaVersion = "2.0.0" -// Language is the analyzer's language tag in the manifest and `can:///…`. +// Language is the analyzer's language tag in the manifest and `can:////…`. const Language = "go" // ─── Positioning ─────────────────────────────────────────────────────────── @@ -43,11 +46,13 @@ type Span struct { // Analysis is the top-level payload written to analysis.json / stdout. Its // siblings of `application` carry the manifest; consumers read `max_level` -// rather than sniffing for keys. +// rather than sniffing for keys. k_limit (the L3 field-path depth bound) is +// present only at max_level >= 3. type Analysis struct { SchemaVersion string `json:"schema_version"` Language string `json:"language"` MaxLevel int `json:"max_level"` + KLimit int `json:"k_limit,omitempty"` Analyzer AnalyzerTag `json:"analyzer"` Application Application `json:"application"` } @@ -60,8 +65,8 @@ type AnalyzerTag struct { // Application is the root node of the containment tree. type Application struct { - // ID is `can:///` — the app segment disambiguates apps in one - // language. is the --app-name value (default: input dir base name). + // ID is `can://` — the app segment is outermost, so a polyglot + // app shares one prefix across languages. is the --app-name value (default: input dir base name). ID string `json:"id"` Kind string `json:"kind"` // always "application" // SymbolTable is the L1 named map keyed by relative file path. @@ -69,6 +74,33 @@ type Application struct { // CallGraph holds the L2 cross-function edges (callable → callable). Empty // at L1. CallGraph []Edge `json:"call_graph"` + // ExternalSymbols holds the out-of-project (library) callables the call graph + // reaches, keyed by their `can:///@external//` id. + // Absent when the analysis saw no library call. + ExternalSymbols map[string]ExternalSymbol `json:"external_symbols,omitempty"` +} + +// ExternalSymbol is a library callable that call-graph edges and call nodes +// target but whose body is not in the project. It is a leaf node: identity and +// signature only. +type ExternalSymbol struct { + ID string `json:"id"` + Kind string `json:"kind"` // always "external" + Name string `json:"name"` + // Module is the package import path the symbol lives in. + Module string `json:"module"` + // Signature is the id's last segment: [Type.]Name, no import-path prefix. + Signature string `json:"signature"` + Parameters []ExternalParameter `json:"parameters"` + ReturnType string `json:"return_type,omitempty"` +} + +// ExternalParameter is a library callee's formal parameter. It has no span: the +// declaration is not in the project, so there is no location to slice. +type ExternalParameter struct { + Name string `json:"name"` + Type string `json:"type"` + IsVariadic bool `json:"is_variadic,omitempty"` } // ─── module (per-file compilation unit) ─────────────────────────────────────── @@ -142,7 +174,8 @@ type Callable struct { Signature string `json:"signature"` // SourceFile names the callable's declaring file (a symbol_table key) when it // differs from the module this callable is nested under — the Go case where a - // method is declared in a different file than its receiver type. When set, + // method is declared in a different file than its receiver type, and every + // closure (nested closures included) inside such a method. When set, // Span.Bytes index symbol_table[SourceFile].source, NOT the nesting module's // source; text = symbol_table[SourceFile].source[Span.Bytes]. Absent (the // common case) means the declaring file is the nesting module. See CLAUDE.md @@ -159,6 +192,11 @@ type Callable struct { Metrics map[string]int `json:"metrics,omitempty"` // Body holds call nodes at L1 (keyed by local id), completing at L3. Body map[string]BodyNode `json:"body"` + // Cfg, Cdg and Ddg are the L3 intra-callable edge lists; endpoints are + // local body keys. Absent below L3 and for a callable with no body. + Cfg []CfgEdge `json:"cfg,omitempty"` + Cdg []CdgEdge `json:"cdg,omitempty"` + Ddg []DdgEdge `json:"ddg,omitempty"` // Callables are nested closures / function literals, each with its own // can:// id (replaces v1 InnerCallables). Callables map[string]Callable `json:"callables,omitempty"` @@ -175,11 +213,22 @@ type Parameter struct { // ─── body nodes ──────────────────────────────────────────────────────────────── // BodyNode is a node inside a callable's body, keyed by its local id -// (line:col for real nodes). At L1 the only kind emitted is "call"; L3 adds the -// remaining statement kinds. +// (line:col for real nodes, "@entry"/"@exit" for the synthetic ones). At L1 +// the only kind emitted is "call"; L3 adds statement | branch | loop | switch +// | return | entry | exit. type BodyNode struct { Kind string `json:"kind"` // "call" at L1 - Span Span `json:"span"` + // ID is the node's global ordinal id, v2.GlobalOrdinal(callable id, key). + // Emitted at every level (from L1, R15 — Python parity), on every body + // node. + ID string `json:"id,omitempty"` + // Span is the node's source range. Absent (zero) only on the synthetic + // entry/exit nodes, which have no source. + Span Span `json:"span,omitzero"` + // Parent is, at L3+, the local key of the statement a satellite call + // node is nested in (the sanctioned parent refinement). Absent on CFG + // nodes. + Parent string `json:"parent,omitempty"` // Callee is the sanctioned null→id refinement slot on a call node: null at // L1, backfilled to a callable id at L2. A pointer with NO omitempty so it // always serializes — as JSON null at L1 (the one place null is allowed), @@ -191,8 +240,46 @@ type BodyNode struct { IsDeferred bool `json:"is_deferred,omitempty"` } +// MarshalJSON writes the callee key on call nodes only: it is the call +// node's refinement slot (always present there, null until resolved), and a +// statement / control / synthetic node has no callee fact to carry. +func (n BodyNode) MarshalJSON() ([]byte, error) { + type plain BodyNode + if n.Kind == "call" { + return json.Marshal(plain(n)) + } + return json.Marshal(struct { + plain + Callee *string `json:"callee,omitempty"` + }{plain: plain(n)}) +} + // ─── edges ───────────────────────────────────────────────────────────────────── +// CfgEdge is an L3 control-flow edge between two local body keys. Kind is a +// shared CFG edge kind or the Go add "defer_resume". +type CfgEdge struct { + Src string `json:"src"` + Dst string `json:"dst"` + Kind string `json:"kind"` +} + +// CdgEdge is an L3 control-dependence edge between two local body keys. +type CdgEdge struct { + Src string `json:"src"` + Dst string `json:"dst"` +} + +// DdgEdge is a data-dependence edge between two local body keys on the +// k-limited access path Var. Prov is ["ssa"] at L3 (syntactic reaching +// definitions; the keystone's token, shared with the siblings). +type DdgEdge struct { + Src string `json:"src"` + Dst string `json:"dst"` + Var string `json:"var"` + Prov []string `json:"prov"` +} + // Edge is a typed edge overlay record. The list it lives in IS its type (there // is no `type` field). src/dst are node ids; no dangling endpoints. type Edge struct { diff --git a/internal/semantic_analysis/call_graph.go b/internal/semantic_analysis/call_graph.go index af2101a..8279c6c 100644 --- a/internal/semantic_analysis/call_graph.go +++ b/internal/semantic_analysis/call_graph.go @@ -11,9 +11,14 @@ package semantic_analysis import ( + "fmt" "go/ast" "go/token" "go/types" + "os" + "path/filepath" + "sort" + "strings" "golang.org/x/tools/go/packages" @@ -26,25 +31,35 @@ type CallGraphBuilder struct { projectDir string fset *token.FileSet pkgs map[string]*packages.Package + indexes map[*packages.Package]callSiteIndex + externals map[string]schema.GoExternalCallee } // NewCallGraphBuilder creates a builder using the same pkgs/fset loaded for the symbol table. func NewCallGraphBuilder(projectDir string, fset *token.FileSet, pkgs map[string]*packages.Package) *CallGraphBuilder { - return &CallGraphBuilder{projectDir: projectDir, fset: fset, pkgs: pkgs} + return &CallGraphBuilder{projectDir: projectDir, fset: fset, pkgs: pkgs, indexes: map[*packages.Package]callSiteIndex{}} } +// Externals returns the library callees recorded by Build, keyed by the full +// callee signature. Nil before Build or when no library call was seen. +func (cg *CallGraphBuilder) Externals() map[string]schema.GoExternalCallee { return cg.externals } + // Build resolves all call sites in symbolTable, backfills callee_signature, and // returns the identity-only edge list. Never crashes on unresolved sites — it logs // and skips the edge while leaving callee_signature nil. func (cg *CallGraphBuilder) Build(symbolTable map[string]schema.GoFile) []schema.GoCallEdge { var edges []schema.GoCallEdge - seen := map[[2]string]bool{} + seen := map[edgeKey]bool{} + cg.externals = nil - // Build the known-sig set once. Only emit edges where the target is in the - // project's symbol table (drop stdlib / external callees, same as Python/Jedi). + // Build the known-sig set once. It separates in-project callees from library + // callees, which are recorded in cg.externals rather than dropped. knownSigs := buildKnownSigs(symbolTable) - for fileKey, goFile := range symbolTable { + // Visit files, types and callables in sorted key order so the edge list and + // the first-writer content of cg.externals never depend on map iteration. + for _, fileKey := range sortedKeys(symbolTable) { + goFile := symbolTable[fileKey] // Find the package for this file. pkg := cg.packageForFile(fileKey) if pkg == nil || pkg.TypesInfo == nil { @@ -52,9 +67,10 @@ func (cg *CallGraphBuilder) Build(symbolTable map[string]schema.GoFile) []schema } // Resolve methods on types. - for typeSig, goType := range goFile.Types { - for methSig, callable := range goType.Methods { - newCallable, newEdges := cg.resolveCallable(pkg, callable, seen, knownSigs) + for _, typeSig := range sortedKeys(goFile.Types) { + goType := goFile.Types[typeSig] + for _, methSig := range sortedKeys(goType.Methods) { + newCallable, newEdges := cg.resolveCallable(pkg, goType.Methods[methSig], seen, knownSigs) goType.Methods[methSig] = newCallable edges = append(edges, newEdges...) } @@ -62,8 +78,8 @@ func (cg *CallGraphBuilder) Build(symbolTable map[string]schema.GoFile) []schema } // Resolve package-level functions. - for fnSig, callable := range goFile.Functions { - newCallable, newEdges := cg.resolveCallable(pkg, callable, seen, knownSigs) + for _, fnSig := range sortedKeys(goFile.Functions) { + newCallable, newEdges := cg.resolveCallable(pkg, goFile.Functions[fnSig], seen, knownSigs) goFile.Functions[fnSig] = newCallable edges = append(edges, newEdges...) } @@ -74,6 +90,20 @@ func (cg *CallGraphBuilder) Build(symbolTable map[string]schema.GoFile) []schema return edges } +// edgeKey dedups call edges per source callable. The declaring file is part of +// the key because a signature is not unique: every file of a package may declare +// its own func init(), and all of them share the signature ".init". +type edgeKey struct{ path, src, dst string } + +func sortedKeys[V any](m map[string]V) []string { + ks := make([]string, 0, len(m)) + for k := range m { + ks = append(ks, k) + } + sort.Strings(ks) + return ks +} + // buildKnownSigs returns the set of all callable signatures present in the symbol table. func buildKnownSigs(symbolTable map[string]schema.GoFile) map[string]bool { sigs := make(map[string]bool) @@ -91,13 +121,16 @@ func buildKnownSigs(symbolTable map[string]schema.GoFile) map[string]bool { } // resolveCallable backfills callee_signature on each call site and produces edges. -// Only emits edges where the target is in knownSigs (project-internal callees). -// External/stdlib callees have callee_signature backfilled but no edge emitted — -// matching Python/Jedi's behavior of dropping unresolved external sites. +// +// An in-project callee (in knownSigs) gets an edge. A library callee (a package +// outside the project) is recorded in cg.externals, its site is tagged with the +// callee's import path, and it gets an edge too (target = the full callee +// signature). A callee in a project package that is not in the symbol table +// (e.g. an interface method) is backfilled but gets no edge. func (cg *CallGraphBuilder) resolveCallable( pkg *packages.Package, callable schema.GoCallable, - seen map[[2]string]bool, + seen map[edgeKey]bool, knownSigs map[string]bool, ) (schema.GoCallable, []schema.GoCallEdge) { var edges []schema.GoCallEdge @@ -108,22 +141,26 @@ func (cg *CallGraphBuilder) resolveCallable( continue // already resolved } - calleeSig := cg.resolveCallSite(pkg, site) - if calleeSig == "" { + fn := cg.resolveCallSite(pkg, callable.Path, site) + if fn == nil { utils.Debug("unresolved call site: %s in %s", site.MethodName, callable.Signature) continue } + calleeSig := calleeSignatureOf(fn) - // Backfill the site regardless of whether the callee is in-project. + // Backfill the site whether the callee is in-project or a library. site.CalleeSignature = &calleeSig - // Only emit an edge when the callee is in the project's symbol table. if !knownSigs[calleeSig] { - utils.Debug("external callee (no edge): %s", calleeSig) - continue + if !cg.isLibrary(fn) { + utils.Debug("in-project callee without a symbol-table entry (no edge): %s", calleeSig) + continue + } + site.CalleeImportPath = fn.Pkg().Path() + cg.recordExternal(fn, calleeSig) } - key := [2]string{callable.Signature, calleeSig} + key := edgeKey{callable.Path, callable.Signature, calleeSig} if seen[key] { continue } @@ -147,68 +184,159 @@ func (cg *CallGraphBuilder) resolveCallable( return callable, edges } -// resolveCallSite resolves a single call site to a callee signature string. -// Returns "" when the site cannot be resolved (graceful fallback). -func (cg *CallGraphBuilder) resolveCallSite(pkg *packages.Package, site *schema.GoCallsite) string { - if pkg.TypesInfo == nil { - return "" +// isLibrary reports whether fn lives outside the project: its declaring file is +// outside projectDir, or under a vendored directory within it. This does not depend on which packages were loaded as +// roots, so it also holds in incremental (-t) mode, where sibling project +// packages load only as dependencies. When the position is unavailable it falls +// back to comparing the import path against the project's module path. +func (cg *CallGraphBuilder) isLibrary(fn *types.Func) bool { + if fn.Pkg() == nil { + return false + } + if tf := cg.fset.File(fn.Pos()); tf != nil { + file := absPath(tf.Name()) + root := cg.absProjectDir() + if !utils.IsWithin(root, file) { + return true + } + // A vendored dependency sits inside projectDir but is not project code + // (the symbol table skips it), so it is a library too. + return utils.IsVendored(utils.RelativePath(root, file)) + } + if mod := cg.modulePath(); mod != "" { + p := fn.Pkg().Path() + return p != mod && !strings.HasPrefix(p, mod+"/") } + _, inProject := cg.pkgs[fn.Pkg().Path()] + return !inProject +} - // Walk the package's syntax to find the call expression at this location. - for _, astFile := range pkg.Syntax { - var result string - ast.Inspect(astFile, func(n ast.Node) bool { - if result != "" { - return false - } - call, ok := n.(*ast.CallExpr) - if !ok { - return true - } - pos := cg.fset.Position(call.Pos()) - if pos.Line != site.StartLine || pos.Column != site.StartColumn { - return true - } - result = cg.resolveCallExpr(pkg, call) - return false - }) - if result != "" { - return result +func absPath(p string) string { + if abs, err := filepath.Abs(p); err == nil { + return abs + } + return p +} + +func (cg *CallGraphBuilder) absProjectDir() string { return absPath(cg.projectDir) } + +// modulePath reads the project's module path from go.mod ("" when unavailable). +func (cg *CallGraphBuilder) modulePath() string { + data, err := os.ReadFile(filepath.Join(cg.projectDir, "go.mod")) + if err != nil { + return "" + } + for _, line := range strings.Split(string(data), "\n") { + if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "module "); ok { + return strings.Trim(strings.TrimSpace(rest), "\"") } } return "" } -// resolveCallExpr resolves a *ast.CallExpr to a callee signature using go/types. -func (cg *CallGraphBuilder) resolveCallExpr(pkg *packages.Package, call *ast.CallExpr) string { +// recordExternal registers a library callee, keyed by its full signature. +// Parameter names and types come from the callee's *types.Signature; unnamed +// parameters become "_". The receiver is not a parameter. +func (cg *CallGraphBuilder) recordExternal(fn *types.Func, fullSig string) { + if _, ok := cg.externals[fullSig]; ok { + return + } + if cg.externals == nil { + cg.externals = map[string]schema.GoExternalCallee{} + } + importPath := fn.Pkg().Path() + // Record the declared signature, not the instance the call site selected: a + // method on a generic library type (atomic.Pointer[A].Store) must yield one + // order-independent entry whose params mention the type parameter (*T). + fn = fn.Origin() + sig := fn.Type().(*types.Signature) + + var params []schema.GoParameter + for i := 0; i < sig.Params().Len(); i++ { + p := sig.Params().At(i) + name := p.Name() + if name == "" || name == "_" { + name = fmt.Sprintf("_%d", i) + } + params = append(params, schema.GoParameter{ + Name: name, + Type: types.TypeString(p.Type(), nil), + IsVariadic: sig.Variadic() && i == sig.Params().Len()-1, + }) + } + + var rets []string + for i := 0; i < sig.Results().Len(); i++ { + rets = append(rets, types.TypeString(sig.Results().At(i).Type(), nil)) + } + ret := "" + switch len(rets) { + case 0: + case 1: + ret = rets[0] + default: + ret = "(" + strings.Join(rets, ", ") + ")" + } + + cg.externals[fullSig] = schema.GoExternalCallee{ + ImportPath: importPath, + Sig: strings.TrimPrefix(fullSig, importPath+"."), + Name: fn.Name(), + Params: params, + ReturnType: ret, + } +} + +// resolveCallSite resolves a single call site to its callee function by looking +// it up in the package's call-site index under the key L1 recorded it with: +// relPath (the file declaring the enclosing callable) and the site's +// directive-adjusted start and end line:col. Returns nil when the site cannot be +// resolved (graceful fallback). +func (cg *CallGraphBuilder) resolveCallSite(pkg *packages.Package, relPath string, site *schema.GoCallsite) *types.Func { + if pkg.TypesInfo == nil { + return nil + } + key := callKey{relPath, site.StartLine, site.StartColumn, site.EndLine, site.EndColumn} + call, ok := cg.indexFor(pkg)[key] + if !ok { + return nil + } + return cg.resolveCallExpr(pkg, call) +} + +// indexFor returns the (lazily built, cached) call-site index of pkg. +func (cg *CallGraphBuilder) indexFor(pkg *packages.Package) callSiteIndex { + if idx, ok := cg.indexes[pkg]; ok { + return idx + } + idx := newCallSiteIndex(cg.projectDir, cg.fset, pkg) + cg.indexes[pkg] = idx + return idx +} + +// resolveCallExpr resolves a *ast.CallExpr to its callee *types.Func using +// go/types. Type conversions, builtins and calls through function values are +// not calls to a function and resolve to nil. +func (cg *CallGraphBuilder) resolveCallExpr(pkg *packages.Package, call *ast.CallExpr) *types.Func { info := pkg.TypesInfo switch fn := call.Fun.(type) { case *ast.Ident: - if obj := info.ObjectOf(fn); obj != nil { - if f, ok := obj.(*types.Func); ok { - return calleeSignatureOf(f) - } - // Type conversion — normalize as constructor. - if tn, ok := obj.(*types.TypeName); ok { - return calleeSignatureOf(tn) + ".__new__" - } + if f, ok := info.ObjectOf(fn).(*types.Func); ok { + return f } case *ast.SelectorExpr: - sel, ok := info.Selections[fn] - if ok { + if sel, ok := info.Selections[fn]; ok { if f, ok := sel.Obj().(*types.Func); ok { - return calleeSignatureOf(f) + return f } } // Package-level function via qualified identifier. - if obj := info.ObjectOf(fn.Sel); obj != nil { - if f, ok := obj.(*types.Func); ok { - return calleeSignatureOf(f) - } + if f, ok := info.ObjectOf(fn.Sel).(*types.Func); ok { + return f } } - return "" + return nil } // calleeSignatureOf is the call-site mirror of signatureOf — same canonicalization. diff --git a/internal/semantic_analysis/call_graph_test.go b/internal/semantic_analysis/call_graph_test.go index a1ec09e..84cb74f 100644 --- a/internal/semantic_analysis/call_graph_test.go +++ b/internal/semantic_analysis/call_graph_test.go @@ -1,10 +1,15 @@ package semantic_analysis_test import ( + "fmt" + "path/filepath" + "reflect" + "sort" "testing" "github.com/codellm-devkit/codeanalyzer-go/internal/schema" "github.com/codellm-devkit/codeanalyzer-go/internal/semantic_analysis" + "github.com/codellm-devkit/codeanalyzer-go/internal/syntactic_analysis" ) func edge(src, tgt string, weight int, prov ...string) schema.GoCallEdge { @@ -114,3 +119,45 @@ func TestMergeEdges_OrderPreserved(t *testing.T) { t.Errorf("order not preserved: %v", result) } } + +// ── Call-site index ─────────────────────────────────────────────────────────── + +// TestCallGraph_IndexMatchesWalk locks the chi edge set (see chiGoldenEdges for the delta vs the walk) after +// resolveCallSite switched from a per-site AST walk to the position index. +func TestCallGraph_IndexMatchesWalk(t *testing.T) { + dir, err := filepath.Abs("../../testdata/chi") + if err != nil { + t.Fatal(err) + } + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("symbol table build failed: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + externals := cg.Externals() + + got := make([]string, 0, len(edges)) + for _, e := range edges { + if _, ext := externals[e.Target]; ext { + continue // the golden set locks in-project edges; library edges are tested apart + } + got = append(got, e.Source+" -> "+e.Target) + } + sort.Strings(got) + + if !reflect.DeepEqual(got, chiGoldenEdges) { + t.Fatalf("edge set changed: got %d edges, want %d\nfirst diff: %s", + len(got), len(chiGoldenEdges), firstDiff(got, chiGoldenEdges)) + } +} + +func firstDiff(a, b []string) string { + for i := 0; i < len(a) && i < len(b); i++ { + if a[i] != b[i] { + return fmt.Sprintf("[%d] got %q want %q", i, a[i], b[i]) + } + } + return fmt.Sprintf("length differs (%d vs %d)", len(a), len(b)) +} diff --git a/internal/semantic_analysis/callsite_index.go b/internal/semantic_analysis/callsite_index.go new file mode 100644 index 0000000..d4ec07e --- /dev/null +++ b/internal/semantic_analysis/callsite_index.go @@ -0,0 +1,54 @@ +package semantic_analysis + +import ( + "go/ast" + "go/token" + + "golang.org/x/tools/go/packages" + + "github.com/codellm-devkit/codeanalyzer-go/internal/utils" +) + +// callKey identifies a call expression exactly as L1 records its site: the +// project-relative path of the file that contains it (the token.File name, the +// same path as the enclosing callable's Path) and its start and end line:col +// from fset.Position, which applies //line directives. Start alone is +// ambiguous: a chained s.A().B() starts where its inner s.A() does. +// +// The key is never turned back into a token.Pos: under a //line directive the +// recorded line names a virtual line (often past the file's real line count), +// so rebuilding a position from it would be wrong or panic. +type callKey struct { + path string + startLine, startCol, endLine, endColumn int +} + +// callSiteIndex maps a call expression's callKey to the call. +type callSiteIndex map[callKey]*ast.CallExpr + +// newCallSiteIndex walks each file of pkg once and indexes every call. When +// two calls of one file share a key (only possible when //line directives map +// both to the same virtual range), the first in source order wins. +func newCallSiteIndex(projectDir string, fset *token.FileSet, pkg *packages.Package) callSiteIndex { + idx := callSiteIndex{} + for _, f := range pkg.Syntax { + tf := fset.File(f.Pos()) + if tf == nil { + continue + } + rel := utils.RelativePath(projectDir, tf.Name()) + ast.Inspect(f, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + start, end := fset.Position(call.Pos()), fset.Position(call.End()) + k := callKey{rel, start.Line, start.Column, end.Line, end.Column} + if _, dup := idx[k]; !dup { + idx[k] = call + } + return true + }) + } + return idx +} diff --git a/internal/semantic_analysis/chi_golden_test.go b/internal/semantic_analysis/chi_golden_test.go new file mode 100644 index 0000000..1d09efb --- /dev/null +++ b/internal/semantic_analysis/chi_golden_test.go @@ -0,0 +1,138 @@ +package semantic_analysis_test + +// chiGoldenEdges is the sorted "src -> dst" call-graph edge set for testdata/chi. +// +// It was captured with the pre-index resolver (a per-site walk of every file in +// the package, matching on line:col alone) and then corrected for that walk's +// defects. The index is file-scoped and keyed on [start, end): +// - three edges the walk invented from a same-position call in a sibling file +// are gone: DefaultLogFormatter.NewLogEntry -> Compressor.SetEncoder, +// Logger -> ThrottleWithOpts, init -> LogFormatter.NewLogEntry; +// - Throttle -> ThrottleWithOpts (throttle.go:33) is gained for the same reason; +// - seven edges from the inner call of a chained call (e.g. +// mx.NotFoundHandler().ServeHTTP, Chain(...).HandlerFunc) are gained, since +// the outer call no longer shadows the co-positioned inner one. +var chiGoldenEdges = []string{ + "github.com/go-chi/chi/v5.Context.RoutePattern -> github.com/go-chi/chi/v5.replaceWildcards", + "github.com/go-chi/chi/v5.Middlewares.Handler -> github.com/go-chi/chi/v5.chain", + "github.com/go-chi/chi/v5.Middlewares.HandlerFunc -> github.com/go-chi/chi/v5.chain", + "github.com/go-chi/chi/v5.Mux.Connect -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Delete -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Find -> github.com/go-chi/chi/v5.Mux.nextRoutePath", + "github.com/go-chi/chi/v5.Mux.Find -> github.com/go-chi/chi/v5.Routes.Find", + "github.com/go-chi/chi/v5.Mux.Find -> github.com/go-chi/chi/v5.node.FindRoute", + "github.com/go-chi/chi/v5.Mux.Get -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Group -> github.com/go-chi/chi/v5.Mux.With", + "github.com/go-chi/chi/v5.Mux.Handle -> github.com/go-chi/chi/v5.Mux.Method", + "github.com/go-chi/chi/v5.Mux.Handle -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.HandleFunc -> github.com/go-chi/chi/v5.Mux.Handle", + "github.com/go-chi/chi/v5.Mux.Head -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Match -> github.com/go-chi/chi/v5.Mux.Find", + "github.com/go-chi/chi/v5.Mux.Method -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.MethodFunc -> github.com/go-chi/chi/v5.Mux.Method", + "github.com/go-chi/chi/v5.Mux.MethodNotAllowed -> github.com/go-chi/chi/v5.Chain", + "github.com/go-chi/chi/v5.Mux.MethodNotAllowed -> github.com/go-chi/chi/v5.Middlewares.HandlerFunc", + "github.com/go-chi/chi/v5.Mux.MethodNotAllowed -> github.com/go-chi/chi/v5.Mux.updateSubRoutes", + "github.com/go-chi/chi/v5.Mux.MethodNotAllowedHandler -> github.com/go-chi/chi/v5.methodNotAllowedHandler", + "github.com/go-chi/chi/v5.Mux.Mount -> github.com/go-chi/chi/v5.Mux.MethodNotAllowed", + "github.com/go-chi/chi/v5.Mux.Mount -> github.com/go-chi/chi/v5.Mux.NotFound", + "github.com/go-chi/chi/v5.Mux.Mount -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Mount -> github.com/go-chi/chi/v5.node.findPattern", + "github.com/go-chi/chi/v5.Mux.NotFound -> github.com/go-chi/chi/v5.Chain", + "github.com/go-chi/chi/v5.Mux.NotFound -> github.com/go-chi/chi/v5.Middlewares.HandlerFunc", + "github.com/go-chi/chi/v5.Mux.NotFound -> github.com/go-chi/chi/v5.Mux.updateSubRoutes", + "github.com/go-chi/chi/v5.Mux.Options -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Patch -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Post -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Put -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.Route -> github.com/go-chi/chi/v5.Mux.Mount", + "github.com/go-chi/chi/v5.Mux.Route -> github.com/go-chi/chi/v5.NewRouter", + "github.com/go-chi/chi/v5.Mux.Routes -> github.com/go-chi/chi/v5.node.routes", + "github.com/go-chi/chi/v5.Mux.ServeHTTP -> github.com/go-chi/chi/v5.Context.Reset", + "github.com/go-chi/chi/v5.Mux.ServeHTTP -> github.com/go-chi/chi/v5.Mux.NotFoundHandler", + "github.com/go-chi/chi/v5.Mux.Trace -> github.com/go-chi/chi/v5.Mux.handle", + "github.com/go-chi/chi/v5.Mux.With -> github.com/go-chi/chi/v5.Mux.updateRouteHandler", + "github.com/go-chi/chi/v5.Mux.handle -> github.com/go-chi/chi/v5.Chain", + "github.com/go-chi/chi/v5.Mux.handle -> github.com/go-chi/chi/v5.Middlewares.Handler", + "github.com/go-chi/chi/v5.Mux.handle -> github.com/go-chi/chi/v5.Mux.updateRouteHandler", + "github.com/go-chi/chi/v5.Mux.handle -> github.com/go-chi/chi/v5.node.InsertRoute", + "github.com/go-chi/chi/v5.Mux.routeHTTP -> github.com/go-chi/chi/v5.Context.RoutePattern", + "github.com/go-chi/chi/v5.Mux.routeHTTP -> github.com/go-chi/chi/v5.Mux.MethodNotAllowedHandler", + "github.com/go-chi/chi/v5.Mux.routeHTTP -> github.com/go-chi/chi/v5.Mux.NotFoundHandler", + "github.com/go-chi/chi/v5.Mux.routeHTTP -> github.com/go-chi/chi/v5.node.FindRoute", + "github.com/go-chi/chi/v5.Mux.updateRouteHandler -> github.com/go-chi/chi/v5.chain", + "github.com/go-chi/chi/v5.Mux.updateSubRoutes -> github.com/go-chi/chi/v5.node.routes", + "github.com/go-chi/chi/v5.NewRouter -> github.com/go-chi/chi/v5.NewMux", + "github.com/go-chi/chi/v5.URLParam -> github.com/go-chi/chi/v5.Context.URLParam", + "github.com/go-chi/chi/v5.URLParam -> github.com/go-chi/chi/v5.RouteContext", + "github.com/go-chi/chi/v5.URLParamFromCtx -> github.com/go-chi/chi/v5.Context.URLParam", + "github.com/go-chi/chi/v5.URLParamFromCtx -> github.com/go-chi/chi/v5.RouteContext", + "github.com/go-chi/chi/v5.Walk -> github.com/go-chi/chi/v5.walk", + "github.com/go-chi/chi/v5.node.FindRoute -> github.com/go-chi/chi/v5.node.findRoute", + "github.com/go-chi/chi/v5.node.InsertRoute -> github.com/go-chi/chi/v5.longestPrefix", + "github.com/go-chi/chi/v5.node.InsertRoute -> github.com/go-chi/chi/v5.node.addChild", + "github.com/go-chi/chi/v5.node.InsertRoute -> github.com/go-chi/chi/v5.node.getEdge", + "github.com/go-chi/chi/v5.node.InsertRoute -> github.com/go-chi/chi/v5.node.replaceChild", + "github.com/go-chi/chi/v5.node.InsertRoute -> github.com/go-chi/chi/v5.node.setEndpoint", + "github.com/go-chi/chi/v5.node.InsertRoute -> github.com/go-chi/chi/v5.patNextSegment", + "github.com/go-chi/chi/v5.node.addChild -> github.com/go-chi/chi/v5.node.addChild", + "github.com/go-chi/chi/v5.node.addChild -> github.com/go-chi/chi/v5.nodes.Sort", + "github.com/go-chi/chi/v5.node.addChild -> github.com/go-chi/chi/v5.patNextSegment", + "github.com/go-chi/chi/v5.node.findPattern -> github.com/go-chi/chi/v5.longestPrefix", + "github.com/go-chi/chi/v5.node.findPattern -> github.com/go-chi/chi/v5.node.findEdge", + "github.com/go-chi/chi/v5.node.findPattern -> github.com/go-chi/chi/v5.node.findPattern", + "github.com/go-chi/chi/v5.node.findRoute -> github.com/go-chi/chi/v5.node.findRoute", + "github.com/go-chi/chi/v5.node.findRoute -> github.com/go-chi/chi/v5.node.isLeaf", + "github.com/go-chi/chi/v5.node.findRoute -> github.com/go-chi/chi/v5.nodes.findEdge", + "github.com/go-chi/chi/v5.node.routes -> github.com/go-chi/chi/v5.node.walk", + "github.com/go-chi/chi/v5.node.setEndpoint -> github.com/go-chi/chi/v5.endpoints.Value", + "github.com/go-chi/chi/v5.node.setEndpoint -> github.com/go-chi/chi/v5.patParamKeys", + "github.com/go-chi/chi/v5.node.walk -> github.com/go-chi/chi/v5.node.walk", + "github.com/go-chi/chi/v5.nodes.Sort -> github.com/go-chi/chi/v5.nodes.tailSort", + "github.com/go-chi/chi/v5.nodes.tailSort -> github.com/go-chi/chi/v5.nodes.Swap", + "github.com/go-chi/chi/v5.patParamKeys -> github.com/go-chi/chi/v5.patNextSegment", + "github.com/go-chi/chi/v5.walk -> github.com/go-chi/chi/v5.Routes.Middlewares", + "github.com/go-chi/chi/v5.walk -> github.com/go-chi/chi/v5.Routes.Routes", + "github.com/go-chi/chi/v5.walk -> github.com/go-chi/chi/v5.walk", + "github.com/go-chi/chi/v5/middleware.Compress -> github.com/go-chi/chi/v5/middleware.NewCompressor", + "github.com/go-chi/chi/v5/middleware.Compressor.selectEncoder -> github.com/go-chi/chi/v5/middleware.ioResetterWriter.Reset", + "github.com/go-chi/chi/v5/middleware.Compressor.selectEncoder -> github.com/go-chi/chi/v5/middleware.matchAcceptEncoding", + "github.com/go-chi/chi/v5/middleware.DefaultLogFormatter.NewLogEntry -> github.com/go-chi/chi/v5/middleware.GetClientIP", + "github.com/go-chi/chi/v5/middleware.DefaultLogFormatter.NewLogEntry -> github.com/go-chi/chi/v5/middleware.GetReqID", + "github.com/go-chi/chi/v5/middleware.DefaultLogFormatter.NewLogEntry -> github.com/go-chi/chi/v5/middleware.cW", + "github.com/go-chi/chi/v5/middleware.GetClientIP -> github.com/go-chi/chi/v5/middleware.GetClientIPAddr", + "github.com/go-chi/chi/v5/middleware.HeaderRoute.IsMatch -> github.com/go-chi/chi/v5/middleware.Pattern.Match", + "github.com/go-chi/chi/v5/middleware.HeaderRouter.Route -> github.com/go-chi/chi/v5/middleware.NewPattern", + "github.com/go-chi/chi/v5/middleware.HeaderRouter.RouteAny -> github.com/go-chi/chi/v5/middleware.NewPattern", + "github.com/go-chi/chi/v5/middleware.NewCompressor -> github.com/go-chi/chi/v5/middleware.Compressor.SetEncoder", + "github.com/go-chi/chi/v5/middleware.PrintPrettyStack -> github.com/go-chi/chi/v5/middleware.prettyStack.parse", + "github.com/go-chi/chi/v5/middleware.Profiler -> github.com/go-chi/chi/v5.Mux.Get", + "github.com/go-chi/chi/v5/middleware.Profiler -> github.com/go-chi/chi/v5.Mux.Handle", + "github.com/go-chi/chi/v5/middleware.Profiler -> github.com/go-chi/chi/v5.Mux.HandleFunc", + "github.com/go-chi/chi/v5/middleware.Profiler -> github.com/go-chi/chi/v5.Mux.Use", + "github.com/go-chi/chi/v5/middleware.Profiler -> github.com/go-chi/chi/v5.NewRouter", + "github.com/go-chi/chi/v5/middleware.Throttle -> github.com/go-chi/chi/v5/middleware.ThrottleWithOpts", + "github.com/go-chi/chi/v5/middleware.ThrottleBacklog -> github.com/go-chi/chi/v5/middleware.ThrottleWithOpts", + "github.com/go-chi/chi/v5/middleware.basicWriter.Write -> github.com/go-chi/chi/v5/middleware.basicWriter.maybeWriteHeader", + "github.com/go-chi/chi/v5/middleware.basicWriter.maybeWriteHeader -> github.com/go-chi/chi/v5/middleware.basicWriter.WriteHeader", + "github.com/go-chi/chi/v5/middleware.compressResponseWriter.Close -> github.com/go-chi/chi/v5/middleware.compressResponseWriter.writer", + "github.com/go-chi/chi/v5/middleware.compressResponseWriter.Flush -> github.com/go-chi/chi/v5/middleware.compressFlusher.Flush", + "github.com/go-chi/chi/v5/middleware.compressResponseWriter.Flush -> github.com/go-chi/chi/v5/middleware.compressResponseWriter.writer", + "github.com/go-chi/chi/v5/middleware.compressResponseWriter.Hijack -> github.com/go-chi/chi/v5/middleware.compressResponseWriter.writer", + "github.com/go-chi/chi/v5/middleware.compressResponseWriter.Push -> github.com/go-chi/chi/v5/middleware.compressResponseWriter.writer", + "github.com/go-chi/chi/v5/middleware.compressResponseWriter.Write -> github.com/go-chi/chi/v5/middleware.compressResponseWriter.WriteHeader", + "github.com/go-chi/chi/v5/middleware.compressResponseWriter.Write -> github.com/go-chi/chi/v5/middleware.compressResponseWriter.writer", + "github.com/go-chi/chi/v5/middleware.compressResponseWriter.WriteHeader -> github.com/go-chi/chi/v5/middleware.compressResponseWriter.isCompressible", + "github.com/go-chi/chi/v5/middleware.contentEncoding -> github.com/go-chi/chi/v5/middleware.split", + "github.com/go-chi/chi/v5/middleware.defaultLogEntry.Panic -> github.com/go-chi/chi/v5/middleware.PrintPrettyStack", + "github.com/go-chi/chi/v5/middleware.defaultLogEntry.Write -> github.com/go-chi/chi/v5/middleware.LoggerInterface.Print", + "github.com/go-chi/chi/v5/middleware.defaultLogEntry.Write -> github.com/go-chi/chi/v5/middleware.cW", + "github.com/go-chi/chi/v5/middleware.httpFancyWriter.ReadFrom -> github.com/go-chi/chi/v5/middleware.basicWriter.maybeWriteHeader", + "github.com/go-chi/chi/v5/middleware.init -> github.com/go-chi/chi/v5/middleware.RequestLogger", + "github.com/go-chi/chi/v5/middleware.prettyStack.decorateFuncCallLine -> github.com/go-chi/chi/v5/middleware.cW", + "github.com/go-chi/chi/v5/middleware.prettyStack.decorateLine -> github.com/go-chi/chi/v5/middleware.prettyStack.decorateFuncCallLine", + "github.com/go-chi/chi/v5/middleware.prettyStack.decorateLine -> github.com/go-chi/chi/v5/middleware.prettyStack.decorateSourceLine", + "github.com/go-chi/chi/v5/middleware.prettyStack.decorateSourceLine -> github.com/go-chi/chi/v5/middleware.cW", + "github.com/go-chi/chi/v5/middleware.prettyStack.parse -> github.com/go-chi/chi/v5/middleware.cW", + "github.com/go-chi/chi/v5/middleware.prettyStack.parse -> github.com/go-chi/chi/v5/middleware.prettyStack.decorateLine", +} diff --git a/internal/syntactic_analysis/l3units.go b/internal/syntactic_analysis/l3units.go new file mode 100644 index 0000000..3cac08d --- /dev/null +++ b/internal/syntactic_analysis/l3units.go @@ -0,0 +1,124 @@ +package syntactic_analysis + +import ( + "fmt" + "go/ast" + "go/types" + "sort" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" + "github.com/codellm-devkit/codeanalyzer-go/internal/schema" + v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" + "github.com/codellm-devkit/codeanalyzer-go/internal/utils" +) + +// L3Units builds the dataflow.Run input for symbolTable (the table Build +// returned): one unit per emitted callable with a body, closures included, +// sorted by key. +// +// - A FuncDecl is matched to its symbol-table callable by (declaring file, +// signature) — the same pair the emitter looks results up by — and its +// start line. The line check keeps a second `func init()` in one file, +// which the symbol table overwrote, from taking the survivor's key. +// Declarations the symbol table dropped (cgo wrappers, methods whose +// receiver type was not found) get no unit. +// - A FuncLit is matched to its L1 closure by the traversal that named it: +// buildInnerCallables numbers the outermost literals of a body +// closure_1, closure_2, … in ast.Inspect order and recurses per literal, +// so walking the same way pairs each literal with InnerCallables[name]. +// Its start line is checked too. Its key's Path is the enclosing +// callable's declaring file. +// - Calls is v2.CallBodyKeys over the callable's L1 call sites, the exact +// keys the L1 emitter writes. +// +// A mismatch (which would mean the walk drifted from L1's) is logged and the +// callable is left at L1. +func (b *SymbolTableBuilder) L3Units(symbolTable map[string]schema.GoFile) []dataflow.Unit { + byKey := map[dataflow.L3Key]schema.GoCallable{} + for _, f := range symbolTable { + for _, c := range f.Functions { + byKey[dataflow.L3Key{Path: c.Path, Signature: c.Signature}] = c + } + for _, t := range f.Types { + for _, c := range t.Methods { + if c.Path != "" { // interface methods have no declaration + byKey[dataflow.L3Key{Path: c.Path, Signature: c.Signature}] = c + } + } + } + } + + paths := make([]string, 0, len(b.pkgs)) + for p := range b.pkgs { + paths = append(paths, p) + } + sort.Strings(paths) + + var units []dataflow.Unit + seen := map[dataflow.L3Key]bool{} + for _, pp := range paths { + pkg := b.pkgs[pp] + for _, file := range pkg.Syntax { + for _, d := range file.Decls { + fd, ok := d.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + tf := b.fset.File(fd.Pos()) + if tf == nil || !utils.IsWithin(b.projectDir, tf.Name()) { + continue + } + key := dataflow.L3Key{ + Path: utils.RelativePath(b.projectDir, tf.Name()), + Signature: b.declSignature(pkg, fd), + } + c, ok := byKey[key] + if !ok || seen[key] || c.StartLine != b.fset.Position(fd.Pos()).Line { + continue + } + seen[key] = true + units = append(units, unit(key, fd, pkg.TypesInfo, c)) + units = b.closureUnits(units, key.Path, fd.Body, c, pkg.TypesInfo) + } + } + } + sort.Slice(units, func(i, j int) bool { + a, b := units[i].Key, units[j].Key + if a.Path != b.Path { + return a.Path < b.Path + } + return a.Signature < b.Signature + }) + return units +} + +// closureUnits appends a unit for every function literal of body, mirroring +// buildInnerCallables' naming walk. +func (b *SymbolTableBuilder) closureUnits(units []dataflow.Unit, path string, body *ast.BlockStmt, outer schema.GoCallable, info *types.Info) []dataflow.Unit { + n := 0 + ast.Inspect(body, func(node ast.Node) bool { + lit, ok := node.(*ast.FuncLit) + if !ok { + return true + } + n++ + name := fmt.Sprintf("closure_%d", n) + ic, ok := outer.InnerCallables[name] + if !ok || ic.StartLine != b.fset.Position(lit.Pos()).Line { + utils.Warn("L3 skipped for %s.%s (%s): no matching L1 closure", outer.Signature, name, path) + return false + } + units = append(units, unit(dataflow.L3Key{Path: path, Signature: ic.Signature}, lit, info, ic)) + units = b.closureUnits(units, path, lit.Body, ic, info) + return false + }) + return units +} + +func unit(key dataflow.L3Key, fn ast.Node, info *types.Info, c schema.GoCallable) dataflow.Unit { + pos := make([][2]int, len(c.CallSites)) + for i, cs := range c.CallSites { + pos[i] = [2]int{cs.StartLine, cs.StartColumn} + } + return dataflow.Unit{Key: key, Fn: fn, Info: info, Calls: v2.CallBodyKeys(pos)} +} diff --git a/internal/syntactic_analysis/l3units_test.go b/internal/syntactic_analysis/l3units_test.go new file mode 100644 index 0000000..b1276ae --- /dev/null +++ b/internal/syntactic_analysis/l3units_test.go @@ -0,0 +1,51 @@ +package syntactic_analysis_test + +import ( + "path/filepath" + "runtime" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" + "github.com/codellm-devkit/codeanalyzer-go/internal/syntactic_analysis" +) + +// TestL3Units_UnmatchedClosureSkipped: a function literal whose L1 closure +// entry does not match (here its recorded start line is off by one) gets no +// unit — never another closure's key — while its enclosing callable keeps +// its unit. +func TestL3Units_UnmatchedClosureSkipped(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata", "dataflow") + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatal(err) + } + path := filepath.Join("flow", "flow.go") + outer := dataflow.L3Key{Path: path, Signature: "example.com/dataflow/flow.Closure"} + closure := dataflow.L3Key{Path: path, Signature: outer.Signature + ".closure_1"} + + has := func() map[dataflow.L3Key]bool { + m := map[dataflow.L3Key]bool{} + for _, u := range b.L3Units(st) { + m[u.Key] = true + } + return m + } + if got := has(); !got[outer] || !got[closure] { + t.Fatalf("baseline: outer %v, closure %v; want both", got[outer], got[closure]) + } + + fn := st[path].Functions[outer.Signature] + ic := fn.InnerCallables["closure_1"] + ic.StartLine++ + fn.InnerCallables["closure_1"] = ic + + got := has() + if got[closure] { + t.Error("mismatched closure still got a unit") + } + if !got[outer] { + t.Error("enclosing callable lost its unit") + } +} diff --git a/internal/syntactic_analysis/symbol_table.go b/internal/syntactic_analysis/symbol_table.go index 93d32ab..a0e2b35 100644 --- a/internal/syntactic_analysis/symbol_table.go +++ b/internal/syntactic_analysis/symbol_table.go @@ -73,16 +73,22 @@ func (b *SymbolTableBuilder) Build(targetFiles []string, skipTests bool) (map[st } // Collect packages, warn on errors but don't abort (graceful partial analysis). + var tooNew toolchainGap for _, pkg := range pkgList { - if len(pkg.Errors) > 0 { - for _, e := range pkg.Errors { - utils.Warn("package %s: %v", pkg.PkgPath, e) + for _, e := range pkg.Errors { + if tooNew.record(pkg.PkgPath, e.Msg) { + // Summarized once below; one line per package is noise on a + // large module and hides the cause. + utils.Debug("package %s: %v", pkg.PkgPath, e) + continue } + utils.Warn("package %s: %v", pkg.PkgPath, e) } if pkg.Types != nil { b.pkgs[pkg.PkgPath] = pkg } } + tooNew.warn() symbolTable := map[string]schema.GoFile{} @@ -360,18 +366,20 @@ func (b *SymbolTableBuilder) buildType( end := b.fset.Position(spec.End()) gt := &schema.GoType{ - Name: typeName, - Signature: typSig, - Comments: b.docComments(decl.Doc), - IsExported: isExported, - Fields: []schema.GoField{}, - Methods: map[string]schema.GoCallable{}, - BaseTypes: []string{}, - InnerTypes: map[string]schema.GoType{}, - StartLine: pos.Line, - EndLine: end.Line, + Name: typeName, + Signature: typSig, + Comments: b.docComments(decl.Doc), + IsExported: isExported, + Fields: []schema.GoField{}, + Methods: map[string]schema.GoCallable{}, + BaseTypes: []string{}, + InnerTypes: map[string]schema.GoType{}, + StartLine: pos.Line, + EndLine: end.Line, } + gt.TypeKind = b.typeKindOf(pkg, spec) + switch t := spec.Type.(type) { case *ast.StructType: gt.IsInterface = false @@ -387,6 +395,37 @@ func (b *SymbolTableBuilder) buildType( return gt } +// typeKindOf classifies a type declaration: alias for `type X = Y`; otherwise +// by the UNDERLYING type (so `type E S` over a struct is struct and `type J I` +// over an interface is interface); defined for any other underlying type. It +// falls back to the AST shape of the right-hand side when type info is missing. +func (b *SymbolTableBuilder) typeKindOf(pkg *packages.Package, spec *ast.TypeSpec) string { + if spec.Assign.IsValid() { + return "alias" + } + if pkg.TypesInfo != nil { + if obj := pkg.TypesInfo.Defs[spec.Name]; obj != nil { + if named, ok := obj.Type().(*types.Named); ok { + switch named.Underlying().(type) { + case *types.Struct: + return "struct" + case *types.Interface: + return "interface" + default: + return "defined" + } + } + } + } + switch spec.Type.(type) { + case *ast.StructType: + return "struct" + case *ast.InterfaceType: + return "interface" + } + return "defined" +} + func (b *SymbolTableBuilder) buildStructFields(pkg *packages.Package, st *ast.StructType) []schema.GoField { var fields []schema.GoField if st.Fields == nil { @@ -466,18 +505,18 @@ func (b *SymbolTableBuilder) collectInterfaceMethods(pkg *packages.Package, it * sig := pkg.Types.Path() + "." + gt.Name + "." + name.Name callable := schema.GoCallable{ - Name: name.Name, - Path: "", - Signature: sig, - Parameters: b.buildFuncTypeParams(pkg, method.Type), - ReturnType: retType, - ReturnTypes: retTypes, - IsExported: unicode.IsUpper(rune(name.Name[0])), - ReceiverType: gt.Signature, - CallSites: []schema.GoCallsite{}, + Name: name.Name, + Path: "", + Signature: sig, + Parameters: b.buildFuncTypeParams(pkg, method.Type), + ReturnType: retType, + ReturnTypes: retTypes, + IsExported: unicode.IsUpper(rune(name.Name[0])), + ReceiverType: gt.Signature, + CallSites: []schema.GoCallsite{}, InnerCallables: map[string]schema.GoCallable{}, - StartLine: pos.Line, - EndLine: end.Line, + StartLine: pos.Line, + EndLine: end.Line, } gt.Methods[sig] = callable } @@ -499,8 +538,8 @@ func (b *SymbolTableBuilder) buildCallable( // They are toolchain artifacts, not project source — emitting them yields // callables whose span cannot be sliced against any project module. Drop // them here, at the point their (cache) file is first known. - if declFile := b.fset.File(decl.Pos()); declFile == nil || - !utils.IsWithin(b.projectDir, declFile.Name()) { + declFile := b.fset.File(decl.Pos()) + if declFile == nil || !utils.IsWithin(b.projectDir, declFile.Name()) { return nil } @@ -508,22 +547,7 @@ func (b *SymbolTableBuilder) buildCallable( pos := b.fset.Position(decl.Pos()) end := b.fset.Position(decl.End()) - var sig string - if pkg.TypesInfo != nil { - if obj, ok := pkg.TypesInfo.Defs[decl.Name]; ok && obj != nil { - sig = signatureOf(obj) - } - } - if sig == "" { - if decl.Recv != nil { - if recvName := b.receiverTypeName(decl.Recv); recvName != "" { - sig = pkg.Types.Path() + "." + recvName + "." + name - } - } - if sig == "" { - sig = pkg.Types.Path() + "." + name - } - } + sig := b.declSignature(pkg, decl) var recvType, recvName string if decl.Recv != nil && len(decl.Recv.List) > 0 { @@ -541,23 +565,23 @@ func (b *SymbolTableBuilder) buildCallable( } callable := &schema.GoCallable{ - Name: name, - Path: utils.RelativePath(b.projectDir, b.fset.File(decl.Pos()).Name()), - Signature: sig, - Comments: b.docComments(decl.Doc), - Parameters: b.buildParams(pkg, decl.Type), - ReturnType: retType, - ReturnTypes: retTypes, - IsExported: isExported, - ReceiverType: recvType, - ReceiverName: recvName, - CallSites: []schema.GoCallsite{}, - InnerCallables: map[string]schema.GoCallable{}, - LocalVariables: []schema.GoVariableDeclaration{}, - AccessedSymbols: []schema.GoSymbol{}, - StartLine: pos.Line, - EndLine: end.Line, - CodeStartLine: bodyStart, + Name: name, + Path: utils.RelativePath(b.projectDir, declFile.Name()), + Signature: sig, + Comments: b.docComments(decl.Doc), + Parameters: b.buildParams(pkg, decl.Type), + ReturnType: retType, + ReturnTypes: retTypes, + IsExported: isExported, + ReceiverType: recvType, + ReceiverName: recvName, + CallSites: []schema.GoCallsite{}, + InnerCallables: map[string]schema.GoCallable{}, + LocalVariables: []schema.GoVariableDeclaration{}, + AccessedSymbols: []schema.GoSymbol{}, + StartLine: pos.Line, + EndLine: end.Line, + CodeStartLine: bodyStart, CyclomaticComplexity: b.cyclomaticComplexity(decl), } @@ -571,6 +595,24 @@ func (b *SymbolTableBuilder) buildCallable( return callable } +// declSignature is a FuncDecl's callable signature: signatureOf its object, +// or, without type info, [.].. +func (b *SymbolTableBuilder) declSignature(pkg *packages.Package, decl *ast.FuncDecl) string { + if pkg.TypesInfo != nil { + if obj, ok := pkg.TypesInfo.Defs[decl.Name]; ok && obj != nil { + if sig := signatureOf(obj); sig != "" { + return sig + } + } + } + if decl.Recv != nil { + if recvName := b.receiverTypeName(decl.Recv); recvName != "" { + return pkg.Types.Path() + "." + recvName + "." + decl.Name.Name + } + } + return pkg.Types.Path() + "." + decl.Name.Name +} + // ─── Parameters ─────────────────────────────────────────────────────────────── func (b *SymbolTableBuilder) buildParams(pkg *packages.Package, ft *ast.FuncType) []schema.GoParameter { @@ -646,24 +688,27 @@ func (b *SymbolTableBuilder) joinReturnTypes(parts []string) string { // callee_signature is left nil here — the resolver backfills it in the call-graph stage. func (b *SymbolTableBuilder) buildCallSites(pkg *packages.Package, body *ast.BlockStmt) []schema.GoCallsite { var sites []schema.GoCallsite + // goroutine / deferred hold the call of each `go` / `defer` statement seen + // so far. Pre-order traversal visits the statement before its call, so the + // CallExpr case can tag it, and continues into the call so calls in its + // function value and arguments (evaluated by the launching goroutine, at + // the statement) are recorded as ordinary sites. The L3 walk + // (dataflow.recordedCalls) must record the same calls in the same order. + goroutine := map[*ast.CallExpr]bool{} + deferred := map[*ast.CallExpr]bool{} ast.Inspect(body, func(n ast.Node) bool { switch node := n.(type) { case *ast.GoStmt: - // Goroutine launch — record with is_goroutine=true. - if call, ok := node.Call.Fun.(*ast.CallExpr); ok { - _ = call - } - site := b.callExprToSite(pkg, node.Call, true) - if site != nil { - sites = append(sites, *site) - } - return false + goroutine[node.Call] = true + case *ast.DeferStmt: + deferred[node.Call] = true case *ast.FuncLit: // Closure bodies are handled by buildInnerCallables; don't double-count. return false case *ast.CallExpr: - site := b.callExprToSite(pkg, node, false) + site := b.callExprToSite(pkg, node, goroutine[node]) if site != nil { + site.IsDeferred = deferred[node] sites = append(sites, *site) } } @@ -957,8 +1002,16 @@ func (b *SymbolTableBuilder) exprString(expr ast.Expr) string { } } -// nodeSource extracts the raw source text of a node (best effort). +// nodeSource extracts the raw source text of a node (best effort). It returns +// "" when the text cannot be recovered: an unreadable file, or a node from a +// malformed file whose Pos or End has no token.File (or ends outside the file +// it starts in), which the parser produces when it recovers from a syntax +// error. func (b *SymbolTableBuilder) nodeSource(node ast.Node) string { + tf := b.fset.File(node.Pos()) + if tf == nil || b.fset.File(node.End()) != tf { + return "" + } pos := b.fset.Position(node.Pos()) if pos.Filename == "" { return "" @@ -967,8 +1020,8 @@ func (b *SymbolTableBuilder) nodeSource(node ast.Node) string { if err != nil { return "" } - startOff := b.fset.File(node.Pos()).Offset(node.Pos()) - endOff := b.fset.File(node.End()).Offset(node.End()) + startOff := tf.Offset(node.Pos()) + endOff := tf.Offset(node.End()) if startOff < 0 || endOff > len(data) || startOff >= endOff { return "" } diff --git a/internal/syntactic_analysis/toolchain.go b/internal/syntactic_analysis/toolchain.go new file mode 100644 index 0000000..27aa1fb --- /dev/null +++ b/internal/syntactic_analysis/toolchain.go @@ -0,0 +1,55 @@ +package syntactic_analysis + +import ( + "go/version" + "regexp" + "runtime" + + "github.com/codellm-devkit/codeanalyzer-go/internal/utils" +) + +// tooNewRE matches go/types' TooNew error: "package requires newer Go version +// go1.27 (application built with go1.26)", or "file requires …" for a file +// whose //go:build line names a newer release. +var tooNewRE = regexp.MustCompile(`requires newer Go version (go[0-9][0-9A-Za-z.\-]*)`) + +// toolchainGap collects the packages that failed to type-check because they +// need a newer Go than the one this analyzer was built with. go/types is +// compiled into the binary, so such a package is only partially typed and the +// analysis degrades silently (exit 0); warn records one stderr summary. +type toolchainGap struct { + pkgs map[string]bool + highest string +} + +// record notes pkg as affected when msg is a TooNew error, and reports +// whether it was one. +func (g *toolchainGap) record(pkg, msg string) bool { + m := tooNewRE.FindStringSubmatch(msg) + if m == nil { + return false + } + if g.pkgs == nil { + g.pkgs = map[string]bool{} + } + g.pkgs[pkg] = true + if g.highest == "" || version.Compare(m[1], g.highest) > 0 { + g.highest = m[1] + } + return true +} + +// warn prints the one consolidated warning, if any package was affected. +func (g *toolchainGap) warn() { + if len(g.pkgs) == 0 { + return + } + noun := "packages" + if len(g.pkgs) == 1 { + noun = "package" + } + utils.Warn("%d %s require a newer Go version (highest %s) than this analyzer was built with (%s); "+ + "their type information is partial and the analysis is degraded. "+ + "Rebuild codeanalyzer-go with %s or newer (the latest Go release is recommended); run with -vv to list them", + len(g.pkgs), noun, g.highest, runtime.Version(), g.highest) +} diff --git a/internal/syntactic_analysis/v2emit/determinism_freshbuild_test.go b/internal/syntactic_analysis/v2emit/determinism_freshbuild_test.go index 25fe1e3..f178a3e 100644 --- a/internal/syntactic_analysis/v2emit/determinism_freshbuild_test.go +++ b/internal/syntactic_analysis/v2emit/determinism_freshbuild_test.go @@ -63,8 +63,8 @@ func buildChiL2Fresh(t *testing.T, dir string) *v2.Analysis { } cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) edges := cg.Build(st) - app := &schema.GoApplication{SymbolTable: st, CallGraph: edges} - return Emit(app, "chi", dir, 2, "gate-test") + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()} + return Emit(app, "chi", dir, 2, "gate-test", 0, nil) } // TestDeterminism_FreshBuildsByteIdentical locks that independent from-disk @@ -151,3 +151,34 @@ func keysOf(m map[string]bool) []string { } return out } + +// TestDeterminism_ExternalsByteIdentical locks that the WHOLE L2 payload of chi, +// including application.external_symbols, is byte-identical across fresh builds, +// and that library edges are actually present (so the comparison is not vacuous). +func TestDeterminism_ExternalsByteIdentical(t *testing.T) { + dir := chiDir(t) + var first []byte + for i := 0; i < 4; i++ { + out := buildChiL2Fresh(t, dir) + raw, err := json.Marshal(out) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if i == 0 { + first = raw + ext := 0 + for _, e := range out.Application.CallGraph { + if _, ok := out.Application.ExternalSymbols[e.Dst]; ok { + ext++ + } + } + if ext == 0 || len(out.Application.ExternalSymbols) == 0 { + t.Fatalf("chi L2 has %d external edges / %d external symbols; want >0", ext, len(out.Application.ExternalSymbols)) + } + continue + } + if string(raw) != string(first) { + t.Fatalf("build %d differs from build 0 (output not byte-identical)", i) + } + } +} diff --git a/internal/syntactic_analysis/v2emit/determinism_test.go b/internal/syntactic_analysis/v2emit/determinism_test.go index a7c758a..b7d4d81 100644 --- a/internal/syntactic_analysis/v2emit/determinism_test.go +++ b/internal/syntactic_analysis/v2emit/determinism_test.go @@ -41,7 +41,7 @@ func TestDeterminism_CallGraphStableAcrossEmits(t *testing.T) { // edges slice without a stable sort, repeated marshals still match (same // slice), so to actually exercise order-independence we marshal a shuffled // copy of the edge slice too. - first := mustMarshalCallGraph(t, Emit(app, "multipackage", dir, 2, "det-test")) + first := mustMarshalCallGraph(t, Emit(app, "multipackage", dir, 2, "det-test", 0, nil)) // Reverse the input edge order and re-emit: a correct emitter sorts, so the // serialized call_graph must be unchanged. @@ -50,7 +50,7 @@ func TestDeterminism_CallGraphStableAcrossEmits(t *testing.T) { rev[len(edges)-1-i] = e } appRev := &schema.GoApplication{SymbolTable: st, CallGraph: rev} - second := mustMarshalCallGraph(t, Emit(appRev, "multipackage", dir, 2, "det-test")) + second := mustMarshalCallGraph(t, Emit(appRev, "multipackage", dir, 2, "det-test", 0, nil)) if first != second { t.Errorf("call_graph not order-independent: emitCallGraph must sort edges to a total order\nfirst=%s\nsecond=%s", first, second) diff --git a/internal/syntactic_analysis/v2emit/emit.go b/internal/syntactic_analysis/v2emit/emit.go index acfa2d1..888f8b6 100644 --- a/internal/syntactic_analysis/v2emit/emit.go +++ b/internal/syntactic_analysis/v2emit/emit.go @@ -6,6 +6,7 @@ import ( "sort" "strings" + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" "github.com/codellm-devkit/codeanalyzer-go/internal/schema" v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" ) @@ -13,18 +14,24 @@ import ( // Emit transforms the v1 GoApplication into the v2 Analysis payload. appName is // the application anchor (--app-name); projectDir is the absolute input root, // used to read each module's source for span slicing. maxLevel records how -// deeply the tree was populated (1 = symbol table, 2 = + call_graph); -// analyzerVersion is stamped into the manifest's analyzer{} tag. +// deeply the tree was populated (1 = symbol table, 2 = + call_graph, 3 = + +// cfg/cdg/ddg); analyzerVersion is stamped into the manifest's analyzer{} tag. // -// This is a pure re-serialization: every fact comes from the v1 model, except -// span byte offsets, which are derived from source via lineIndex. -func Emit(app *schema.GoApplication, appName, projectDir string, maxLevel int, analyzerVersion string) *v2.Analysis { +// kLimit is the L3 access-path bound (--graph-field-depth), written as the +// envelope's k_limit at maxLevel >= 3 only. l3 is dataflow.Run's side table, +// nil below L3; see emit_l3.go for how it merges into each callable. +// +// This is a pure re-serialization: every fact comes from the v1 model and the +// L3 side table, except span byte offsets, which are derived from source via +// lineIndex. +func Emit(app *schema.GoApplication, appName, projectDir string, maxLevel int, analyzerVersion string, kLimit int, l3 map[dataflow.L3Key]dataflow.Result) *v2.Analysis { appID := v2.AppID(appName) // The signature→can:// id index translates v1 identity (signature strings) // into v2 node ids. It is what the L2 refinement uses to backfill body // callees and to map call_graph edge endpoints onto real callable ids. idx := buildSigIndex(appID, app.SymbolTable) + idx.externals = externalIDs(appID, app.Externals) // One lineIndex per source file, read once and shared. A method whose // receiver type lives in a different file is nested under that type's module @@ -32,6 +39,7 @@ func Emit(app *schema.GoApplication, appName, projectDir string, maxLevel int, a // decisions → callable.source_file); the cache lets emitCallable reach any // file's index without re-reading it per callable. srcs := newSourceCache(projectDir) + flow := newL3Overlay(maxLevel, l3) out := &v2.Analysis{ SchemaVersion: v2.SchemaVersion, @@ -45,12 +53,16 @@ func Emit(app *schema.GoApplication, appName, projectDir string, maxLevel int, a CallGraph: emitCallGraph(app.CallGraph, idx), }, } + out.Application.ExternalSymbols = emitExternalSymbols(appID, app.Externals) + if flow.on { + out.KLimit = kLimit + } // Deterministic module order does not affect the JSON map, but iterating // sorted keeps any incidental ordering (and logs) stable across runs. for _, relPath := range sortedFileKeys(app.SymbolTable) { file := app.SymbolTable[relPath] - out.Application.SymbolTable[relPath] = emitModule(appID, relPath, file, idx, srcs) + out.Application.SymbolTable[relPath] = emitModule(appID, relPath, file, idx, srcs, flow) } return out } @@ -62,7 +74,7 @@ func Emit(app *schema.GoApplication, appName, projectDir string, maxLevel int, a // modRel is this module's own file path (a symbol_table key). It is threaded to // callables so a method declared in another file can be detected and given a // span against its own source plus a source_file pointer. -func emitModule(appID, modRel string, file schema.GoFile, idx sigIndex, srcs *sourceCache) v2.Module { +func emitModule(appID, modRel string, file schema.GoFile, idx sigIndex, srcs *sourceCache, flow l3Overlay) v2.Module { li := srcs.index(modRel) source := li.source modID := v2.ModuleID(appID, modRel) @@ -82,10 +94,10 @@ func emitModule(appID, modRel string, file schema.GoFile, idx sigIndex, srcs *so } for name, t := range file.Types { - mod.Types[name] = emitType(modID, modRel, t, idx, srcs) + mod.Types[name] = emitType(modID, modRel, t, idx, srcs, flow) } for sig, fn := range file.Functions { - mod.Functions[sig] = emitCallable(modID, modRel, li, fn, idx, srcs) + mod.Functions[sig] = emitCallable(modID, modRel, modRel, li, fn, idx, srcs, flow) } return mod } @@ -94,7 +106,7 @@ func emitModule(appID, modRel string, file schema.GoFile, idx sigIndex, srcs *so // kind and splitting embedding (base_types) from computed satisfaction. Methods // resolve into the type's callables{}. modRel is the module the type lives in; // its methods may be declared in other files (handled in emitCallable). -func emitType(modID, modRel string, t schema.GoType, idx sigIndex, srcs *sourceCache) v2.Type { +func emitType(modID, modRel string, t schema.GoType, idx sigIndex, srcs *sourceCache, flow l3Overlay) v2.Type { li := srcs.index(modRel) typeID := v2.TypeID(modID, t.Signature) out := v2.Type{ @@ -116,7 +128,7 @@ func emitType(modID, modRel string, t schema.GoType, idx sigIndex, srcs *sourceC } } for sig, m := range t.Methods { - out.Callables[sig] = emitCallable(typeID, modRel, li, m, idx, srcs) + out.Callables[sig] = emitCallable(typeID, modRel, modRel, li, m, idx, srcs, flow) } return out } @@ -124,21 +136,37 @@ func emitType(modID, modRel string, t schema.GoType, idx sigIndex, srcs *sourceC // emitCallable maps a v1 GoCallable to a v2 callable node, including its body // (call nodes) and nested closures. // -// modRel and modLI are the nesting module's path and lineIndex. When the -// callable's declaring file (c.Path) differs — a method whose receiver type is -// in another file — its span is computed against its OWN file's lineIndex and -// source_file records that file, per CLAUDE.md § Schema decisions. Nested -// closures share their enclosing callable's file (Go has no cross-file -// closures), so they carry the resolved file down unchanged. -func emitCallable(parentID, modRel string, modLI *lineIndex, c schema.GoCallable, idx sigIndex, srcs *sourceCache) v2.Callable { +// modRel is the CONTAINING module's path: the module this callable nests +// under by containment (a method's receiver-type module, and for a closure the +// module its outermost enclosing callable nests under). declRel and declLI are +// the file the callable's positions are relative to and its lineIndex: the +// nesting file for a top-level callable, the enclosing callable's declaring +// file for a closure (Go has no cross-file closures; v1 closures carry no +// Path). A callable's own c.Path, when set, is its declaring file. +// +// When the declaring file differs from the containing module — a method whose +// receiver type is in another file, and every closure (nested ones included) +// inside such a method — the callable records it as source_file, per CLAUDE.md +// § Schema decisions, and its spans index that file's source. Under the SDK's +// text-recovery rule (symbol_table[source_file].source when present, else the +// containing module's source) each of these nodes then slices the right file. +// +// At L3 the callable's dataflow result, looked up by (declaring file, +// signature), completes its body and adds cfg/cdg/ddg (emit_l3.go). Its spans +// use the same declaring-file lineIndex, so a cross-file callable's statement +// spans index its declaring file's source, like its own span. +func emitCallable(parentID, modRel, declRel string, declLI *lineIndex, c schema.GoCallable, idx sigIndex, srcs *sourceCache, flow l3Overlay) v2.Callable { callID := v2.CallableID(parentID, c.Signature) // Resolve the file this callable's positions are relative to. - li := modLI - sourceFile := "" - if c.Path != "" && c.Path != modRel { + li := declLI + if c.Path != "" && c.Path != declRel { li = srcs.index(c.Path) - sourceFile = c.Path + declRel = c.Path + } + sourceFile := "" + if declRel != modRel { + sourceFile = declRel } out := v2.Callable{ @@ -159,57 +187,101 @@ func emitCallable(parentID, modRel string, modLI *lineIndex, c schema.GoCallable } if len(c.InnerCallables) > 0 { out.Callables = make(map[string]v2.Callable, len(c.InnerCallables)) - // Closures live in the same file as their enclosing callable, so the - // resolved file (modRel-or-c.Path) becomes their nesting file. - enclRel := modRel - if sourceFile != "" { - enclRel = sourceFile - } + // Closures nest in the same containing module and are declared in the + // same file as their enclosing callable. for sig, inner := range c.InnerCallables { - out.Callables[sig] = emitCallable(callID, enclRel, li, inner, idx, srcs) + out.Callables[sig] = emitCallable(callID, modRel, declRel, li, inner, idx, srcs, flow) } } + flow.apply(&out, li, dataflow.L3Key{Path: declRel, Signature: c.Signature}) return out } // emitBody builds the body map: one `call` node per call site, keyed by its // line:col local id, carrying the sanctioned callee null->id refinement slot. +// Every node carries its global id, v2.GlobalOrdinal(callable id, key), from L1 +// on (R15; Python parity: codeanalyzer-python stamps body ids at L1). // // callee resolution: // - L1: the v1 site has no backfilled signature, so callee stays null. // - L2: the resolver backfilled the site's callee signature. When that // signature names an in-tree callable, callee becomes its can:// id; when -// it names an external/stdlib callee (no in-tree node), callee stays null — -// the honest-unresolved fallback the L2 gate expects. +// it names a library callee, callee becomes its @external id (a key of +// application.external_symbols); otherwise (conversions, builtins, calls +// through function values) callee stays null. func emitBody(callableID string, li *lineIndex, sites []schema.GoCallsite, idx sigIndex) map[string]v2.BodyNode { + keys := callBodyKeys(sites) body := make(map[string]v2.BodyNode, len(sites)) - for _, cs := range sites { - local := v2.LocalID(cs.StartLine, cs.StartColumn) + for i, cs := range sites { + local := keys[i] body[local] = v2.BodyNode{ Kind: "call", + ID: v2.GlobalOrdinal(callableID, local), Span: colSpan(li, cs.StartLine, cs.StartColumn, cs.EndLine, cs.EndColumn), Callee: calleeID(cs.CalleeSignature, idx), IsGoroutine: cs.IsGoroutine, + IsDeferred: cs.IsDeferred, } } return body } +// callBodyKeys maps each call site to its body-map key. It is a thin wrapper: +// the co-positioned-call keying (l:c, l:c/2, …) lives in v2.CallBodyKeys so the +// L3 body builder shares it. +func callBodyKeys(sites []schema.GoCallsite) []string { + pos := make([][2]int, len(sites)) + for i, cs := range sites { + pos[i] = [2]int{cs.StartLine, cs.StartColumn} + } + return v2.CallBodyKeys(pos) +} + // calleeID translates a v1 backfilled callee signature into the v2 can:// id -// refinement slot: nil (JSON null) when unresolved or external, a pointer to the -// in-tree callable id when resolved. Never returns the raw v1 signature — a -// call node's callee is always a node id or null. +// refinement slot: nil (JSON null) when unresolved, a pointer to the in-tree +// callable id or the library callee's @external id when resolved. Never returns +// the raw v1 signature — a call node's callee is always a node id or null. func calleeID(sig *string, idx sigIndex) *string { if sig == nil { return nil } - id, ok := idx.idFor(*sig) + id, ok := idx.resolve(*sig) if !ok { return nil } return &id } +// externalIDs maps each library callee's full signature to its @external id. +func externalIDs(appID string, externals map[string]schema.GoExternalCallee) map[string]string { + ids := make(map[string]string, len(externals)) + for fullSig, ext := range externals { + ids[fullSig] = v2.ExternalID(appID, ext.ImportPath, ext.Sig) + } + return ids +} + +// emitExternalSymbols builds application.external_symbols, keyed by @external +// id. Nil (omitted from JSON) when no library callee was recorded. +func emitExternalSymbols(appID string, externals map[string]schema.GoExternalCallee) map[string]v2.ExternalSymbol { + if len(externals) == 0 { + return nil + } + out := make(map[string]v2.ExternalSymbol, len(externals)) + for _, ext := range externals { + id := v2.ExternalID(appID, ext.ImportPath, ext.Sig) + params := make([]v2.ExternalParameter, 0, len(ext.Params)) + for _, p := range ext.Params { + params = append(params, v2.ExternalParameter{Name: p.Name, Type: p.Type, IsVariadic: p.IsVariadic}) + } + out[id] = v2.ExternalSymbol{ + ID: id, Kind: "external", Name: ext.Name, Module: ext.ImportPath, + Signature: ext.Sig, Parameters: params, ReturnType: ext.ReturnType, + } + } + return out +} + func emitParams(li *lineIndex, params []schema.GoParameter) []v2.Parameter { out := make([]v2.Parameter, 0, len(params)) for _, p := range params { @@ -238,10 +310,10 @@ func emitImports(li *lineIndex, imports []schema.GoImport) []v2.Import { // emitCallGraph maps v1 identity-only edges to the v2 {src,dst,prov,weight} // shape: the source/target/provenance rename, PLUS the endpoint translation from -// v1 signatures to can:// callable ids. An edge whose src or dst does not resolve -// to an in-tree node is dropped rather than emitted with a dangling endpoint — -// the L2 gate requires every endpoint to be a real callable id. (The v1 resolver -// only emits edges to in-project targets, so this drop is defensive.) +// v1 signatures to can:// ids. A dst may be an in-tree callable or a library +// callee's @external id (a key of external_symbols). An edge whose src or dst +// does not resolve is dropped rather than emitted with a dangling endpoint — +// the L2 gate requires every endpoint to be a real node id. func emitCallGraph(edges []schema.GoCallEdge, idx sigIndex) []v2.Edge { out := make([]v2.Edge, 0, len(edges)) for _, e := range edges { @@ -251,7 +323,7 @@ func emitCallGraph(edges []schema.GoCallEdge, idx sigIndex) []v2.Edge { // The target stays signature-only — a call target is never an init // (Go forbids calling init), and every other signature is unique. src, srcOK := idx.idForWithPath(e.Source, e.SourcePath) - dst, dstOK := idx.idFor(e.Target) + dst, dstOK := idx.resolve(e.Target) if !srcOK || !dstOK { continue } @@ -289,10 +361,13 @@ func emitCallGraph(edges []schema.GoCallEdge, idx sigIndex) []v2.Edge { // ─── kind mapping ──────────────────────────────────────────────────────────── -// typeKind collapses the v1 is_interface boolean into the v2 kind vocabulary. -// v1 only distinguishes struct vs interface; alias/defined await richer v1 -// facts, so a non-interface type maps to struct for now. +// typeKind reports the v2 kind (struct | interface | alias | defined), recorded +// by the symbol-table builder from the type declaration. The IsInterface +// fallback covers hand-built v1 types that carry no TypeKind. func typeKind(t schema.GoType) string { + if t.TypeKind != "" { + return t.TypeKind + } if t.IsInterface { return "interface" } diff --git a/internal/syntactic_analysis/v2emit/emit_l3.go b/internal/syntactic_analysis/v2emit/emit_l3.go new file mode 100644 index 0000000..3805138 --- /dev/null +++ b/internal/syntactic_analysis/v2emit/emit_l3.go @@ -0,0 +1,74 @@ +package v2emit + +import ( + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" + v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" +) + +// l3Overlay merges dataflow.Run's side table into the emitted tree. on is +// maxLevel >= 3; below that the overlay is inert and the tree is exactly the +// L1/L2 one (call nodes with their ids, no parent, no edge lists). +type l3Overlay struct { + on bool + res map[dataflow.L3Key]dataflow.Result +} + +func newL3Overlay(maxLevel int, res map[dataflow.L3Key]dataflow.Result) l3Overlay { + return l3Overlay{on: maxLevel >= 3, res: res} +} + +// apply completes c's body from the result keyed by key (the callable's +// declaring file and signature) and copies its edges. li is the declaring +// file's lineIndex, so every span indexes the file c's own span indexes. +// +// The merge is additive over the L1 body: +// - every L1 call node is kept unchanged — kind, id, span (the CallExpr's, +// R7), callee, flags — even one the CFG pruned as unreachable (it simply +// has no edges); a result node on an L1 key only contributes its Parent; +// - every other result node (the pruned set's statement, control and +// synthetic nodes) is added with its span from Pos/End and its ID = +// v2.GlobalOrdinal(c.ID, key), the rule emitBody applies to the L1 nodes +// (R15), so the id field is uniform at every level. +// +// Run works on local keys only; ids are minted here and in emitBody. +func (o l3Overlay) apply(c *v2.Callable, li *lineIndex, key dataflow.L3Key) { + if !o.on { + return + } + if r, ok := o.res[key]; ok { + for _, n := range r.Nodes { + if bn, ok := c.Body[n.Key]; ok { + bn.Parent = n.Parent + c.Body[n.Key] = bn + continue + } + bn := l3BodyNode(n, li) + bn.ID = v2.GlobalOrdinal(c.ID, n.Key) + c.Body[n.Key] = bn + } + c.Cfg = make([]v2.CfgEdge, len(r.Cfg)) + for i, e := range r.Cfg { + c.Cfg[i] = v2.CfgEdge{Src: e.Src, Dst: e.Dst, Kind: e.Kind} + } + c.Cdg = make([]v2.CdgEdge, len(r.Cdg)) + for i, e := range r.Cdg { + c.Cdg[i] = v2.CdgEdge{Src: e.Src, Dst: e.Dst} + } + c.Ddg = make([]v2.DdgEdge, len(r.Ddg)) + for i, e := range r.Ddg { + c.Ddg[i] = v2.DdgEdge{Src: e.Src, Dst: e.Dst, Var: e.Var, Prov: []string{"ssa"}} // own slice per edge: no aliasing + } + } +} + +// l3BodyNode is a non-L1 body node: its kind, and its span from the node's +// [line, col] range (zero for @entry/@exit). A call node reaching here is a +// call L1 did not key — impossible when Run's count check passed — and keeps +// the call shape with a null callee. +func l3BodyNode(n dataflow.Node, li *lineIndex) v2.BodyNode { + bn := v2.BodyNode{Kind: string(n.Kind), Parent: n.Parent} + if n.Kind != dataflow.KindEntry && n.Kind != dataflow.KindExit { + bn.Span = colSpan(li, n.StartLC[0], n.StartLC[1], n.EndLC[0], n.EndLC[1]) + } + return bn +} diff --git a/internal/syntactic_analysis/v2emit/emit_test.go b/internal/syntactic_analysis/v2emit/emit_test.go index 6dd39ee..c5ae286 100644 --- a/internal/syntactic_analysis/v2emit/emit_test.go +++ b/internal/syntactic_analysis/v2emit/emit_test.go @@ -37,7 +37,7 @@ func buildGreeter(t *testing.T) (*schema.GoApplication, string) { func TestEmit_Envelope(t *testing.T) { app, dir := buildGreeter(t) - out := Emit(app, "greeter", dir, 1, "test") + out := Emit(app, "greeter", dir, 1, "test", 0, nil) if out.SchemaVersion != "2.0.0" { t.Errorf("schema_version = %q, want 2.0.0", out.SchemaVersion) @@ -48,8 +48,8 @@ func TestEmit_Envelope(t *testing.T) { if out.MaxLevel != 1 { t.Errorf("max_level = %d, want 1", out.MaxLevel) } - if out.Application.ID != "can://go/greeter" { - t.Errorf("application.id = %q, want can://go/greeter", out.Application.ID) + if out.Application.ID != "can://greeter" { + t.Errorf("application.id = %q, want can://greeter", out.Application.ID) } if out.Application.Kind != "application" { t.Errorf("application.kind = %q, want application", out.Application.Kind) @@ -61,7 +61,7 @@ func TestEmit_Envelope(t *testing.T) { func TestEmit_ModuleSourceAndIDs(t *testing.T) { app, dir := buildGreeter(t) - out := Emit(app, "greeter", dir, 1, "test") + out := Emit(app, "greeter", dir, 1, "test", 0, nil) mod, ok := out.Application.SymbolTable["main.go"] if !ok { @@ -70,8 +70,8 @@ func TestEmit_ModuleSourceAndIDs(t *testing.T) { if mod.Kind != "module" { t.Errorf("module.kind = %q, want module", mod.Kind) } - if mod.ID != "can://go/greeter/main.go" { - t.Errorf("module.id = %q, want can://go/greeter/main.go", mod.ID) + if mod.ID != "can://greeter/go/main.go" { + t.Errorf("module.id = %q, want can://greeter/go/main.go", mod.ID) } if !strings.HasPrefix(mod.Source, "package main") { t.Errorf("module.source should hold the file text; got prefix %q", head(mod.Source, 20)) @@ -84,7 +84,7 @@ func TestEmit_ModuleSourceAndIDs(t *testing.T) { func TestEmit_CallableBodyAndSpanSlice(t *testing.T) { app, dir := buildGreeter(t) - out := Emit(app, "greeter", dir, 1, "test") + out := Emit(app, "greeter", dir, 1, "test", 0, nil) mod := out.Application.SymbolTable["main.go"] var main v2.Callable @@ -123,7 +123,7 @@ func TestEmit_CallableBodyAndSpanSlice(t *testing.T) { func TestEmit_CalleeSerializesAsNull(t *testing.T) { app, dir := buildGreeter(t) - out := Emit(app, "greeter", dir, 1, "test") + out := Emit(app, "greeter", dir, 1, "test", 0, nil) data, err := json.Marshal(out) if err != nil { t.Fatalf("marshal failed: %v", err) @@ -144,7 +144,7 @@ func TestEmit_CalleeSerializesAsNull(t *testing.T) { func TestEmit_TypeKindAndMethods(t *testing.T) { app, dir := buildGreeter(t) - out := Emit(app, "greeter", dir, 1, "test") + out := Emit(app, "greeter", dir, 1, "test", 0, nil) // Find the Greeter type wherever it lives; assert struct kind + methods. var found bool diff --git a/internal/syntactic_analysis/v2emit/gate_l2_inits_test.go b/internal/syntactic_analysis/v2emit/gate_l2_inits_test.go new file mode 100644 index 0000000..529329c --- /dev/null +++ b/internal/syntactic_analysis/v2emit/gate_l2_inits_test.go @@ -0,0 +1,85 @@ +package v2emit + +// Repeated `.init` gate. Go allows a func init() in every file of a +// package, and all of them share the signature ".init". The resolver must +// keep each init's own call edges: when two inits call the same callee, both +// edges survive, and the result does not depend on map iteration order. +// Fixture: testdata/dataflow/inits (a.go and b.go each declare init(), both +// call the in-package setup() and the library os.Getenv). + +import ( + "bytes" + "encoding/json" + "path/filepath" + "runtime" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/schema" + v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" + "github.com/codellm-devkit/codeanalyzer-go/internal/semantic_analysis" + "github.com/codellm-devkit/codeanalyzer-go/internal/syntactic_analysis" +) + +// buildDataflowL2Fresh runs an independent L2 analysis of testdata/dataflow +// (new symbol table, resolver and externals), so map order varies per call. +func buildDataflowL2Fresh(t *testing.T) *v2.Analysis { + t.Helper() + _, thisFile, _, _ := runtime.Caller(0) + dir, err := filepath.Abs(filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "testdata", "dataflow")) + if err != nil { + t.Fatalf("resolving fixture dir: %v", err) + } + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("symbol table build failed: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()} + return Emit(app, "dataflow", dir, 2, "gate-test", 0, nil) +} + +// TestGateL2_EveryInitKeepsItsEdges: each file's init has its own edge to the +// shared in-package callee and to the shared library callee. +func TestGateL2_EveryInitKeepsItsEdges(t *testing.T) { + out := buildDataflowL2Fresh(t) + const initSig = "example.com/dataflow/inits.init" + setup := functionByName(t, mustModule(t, out, "inits/a.go"), "setup").ID + getenv := v2.ExternalID(out.Application.ID, "os", "Getenv") + + has := map[[2]string]bool{} + for _, e := range out.Application.CallGraph { + has[[2]string{e.Src, e.Dst}] = true + } + for _, rel := range []string{"inits/a.go", "inits/b.go"} { + fn, ok := mustModule(t, out, rel).Functions[initSig] + if !ok { + t.Fatalf("%s: init missing", rel) + } + for _, dst := range []string{setup, getenv} { + if !has[[2]string{fn.ID, dst}] { + t.Errorf("missing call_graph edge %s -> %s", fn.ID, dst) + } + } + } +} + +// TestGateL2_InitsFreshBuildsByteIdentical: 10 fresh builds of the dataflow +// fixture serialize to byte-identical JSON. +func TestGateL2_InitsFreshBuildsByteIdentical(t *testing.T) { + var want []byte + for i := 0; i < 10; i++ { + got, err := json.Marshal(buildDataflowL2Fresh(t)) + if err != nil { + t.Fatalf("build %d: marshal: %v", i, err) + } + if i == 0 { + want = got + continue + } + if !bytes.Equal(got, want) { + t.Fatalf("build %d differs from build 0", i) + } + } +} diff --git a/internal/syntactic_analysis/v2emit/gate_l2_test.go b/internal/syntactic_analysis/v2emit/gate_l2_test.go index 8a50883..5e46baf 100644 --- a/internal/syntactic_analysis/v2emit/gate_l2_test.go +++ b/internal/syntactic_analysis/v2emit/gate_l2_test.go @@ -16,6 +16,11 @@ package v2emit // - the L1 ⊆ L2 superset holds: the tree is unchanged except the callee fill. import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" "testing" "github.com/codellm-devkit/codeanalyzer-go/internal/schema" @@ -36,8 +41,8 @@ func buildMultipackageL2(t *testing.T) *v2.Analysis { } cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) edges := cg.Build(st) // mutates st in place: backfills callee_signature - app := &schema.GoApplication{SymbolTable: st, CallGraph: edges} - return Emit(app, "multipackage", dir, 2, "gate-test") + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()} + return Emit(app, "multipackage", dir, 2, "gate-test", 0, nil) } // allCallableIDs collects every callable/type id in the emitted tree. @@ -80,6 +85,10 @@ func TestGateL2_MaxLevelAndEdgesPresent(t *testing.T) { func TestGateL2_NoDanglingEndpoints(t *testing.T) { out := buildMultipackageL2(t) ids := allCallableIDs(out) + // A library callee is a real node too: its id is a key of external_symbols. + for id := range out.Application.ExternalSymbols { + ids[id] = true + } for i, e := range out.Application.CallGraph { if !ids[e.Src] { @@ -101,8 +110,8 @@ func TestGateL2_NamedEdgePresent(t *testing.T) { out := buildMultipackageL2(t) const ( - src = "can://go/multipackage/worker/worker.go/example.com/multipackage/worker.Worker/example.com/multipackage/worker.Worker.Run" - dst = "can://go/multipackage/worker/worker.go/example.com/multipackage/worker.Worker/example.com/multipackage/worker.Worker.execute" + src = "can://multipackage/go/worker/worker.go/example.com/multipackage/worker.Worker/example.com/multipackage/worker.Worker.Run" + dst = "can://multipackage/go/worker/worker.go/example.com/multipackage/worker.Worker/example.com/multipackage/worker.Worker.execute" ) found := false for _, e := range out.Application.CallGraph { @@ -124,8 +133,8 @@ func TestGateL2_CrossPackageEdge(t *testing.T) { out := buildMultipackageL2(t) const ( - src = "can://go/multipackage/main.go/example.com/multipackage.main" - dst = "can://go/multipackage/server/server.go/example.com/multipackage/server.New" + src = "can://multipackage/go/main.go/example.com/multipackage.main" + dst = "can://multipackage/go/server/server.go/example.com/multipackage/server.New" ) found := false for _, e := range out.Application.CallGraph { @@ -159,8 +168,8 @@ func TestGateL2_CalleeBackfilled(t *testing.T) { if node.Callee == nil { continue } - if !ids[*node.Callee] { - t.Errorf("body[%s].callee = %q is not an in-tree callable id", local, *node.Callee) + if _, ext := out.Application.ExternalSymbols[*node.Callee]; !ids[*node.Callee] && !ext { + t.Errorf("body[%s].callee = %q is neither an in-tree callable nor an external symbol", local, *node.Callee) } sawBackfilled = true } @@ -180,12 +189,12 @@ func TestGateL2_SupersetOfL1(t *testing.T) { t.Fatalf("symbol table build failed: %v", err) } // L1 snapshot BEFORE the resolver mutates call sites. - l1 := Emit(&schema.GoApplication{SymbolTable: st, CallGraph: nil}, "multipackage", dir, 1, "gate-test") + l1 := Emit(&schema.GoApplication{SymbolTable: st, CallGraph: nil}, "multipackage", dir, 1, "gate-test", 0, nil) // Now run the resolver (mutates st) and emit L2. cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) edges := cg.Build(st) - l2 := Emit(&schema.GoApplication{SymbolTable: st, CallGraph: edges}, "multipackage", dir, 2, "gate-test") + l2 := Emit(&schema.GoApplication{SymbolTable: st, CallGraph: edges}, "multipackage", dir, 2, "gate-test", 0, nil) if len(l1.Application.SymbolTable) != len(l2.Application.SymbolTable) { t.Fatalf("module count changed L1->L2: %d vs %d", @@ -241,3 +250,314 @@ func assertCallablesEqualModuloCallee(t *testing.T, where string, a, b map[strin assertCallablesEqualModuloCallee(t, where+" "+sig, c1.Callables, c2.Callables) } } + +func anyCallNodeHasCallee(out *v2.Analysis, id string) bool { + found := false + var walk func(c v2.Callable) + walk = func(c v2.Callable) { + for _, n := range c.Body { + if n.Callee != nil && *n.Callee == id { + found = true + } + } + for _, inner := range c.Callables { + walk(inner) + } + } + for _, mod := range out.Application.SymbolTable { + for _, fn := range mod.Functions { + walk(fn) + } + for _, ty := range mod.Types { + for _, m := range ty.Callables { + walk(m) + } + } + } + return found +} + +func anyEdgeTo(edges []v2.Edge, dst string) bool { + for _, e := range edges { + if e.Dst == dst { + return true + } + } + return false +} + +// TestGateL2_ExternalCalleeBackfilled locks that a call into a library keeps its +// edge: multipackage/main.go calls fmt.Println, so external_symbols carries it, +// the call node's callee is its @external id, and a call_graph edge targets it. +func TestGateL2_ExternalCalleeBackfilled(t *testing.T) { + out := buildMultipackageL2(t) + ext := v2.ExternalID(out.Application.ID, "fmt", "Println") + + sym, ok := out.Application.ExternalSymbols[ext] + if !ok { + t.Fatalf("external_symbols missing %s", ext) + } + want := v2.ExternalSymbol{ID: ext, Kind: "external", Name: "Println", Module: "fmt", Signature: "Println"} + if sym.ID != want.ID || sym.Kind != want.Kind || sym.Name != want.Name || + sym.Module != want.Module || sym.Signature != want.Signature { + t.Errorf("external symbol = %+v, want identity fields %+v", sym, want) + } + if len(sym.Parameters) != 1 || sym.Parameters[0].Name != "a" || !sym.Parameters[0].IsVariadic { + t.Errorf("Println parameters = %+v, want one variadic param named a", sym.Parameters) + } + if !anyCallNodeHasCallee(out, ext) { + t.Errorf("no call node has callee %s", ext) + } + if !anyEdgeTo(out.Application.CallGraph, ext) { + t.Errorf("no call_graph edge targets %s", ext) + } +} + +// TestGateL2_ExternalMethodSignature locks the method form of the sig segment: +// strings.Builder.WriteString style -- Type.Method, receiver not a parameter. +func TestGateL2_ExternalMethodSignature(t *testing.T) { + out := emitTemp(t, `package p + +import "strings" + +func F() string { + var sb strings.Builder + sb.WriteString("x") + return sb.String() +} +`) + ext := v2.ExternalID(out.Application.ID, "strings", "Builder.WriteString") + sym, ok := out.Application.ExternalSymbols[ext] + if !ok { + t.Fatalf("external_symbols missing %s; have %v", ext, out.Application.ExternalSymbols) + } + if sym.Name != "WriteString" || sym.Signature != "Builder.WriteString" || sym.Module != "strings" { + t.Errorf("unexpected symbol %+v", sym) + } + if len(sym.Parameters) != 1 || sym.Parameters[0].Name != "s" { + t.Errorf("params = %+v, want exactly [s] (receiver excluded)", sym.Parameters) + } +} + +// TestGateL2_ConversionIsNotACall locks that a type conversion is not minted as +// a call: its call node keeps callee null and no "__new__" appears anywhere. +func TestGateL2_ConversionIsNotACall(t *testing.T) { + out := emitTemp(t, `package p + +func F(b []byte) string { + return string(b) +} +`) + fn := out.Application.SymbolTable["p.go"].Functions + if len(fn) != 1 { + t.Fatalf("want 1 function, got %d", len(fn)) + } + for _, c := range fn { + if len(c.Body) != 1 { + t.Fatalf("want 1 call node (the conversion), got %d", len(c.Body)) + } + for _, n := range c.Body { + if n.Callee != nil { + t.Errorf("conversion callee = %q, want null", *n.Callee) + } + } + } + raw, _ := json.Marshal(out) + if strings.Contains(string(raw), "__new__") { + t.Error("output contains __new__") + } + if len(out.Application.ExternalSymbols) != 0 { + t.Errorf("conversion minted external symbols: %v", out.Application.ExternalSymbols) + } +} + +// emitTemp builds a one-file module (p.go) in a temp dir through the real +// builder + resolver and emits v2 at L2. +func emitTemp(t *testing.T, src string) *v2.Analysis { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module example.com/p\n\ngo 1.21\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "p.go"), []byte(src), 0o644); err != nil { + t.Fatal(err) + } + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("symbol table build failed: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()} + return Emit(app, "p", dir, 2, "gate-test", 0, nil) +} + +// TestGateL2_ExternalParamsHaveNoSpan locks that external parameters serialize +// without a span key (no fabricated location). +func TestGateL2_ExternalParamsHaveNoSpan(t *testing.T) { + out := buildMultipackageL2(t) + ext := v2.ExternalID(out.Application.ID, "fmt", "Println") + raw, err := json.Marshal(out.Application.ExternalSymbols[ext]) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "span") { + t.Errorf("external symbol JSON carries a span: %s", raw) + } +} + +// TestGateL2_GenericLibraryMethodUsesDeclaredSignature locks that methods on two +// instantiations of one generic library type (atomic.Pointer[A], [B], called in +// two files) collapse to ONE external whose params are the declared form (*T), +// independent of which instance the resolver visits first. +func TestGateL2_GenericLibraryMethodUsesDeclaredSignature(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "testdata", "dataflow") + for i := 0; i < 6; i++ { // fresh builds: map order varies + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("build: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + out := Emit(&schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()}, "dataflow", dir, 2, "gate-test", 0, nil) + ext := v2.ExternalID(out.Application.ID, "sync/atomic", "Pointer.Store") + sym, ok := out.Application.ExternalSymbols[ext] + if !ok { + t.Fatalf("missing %s; have %v", ext, out.Application.ExternalSymbols) + } + if len(sym.Parameters) != 1 || sym.Parameters[0].Type != "*T" { + t.Fatalf("Pointer.Store params = %+v, want one param of type *T", sym.Parameters) + } + } +} + +// TestGateL2_IncrementalModeKeepsProjectCalleesInternal locks that with +// TargetFiles (sibling project packages load only as deps) calls into the project +// are never minted as @external symbols named after the project's own module. +func TestGateL2_IncrementalModeKeepsProjectCalleesInternal(t *testing.T) { + dir := multipackageDir(t) + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build([]string{filepath.Join(dir, "main.go")}, true) + if err != nil { + t.Fatalf("build: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + out := Emit(&schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()}, "multipackage", dir, 2, "gate-test", 0, nil) + + const projectModule = "example.com/multipackage" + for id, sym := range out.Application.ExternalSymbols { + if sym.Module == projectModule || strings.HasPrefix(sym.Module, projectModule+"/") { + t.Errorf("project callee minted as external: %s (module %s)", id, sym.Module) + } + } + if _, ok := out.Application.ExternalSymbols[v2.ExternalID(out.Application.ID, "fmt", "Println")]; !ok { + t.Error("fmt.Println should still be external in incremental mode") + } +} + +// TestGateL2_VendoredCalleeIsExternal locks that a callee declared under +// /vendor (inside the project dir, but skipped by the symbol table) +// is a library: dataflow/vend calls vlib.F from the vendored example.com/vlib. +func TestGateL2_VendoredCalleeIsExternal(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "testdata", "dataflow") + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("build: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + out := Emit(&schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()}, "dataflow", dir, 2, "gate-test", 0, nil) + + ext := v2.ExternalID(out.Application.ID, "example.com/vlib", "F") + sym, ok := out.Application.ExternalSymbols[ext] + if !ok { + t.Fatalf("external_symbols missing %s; have %v", ext, out.Application.ExternalSymbols) + } + if sym.Module != "example.com/vlib" || sym.Signature != "F" { + t.Errorf("unexpected symbol %+v", sym) + } + if !anyCallNodeHasCallee(out, ext) { + t.Errorf("no call node has callee %s", ext) + } + if !anyEdgeTo(out.Application.CallGraph, ext) { + t.Errorf("no call_graph edge targets %s", ext) + } +} + +// TestGateL2_LineDirectiveCallsResolve (I1): L1 records call sites through +// fset.Position, which applies //line directives, so a site under a directive +// names a virtual line:col (here line 900 of a 17-line file). The resolver +// must find it by that same key and backfill its callee and edge. +func TestGateL2_LineDirectiveCallsResolve(t *testing.T) { + out := buildDataflowL2Fresh(t) + fn := functionByName(t, mustModule(t, out, "linedir/linedir.go"), "Caller") + helper := functionByName(t, mustModule(t, out, "linedir/linedir.go"), "helper").ID + getenv := v2.ExternalID(out.Application.ID, "os", "Getenv") + + callees := map[string]bool{} + for k, n := range fn.Body { + if n.Kind != "call" { + continue + } + if n.Callee == nil { + t.Errorf("call node %s has a null callee", k) + continue + } + callees[*n.Callee] = true + } + if _, virtual := fn.Body["900:2"]; !virtual { + t.Fatalf("no call node at the virtual 900:2; body %v", keys(fn.Body)) + } + edges := map[string]bool{} + for _, e := range out.Application.CallGraph { + if e.Src == fn.ID { + edges[e.Dst] = true + } + } + for _, dst := range []string{helper, getenv} { + if !callees[dst] { + t.Errorf("no call node of Caller has callee %s", dst) + } + if !edges[dst] { + t.Errorf("missing call_graph edge %s -> %s", fn.ID, dst) + } + } +} + +// TestGateL2_ExternalUnnamedParamsArePositional: a library callee's unnamed +// or blank parameters are named by position, _, in external_symbols +// (vlib.G(int, string), vlib.H(_ int, n string)). +func TestGateL2_ExternalUnnamedParamsArePositional(t *testing.T) { + _, thisFile, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "testdata", "dataflow") + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("build: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + out := Emit(&schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()}, "dataflow", dir, 2, "gate-test", 0, nil) + + for sig, want := range map[string][]string{"G": {"_0", "_1"}, "H": {"_0", "n"}} { + ext := v2.ExternalID(out.Application.ID, "example.com/vlib", sig) + sym, ok := out.Application.ExternalSymbols[ext] + if !ok { + t.Errorf("external_symbols missing %s", ext) + continue + } + var got []string + for _, p := range sym.Parameters { + got = append(got, p.Name) + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("%s parameter names = %v, want %v", sig, got, want) + } + } +} diff --git a/internal/syntactic_analysis/v2emit/gate_l3_test.go b/internal/syntactic_analysis/v2emit/gate_l3_test.go new file mode 100644 index 0000000..8f6a976 --- /dev/null +++ b/internal/syntactic_analysis/v2emit/gate_l3_test.go @@ -0,0 +1,466 @@ +package v2emit + +// L3 conformance gate for the v2 emitter. +// +// L3 completes each callable's body (statement / branch / loop / switch / +// return / entry / exit nodes around the L1 call nodes) and adds the +// intra-callable cfg / cdg / ddg edge lists. These gates build a fixture +// through the real symbol-table builder, resolver and dataflow driver (the +// -a 3 wiring) and check: +// +// - every callable with a body has @entry / @exit and a non-empty cfg +// - no dangling endpoint: every cfg / cdg / ddg endpoint and every parent +// is a key of the callable's own body; ddg prov is ["ssa"] +// - every body node id is v2.GlobalOrdinal(callable id, key) +// - json(-a 2) ⊆ json(-a 3), modulo the sanctioned additions +// - spans slice to the statement text, cross-file methods included +// - every side-table entry lands on an emitted callable + +import ( + "encoding/json" + "fmt" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "testing" + + "github.com/codellm-devkit/codeanalyzer-go/internal/dataflow" + "github.com/codellm-devkit/codeanalyzer-go/internal/schema" + v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" + "github.com/codellm-devkit/codeanalyzer-go/internal/semantic_analysis" + "github.com/codellm-devkit/codeanalyzer-go/internal/syntactic_analysis" +) + +type l3Fixture struct { + st map[string]schema.GoFile + l2, l3 *v2.Analysis + res map[dataflow.L3Key]dataflow.Result +} + +var ( + l3Mu sync.Mutex + l3Cache = map[string]*l3Fixture{} +) + +// buildL3 runs the -a 3 pipeline over testdata/ once and emits both the +// L2 and the L3 payload from the same resolved symbol table. +func buildL3(t *testing.T, name string) *l3Fixture { + t.Helper() + l3Mu.Lock() + defer l3Mu.Unlock() + if fx, ok := l3Cache[name]; ok { + return fx + } + _, thisFile, _, _ := runtime.Caller(0) + dir, err := filepath.Abs(filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "testdata", name)) + if err != nil { + t.Fatal(err) + } + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("symbol table build failed: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges, Externals: cg.Externals()} + res := dataflow.Run(b.Fset(), b.L3Units(st), 3, 4) + fx := &l3Fixture{ + st: st, + l2: Emit(app, name, dir, 2, "gate-test", 3, nil), + l3: Emit(app, name, dir, 3, "gate-test", 3, res), + res: res, + } + l3Cache[name] = fx + return fx +} + +// emitted is one callable of the v2 tree with its v1 source and the file its +// spans index. +type emitted struct { + v1 schema.GoCallable + v2 v2.Callable + declFile string +} + +// pairCallables walks the v1 symbol table and the v2 tree in parallel. +func pairCallables(st map[string]schema.GoFile, out *v2.Analysis) []emitted { + var all []emitted + var walk func(c1 schema.GoCallable, c2 v2.Callable, encl string) + walk = func(c1 schema.GoCallable, c2 v2.Callable, encl string) { + file := encl + if c2.SourceFile != "" { + file = c2.SourceFile + } + all = append(all, emitted{c1, c2, file}) + for name, inner := range c1.InnerCallables { + walk(inner, c2.Callables[name], file) + } + } + for path, f := range st { + mod := out.Application.SymbolTable[path] + for sig, fn := range f.Functions { + walk(fn, mod.Functions[sig], path) + } + for tn, ty := range f.Types { + for sig, m := range ty.Methods { + walk(m, mod.Types[tn].Callables[sig], path) + } + } + } + sort.Slice(all, func(i, j int) bool { return all[i].v2.ID < all[j].v2.ID }) + return all +} + +func TestGateL3_EveryCallableHasEntryExit(t *testing.T) { + for name, floor := range map[string]int{"dataflow": 50, "multipackage": 10, "chi": 200} { + gateEntryExit(t, name, floor) + } +} + +func gateEntryExit(t *testing.T, name string, floor int) { + fx := buildL3(t, name) + if fx.l3.MaxLevel != 3 || fx.l3.KLimit != 3 { + t.Errorf("%s: envelope max_level=%d k_limit=%d, want 3, 3", name, fx.l3.MaxLevel, fx.l3.KLimit) + } + withBody := 0 + for _, e := range pairCallables(fx.st, fx.l3) { + if e.v1.Code == "" { + continue // bodyless (interface method) + } + withBody++ + for _, k := range []string{"@entry", "@exit"} { + if _, ok := e.v2.Body[k]; !ok { + t.Errorf("%s: no %s body node", e.v2.ID, k) + } + } + if len(e.v2.Cfg) == 0 { + t.Errorf("%s: empty cfg", e.v2.ID) + } + } + if withBody < floor { + t.Errorf("%s: only %d callables with a body", name, withBody) + } +} + +func TestGateL3_EveryEdgeEndpointIsABodyKey(t *testing.T) { + for name, floor := range map[string]int{"dataflow": 500, "chi": 5000} { + fx := buildL3(t, name) + edges := 0 + for _, e := range pairCallables(fx.st, fx.l3) { + c := e.v2 + in := func(list, k string) { + if _, ok := c.Body[k]; !ok { + t.Errorf("%s: %s endpoint %q is not a body key", c.ID, list, k) + } + } + for _, g := range c.Cfg { + in("cfg", g.Src) + in("cfg", g.Dst) + } + for _, g := range c.Cdg { + in("cdg", g.Src) + in("cdg", g.Dst) + } + for _, g := range c.Ddg { + in("ddg", g.Src) + in("ddg", g.Dst) + if len(g.Prov) != 1 || g.Prov[0] != "ssa" { + t.Errorf("%s: ddg %v prov = %v, want [ssa]", c.ID, g, g.Prov) + } + } + for k, n := range c.Body { + if n.Parent != "" { + in("parent of "+k, n.Parent) + } + } + edges += len(c.Cfg) + len(c.Cdg) + len(c.Ddg) + } + if edges < floor { + t.Errorf("%s: only %d edges", name, edges) + } + t.Logf("%s: %d cfg+cdg+ddg edges", name, edges) + } +} + +func TestGateL3_BodyNodeIDsAreGlobalOrdinals(t *testing.T) { + fx := buildL3(t, "dataflow") + for _, e := range pairCallables(fx.st, fx.l3) { + for k, n := range e.v2.Body { + if want := v2.GlobalOrdinal(e.v2.ID, k); n.ID != want { + t.Errorf("%s body %s id = %q, want %q", e.v2.ID, k, n.ID, want) + } + } + } + // Below L3 every body node carries the same id (R15, Python parity: + // stamp_body_ids runs at L1); parent is still L3+ only. + for _, e := range pairCallables(fx.st, fx.l2) { + for k, n := range e.v2.Body { + if want := v2.GlobalOrdinal(e.v2.ID, k); n.ID != want { + t.Errorf("L2 %s body %s id = %q, want %q", e.v2.ID, k, n.ID, want) + } + if n.Parent != "" { + t.Errorf("L2 %s body %s carries parent %q", e.v2.ID, k, n.Parent) + } + } + } + if fx.l2.KLimit != 0 { + t.Errorf("L2 k_limit = %d, want absent", fx.l2.KLimit) + } +} + +// TestGateL3_SupersetOfL2 is the monotonicity gate: everything in the -a 2 +// JSON is present and equal in the -a 3 JSON. The only sanctioned additions +// are new body nodes, cfg/cdg/ddg on a callable, parent on a body node and +// the envelope's k_limit; max_level is the level marker itself. A body node's +// id is emitted from L1 (R15), so an L1 node's id must be present and equal +// at L3 — it is no longer an allowed addition. +func TestGateL3_SupersetOfL2(t *testing.T) { + for _, name := range []string{"dataflow", "multipackage"} { + fx := buildL3(t, name) + a, b := toAny(t, fx.l2), toAny(t, fx.l3) + a["max_level"], b["max_level"] = nil, nil + var errs []string + subset("$", "", a, b, &errs) + for i, e := range errs { + if i == 20 { + t.Errorf("%s: … %d more", name, len(errs)-20) + break + } + t.Errorf("%s: %s", name, e) + } + } +} + +func toAny(t *testing.T, a *v2.Analysis) map[string]any { + t.Helper() + data, err := json.Marshal(a) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + t.Fatal(err) + } + return m +} + +// subset records where a (the lower level) is not contained in b. parentKey +// is the map key a and b sit under, which tells the sanctioned additions +// apart: any new key in a "body" map, parent in a body node (an object under +// a body map), cfg/cdg/ddg in a callable (an object with a body), k_limit at +// the root. +func subset(path, parentKey string, a, b any, errs *[]string) { + switch av := a.(type) { + case map[string]any: + bv, ok := b.(map[string]any) + if !ok { + *errs = append(*errs, fmt.Sprintf("%s: object became %T", path, b)) + return + } + for k, x := range av { + y, ok := bv[k] + if !ok { + *errs = append(*errs, fmt.Sprintf("%s.%s missing at L3", path, k)) + continue + } + subset(path+"."+k, k, x, y, errs) + } + _, isCallable := av["body"] + for k := range bv { + if _, ok := av[k]; ok { + continue + } + switch { + case parentKey == "body": + case path == "$" && k == "k_limit": + case isCallable && (k == "cfg" || k == "cdg" || k == "ddg"): + case k == "parent" && strings.Contains(path, ".body."): + default: + *errs = append(*errs, fmt.Sprintf("%s.%s: unsanctioned addition", path, k)) + } + } + case []any: + bv, ok := b.([]any) + if !ok || len(av) != len(bv) { + *errs = append(*errs, fmt.Sprintf("%s: array changed", path)) + return + } + for i := range av { + subset(fmt.Sprintf("%s[%d]", path, i), parentKey, av[i], bv[i], errs) + } + default: + if a != b { + *errs = append(*errs, fmt.Sprintf("%s: %v -> %v", path, a, b)) + } + } +} + +// TestGateL3_SpansSliceToStatementText: every L3 node's span slices its +// declaring module's source to text that starts with the construct's keyword, +// and every call node keeps its L1 span (R7). multipackage carries a method +// declared apart from its receiver type (Server.Describe in middleware.go). +func TestGateL3_SpansSliceToStatementText(t *testing.T) { + prefixes := map[string][]string{ + "branch": {"if"}, + "loop": {"for"}, + "switch": {"switch", "select"}, + "return": {"return"}, + } + for _, name := range []string{"dataflow", "multipackage"} { + fx := buildL3(t, name) + l2 := map[string]v2.Callable{} + for _, e := range pairCallables(fx.st, fx.l2) { + l2[e.v2.ID] = e.v2 + } + checked, crossFile := 0, 0 + for _, e := range pairCallables(fx.st, fx.l3) { + src := fx.l3.Application.SymbolTable[e.declFile].Source + for k, n := range e.v2.Body { + text := n.Span.Slice(src) + switch n.Kind { + case "entry", "exit": + continue + case "call": + if l1 := l2[e.v2.ID].Body[k]; l1.Span != n.Span { + t.Errorf("%s call %s span %v, L1 %v", e.v2.ID, k, n.Span, l1.Span) + } + case "statement": + if strings.TrimSpace(text) == "" || text != strings.TrimSpace(text) { + t.Errorf("%s %s statement span slices to %q", e.v2.ID, k, text) + } + default: + ok := false + for _, p := range prefixes[n.Kind] { + ok = ok || strings.HasPrefix(text, p) + } + if !ok { + t.Errorf("%s %s (%s) span slices to %q", e.v2.ID, k, n.Kind, text) + } + } + if want := v2.LocalID(n.Span.Start[0], n.Span.Start[1]); n.Kind != "call" && !strings.HasPrefix(k, want) { + t.Errorf("%s %s span starts at %s", e.v2.ID, k, want) + } + checked++ + if e.v2.SourceFile != "" { + crossFile++ + } + } + } + if checked < 10 { + t.Errorf("%s: only %d nodes checked", name, checked) + } + if name == "multipackage" && crossFile == 0 { + t.Error("multipackage: no cross-file method body node checked") + } + } +} + +func TestEmitL3_NoOrphanSideTableEntries(t *testing.T) { + fx := buildL3(t, "dataflow") + seen := map[dataflow.L3Key]bool{} + for _, e := range pairCallables(fx.st, fx.l3) { + seen[dataflow.L3Key{Path: e.declFile, Signature: e.v2.Signature}] = true + } + for k := range fx.res { + if !seen[k] { + t.Errorf("side-table entry %v matches no emitted callable", k) + } + } + if len(fx.res) < 50 { + t.Errorf("only %d side-table entries", len(fx.res)) + } +} + +// TestGateL3_CrossFileClosureSourceFile (R19): a closure nests under its +// method, which nests under the receiver type's module (xfile/a.go), but its +// positions are in the method's declaring file (xfile/b.go). Every callable +// whose declaring file differs from its containing module carries +// source_file — closures and nested closures included — so the SDK's text +// recovery (symbol_table[source_file].source) slices the right file for the +// closure and for every body node inside it. +func TestGateL3_CrossFileClosureSourceFile(t *testing.T) { + fx := buildL3(t, "dataflow") + for _, out := range []*v2.Analysis{fx.l2, fx.l3} { + mod := mustModule(t, out, "xfile/a.go") + run, ok := mod.Types["T"].Callables["example.com/dataflow/xfile.T.Run"] + if !ok { + t.Fatalf("T.Run missing; types %v", keys(mod.Types)) + } + add, ok := run.Callables["closure_1"] + if !ok { + t.Fatalf("closure_1 missing; have %v", keys(run.Callables)) + } + inc, ok := add.Callables["closure_1"] + if !ok { + t.Fatalf("nested closure missing; have %v", keys(add.Callables)) + } + src := mustModule(t, out, "xfile/b.go").Source + for _, c := range []v2.Callable{run, add, inc} { + if c.SourceFile != "xfile/b.go" { + t.Errorf("L%d %s source_file = %q, want xfile/b.go", out.MaxLevel, c.ID, c.SourceFile) + } + } + // Callable spans are line-level (v1 records closure lines only), so + // the slice starts at the line's first column: it holds the closure's + // func literal from its opening to its closing brace. + for _, w := range []struct { + c v2.Callable + head string + }{{add, "add := func(d int) int {"}, {inc, "inc := func() {"}} { + text := strings.TrimSpace(w.c.Span.Slice(src)) + if !strings.HasPrefix(text, w.head) || !strings.HasSuffix(text, "}") { + t.Errorf("L%d %s span over b.go slices to %q, want %q…}", out.MaxLevel, w.c.ID, text, w.head) + } + } + if out.MaxLevel < 3 { + continue + } + for _, w := range []struct { + c v2.Callable + key string + text string + }{{add, "6:3", "t.n += d"}, {add, "11:3", "return t.n"}, {inc, "8:4", "t.n++"}} { + n, ok := w.c.Body[w.key] + if !ok { + t.Errorf("%s: no body node %s; have %v", w.c.ID, w.key, keys(w.c.Body)) + continue + } + if got := n.Span.Slice(src); !strings.HasPrefix(got, w.text) { + t.Errorf("%s body %s over b.go slices to %q, want %q…", w.c.ID, w.key, got, w.text) + } + } + } + // The negative: a closure of a same-file function carries no source_file. + fn := functionByName(t, mustModule(t, fx.l3, "flow/flow.go"), "Closure") + for _, in := range fn.Callables { + if in.SourceFile != "" { + t.Errorf("%s: same-file closure carries source_file %q", in.ID, in.SourceFile) + } + } +} + +// TestGateL3_GoStatementArgumentCalls: a call in a `go` statement's arguments +// is evaluated at the go statement, so at L3 it is a satellite whose parent is +// the go node (the launched call's key). GoArgs having a result at all shows +// the L1 and L3 call walks agree (a count mismatch leaves it at L1). +func TestGateL3_GoStatementArgumentCalls(t *testing.T) { + fx := buildL3(t, "dataflow") + key := dataflow.L3Key{Path: filepath.Join("flow", "flow.go"), Signature: "example.com/dataflow/flow.GoArgs"} + if _, ok := fx.res[key]; !ok { + t.Fatalf("no L3 result for %v (call-count mismatch?)", key) + } + fn := functionByName(t, mustModule(t, fx.l3, "flow/flow.go"), "GoArgs") + for k, parent := range map[string]string{"534:5": "", "534:7": "534:5", "535:5": "", "535:27": "535:5"} { + n, ok := fn.Body[k] + if !ok { + t.Errorf("GoArgs: no body node %s; have %v", k, keys(fn.Body)) + continue + } + if n.Kind != "call" || n.Parent != parent { + t.Errorf("GoArgs %s kind=%q parent=%q, want call / %q", k, n.Kind, n.Parent, parent) + } + } +} diff --git a/internal/syntactic_analysis/v2emit/gate_test.go b/internal/syntactic_analysis/v2emit/gate_test.go index b021a9d..82a6626 100644 --- a/internal/syntactic_analysis/v2emit/gate_test.go +++ b/internal/syntactic_analysis/v2emit/gate_test.go @@ -25,11 +25,13 @@ import ( "encoding/json" "path/filepath" "runtime" + "sort" "strings" "testing" "github.com/codellm-devkit/codeanalyzer-go/internal/schema" v2 "github.com/codellm-devkit/codeanalyzer-go/internal/schema/v2" + "github.com/codellm-devkit/codeanalyzer-go/internal/semantic_analysis" "github.com/codellm-devkit/codeanalyzer-go/internal/syntactic_analysis" ) @@ -56,7 +58,7 @@ func buildMultipackageV2(t *testing.T) *v2.Analysis { t.Fatalf("symbol table build failed: %v", err) } app := &schema.GoApplication{SymbolTable: st, CallGraph: []schema.GoCallEdge{}} - return Emit(app, "multipackage", dir, 1, "gate-test") + return Emit(app, "multipackage", dir, 1, "gate-test", 0, nil) } // mustModule fetches a module by relative path or fails. @@ -85,7 +87,7 @@ func TestGateL1_ModuleSet(t *testing.T) { if mod.Kind != "module" { t.Errorf("%s: kind = %q, want module", rel, mod.Kind) } - wantID := "can://go/multipackage/" + rel + wantID := "can://multipackage/go/" + rel if mod.ID != wantID { t.Errorf("%s: id = %q, want %q", rel, mod.ID, wantID) } @@ -299,8 +301,8 @@ func TestGateL1_Envelope(t *testing.T) { if out.MaxLevel != 1 { t.Errorf("max_level = %d, want 1", out.MaxLevel) } - if out.Application.ID != "can://go/multipackage" { - t.Errorf("application.id = %q, want can://go/multipackage", out.Application.ID) + if out.Application.ID != "can://multipackage" { + t.Errorf("application.id = %q, want can://multipackage", out.Application.ID) } // L1: no call_graph edges yet. if len(out.Application.CallGraph) != 0 { @@ -320,7 +322,7 @@ func TestGateL1_SupersetOfV1(t *testing.T) { t.Fatalf("symbol table build failed: %v", err) } app := &schema.GoApplication{SymbolTable: st, CallGraph: []schema.GoCallEdge{}} - out := Emit(app, "multipackage", dir, 1, "gate-test") + out := Emit(app, "multipackage", dir, 1, "gate-test", 0, nil) // Every v1 file, type, method and function reappears in v2. for rel, v1file := range st { @@ -368,3 +370,253 @@ func contains(ss []string, want string) bool { } return false } + +// ─── L1 fixes: type kinds, defer marker, co-positioned calls ───────────────── + +// buildDataflowL1V2 emits the L1 payload for the testdata/dataflow fixture. +func buildDataflowL1V2(t *testing.T) *v2.Analysis { + t.Helper() + _, thisFile, _, _ := runtime.Caller(0) + dir, err := filepath.Abs(filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "testdata", "dataflow")) + if err != nil { + t.Fatalf("resolving fixture dir: %v", err) + } + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("symbol table build failed: %v", err) + } + app := &schema.GoApplication{SymbolTable: st, CallGraph: []schema.GoCallEdge{}} + return Emit(app, "dataflow", dir, 1, "gate-test", 0, nil) +} + +func TestGateL1_TypeKindsAliasAndDefined(t *testing.T) { + mod := mustModule(t, buildDataflowL1V2(t), "l1/l1.go") + want := map[string]string{"Celsius": "defined", "Alias": "alias", "S": "struct", + "Base": "struct", "MyErr": "struct", "I1": "interface", "I2": "interface"} + for name, kind := range want { + ty, ok := mod.Types[name] + if !ok { + t.Errorf("type %s missing from l1/l1.go; have %v", name, keys(mod.Types)) + continue + } + if ty.Kind != kind { + t.Errorf("type %s: kind=%q, want %q", name, ty.Kind, kind) + } + } +} + +// functionByName finds a package-level function node by name. +func functionByName(t *testing.T, mod v2.Module, name string) v2.Callable { + t.Helper() + for sig, c := range mod.Functions { + if strings.HasSuffix(sig, "."+name) { + return c + } + } + t.Fatalf("function %q missing; have %v", name, keys(mod.Functions)) + return v2.Callable{} +} + +func TestGateL1_DeferredCallNode(t *testing.T) { + d := functionByName(t, mustModule(t, buildDataflowL1V2(t), "l1/l1.go"), "D") + var deferred, goroutine int + for _, n := range d.Body { + if n.Kind != "call" { + continue + } + if n.IsDeferred { + deferred++ + if n.IsGoroutine { + t.Error("deferred call must not be a goroutine") + } + } + if n.IsGoroutine { + goroutine++ + } + } + if deferred != 1 || goroutine != 1 { + t.Errorf("D body: deferred=%d goroutine=%d, want 1 and 1 (body=%v)", deferred, goroutine, d.Body) + } +} + +func TestGateL1_CoPositionedCallsBothKept(t *testing.T) { + c := functionByName(t, mustModule(t, buildDataflowL1V2(t), "l1/l1.go"), "Chain") + if len(c.Body) != 2 { + t.Fatalf("Chain body has %d nodes, want 2: %v", len(c.Body), c.Body) + } + var bare, second string + for k := range c.Body { + if strings.HasSuffix(k, "/2") { + second = k + } else { + bare = k + } + } + if bare == "" || second != bare+"/2" { + t.Fatalf("want keys L:C and L:C/2, got %v", keys(c.Body)) + } + // The bare key is the outermost call (s.A().B()), which spans further. + if c.Body[bare].Span.End[1] <= c.Body[second].Span.End[1] { + t.Errorf("bare key should hold the outermost (longer) call: %+v vs %+v", c.Body[bare].Span, c.Body[second].Span) + } +} + +// buildDataflowL2V2 emits the L2 payload (resolver run) for the dataflow fixture. +func buildDataflowL2V2(t *testing.T) *v2.Analysis { + t.Helper() + _, thisFile, _, _ := runtime.Caller(0) + dir, err := filepath.Abs(filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "testdata", "dataflow")) + if err != nil { + t.Fatalf("resolving fixture dir: %v", err) + } + b := syntactic_analysis.NewSymbolTableBuilder(dir) + st, err := b.Build(nil, true) + if err != nil { + t.Fatalf("symbol table build failed: %v", err) + } + cg := semantic_analysis.NewCallGraphBuilder(dir, b.Fset(), b.Pkgs()) + edges := cg.Build(st) + app := &schema.GoApplication{SymbolTable: st, CallGraph: edges} + return Emit(app, "dataflow", dir, 2, "gate-test", 0, nil) +} + +// TestGateL2_CoPositionedCallsResolveSeparately: in s.A().B() the outer call +// (bare key) resolves to B and the inner s.A() (key /2) resolves to A; the call +// graph has exactly the edges Chain->A and Chain->B. +func TestGateL2_CoPositionedCallsResolveSeparately(t *testing.T) { + out := buildDataflowL2V2(t) + mod := mustModule(t, out, "l1/l1.go") + chain := functionByName(t, mod, "Chain") + s := mod.Types["S"] + var aID, bID string + for sig, m := range s.Callables { + switch { + case strings.HasSuffix(sig, ".A"): + aID = m.ID + case strings.HasSuffix(sig, ".B"): + bID = m.ID + } + } + if aID == "" || bID == "" { + t.Fatalf("methods A/B not found on S: %v", keys(s.Callables)) + } + var bare, second string + for k := range chain.Body { + if strings.HasSuffix(k, "/2") { + second = k + } else { + bare = k + } + } + got := func(k string) string { + if c := chain.Body[k].Callee; c != nil { + return *c + } + return "" + } + if got(bare) != bID { + t.Errorf("outer call %s callee=%s, want B %s", bare, got(bare), bID) + } + if got(second) != aID { + t.Errorf("inner call %s callee=%s, want A %s", second, got(second), aID) + } + var dsts []string + for _, e := range out.Application.CallGraph { + if e.Src == chain.ID { + dsts = append(dsts, e.Dst) + } + } + sort.Strings(dsts) + want := []string{aID, bID} + sort.Strings(want) + if strings.Join(dsts, ",") != strings.Join(want, ",") { + t.Errorf("Chain edges = %v, want {A, B} = %v", dsts, want) + } +} + +// TestGateL1_BodyNodeIDs (R15): from L1 on, every body node carries its global +// id, v2.GlobalOrdinal(callable id, local key) — Python parity (its +// stamp_body_ids runs at L1) — and no parent (an L3 refinement). +func TestGateL1_BodyNodeIDs(t *testing.T) { + out := buildDataflowL1V2(t) + checked := 0 + var walk func(c v2.Callable) + walk = func(c v2.Callable) { + for k, n := range c.Body { + if want := v2.GlobalOrdinal(c.ID, k); n.ID != want { + t.Errorf("%s body %s id = %q, want %q", c.ID, k, n.ID, want) + } + if n.Parent != "" { + t.Errorf("%s body %s carries parent %q at L1", c.ID, k, n.Parent) + } + checked++ + } + for _, in := range c.Callables { + walk(in) + } + } + for _, m := range out.Application.SymbolTable { + for _, f := range m.Functions { + walk(f) + } + for _, ty := range m.Types { + for _, c := range ty.Callables { + walk(c) + } + } + } + if checked < 30 { + t.Errorf("only %d L1 body nodes checked", checked) + } +} + +// TestGateL2_GoStatementArgumentCalls: calls in a `go` statement's arguments +// (`go f(g(x))`, `go func(y int){…}(h())`) are call sites of their own — +// call nodes with resolved callees and call_graph edges — evaluated in the +// launching goroutine, so only the launched call carries is_goroutine. +func TestGateL2_GoStatementArgumentCalls(t *testing.T) { + out := buildDataflowL2V2(t) + fn := functionByName(t, mustModule(t, out, "flow/flow.go"), "GoArgs") + const pkg = "example.com/dataflow/flow." + for _, w := range []struct { + key, callee string + goroutine bool + }{ + {"534:5", "f", true}, + {"534:7", "g", false}, + {"535:5", "", true}, // the launched func literal: no named callee + {"535:27", "h", false}, + } { + n, ok := fn.Body[w.key] + if !ok { + t.Errorf("GoArgs: no call node %s; have %v", w.key, keys(fn.Body)) + continue + } + if n.IsGoroutine != w.goroutine { + t.Errorf("GoArgs %s is_goroutine = %v, want %v", w.key, n.IsGoroutine, w.goroutine) + } + if w.callee == "" { + continue + } + want := "can://dataflow/go/flow/flow.go/" + pkg + w.callee + if n.Callee == nil || *n.Callee != want { + t.Errorf("GoArgs %s callee = %v, want %s", w.key, n.Callee, want) + } + if !anyEdgeFromTo(out.Application.CallGraph, fn.ID, want) { + t.Errorf("no call_graph edge %s -> %s", fn.ID, want) + } + } + if len(fn.Body) != 4 { + t.Errorf("GoArgs body has %d nodes, want 4: %v", len(fn.Body), keys(fn.Body)) + } +} + +func anyEdgeFromTo(edges []v2.Edge, src, dst string) bool { + for _, e := range edges { + if e.Src == src && e.Dst == dst { + return true + } + } + return false +} diff --git a/internal/syntactic_analysis/v2emit/sigindex.go b/internal/syntactic_analysis/v2emit/sigindex.go index 1f2d5c9..992af19 100644 --- a/internal/syntactic_analysis/v2emit/sigindex.go +++ b/internal/syntactic_analysis/v2emit/sigindex.go @@ -32,6 +32,8 @@ import ( type sigIndex struct { bySig map[string]string bySigPath map[sigPathKey]string + // externals maps a library callee's full signature to its @external id. + externals map[string]string } type sigPathKey struct { @@ -97,6 +99,17 @@ func (idx sigIndex) idFor(sig string) (string, bool) { return id, ok } +// resolve returns the node id for a callee signature: the in-tree callable id +// when the signature names one, else the @external id of a recorded library +// callee. +func (idx sigIndex) resolve(sig string) (string, bool) { + if id, ok := idx.idFor(sig); ok { + return id, true + } + id, ok := idx.externals[sig] + return id, ok +} + // idForWithPath returns the can:// id for a signature declared in a specific // file. It is the precise resolver for a non-unique signature: an init call // site in request_id.go resolves to request_id.go's init node, not an diff --git a/scripts/eval_l3.sh b/scripts/eval_l3.sh new file mode 100755 index 0000000..0cb8fb7 --- /dev/null +++ b/scripts/eval_l3.sh @@ -0,0 +1,436 @@ +#!/usr/bin/env bash +# eval_l3.sh - reproducible real-app evaluation of cango -a 3 (L3 dataflow). +# +# Usage: +# scripts/eval_l3.sh --prepare [APP...] clone, set up and type-check apps (all if none named) +# scripts/eval_l3.sh APP [APP...] run the evaluation on prepared apps, by manifest name +# scripts/eval_l3.sh --all run every app in the manifest (each must be prepared) +# scripts/eval_l3.sh APP_DIR [APP_DIR...] run on plain directories (no manifest, no preparation) +# scripts/eval_l3.sh --clean delete run outputs (runs, caches, notes, built cango) in +# $EVAL_OUT; keep the clones and the prepared state +# scripts/eval_l3.sh --list print the manifest +# +# Example (from the repository root): +# scripts/eval_l3.sh --prepare kubernetes k8s-api +# scripts/eval_l3.sh kubernetes k8s-api +# +# --prepare, for each app: +# 1. Clones into $EVAL_SRC/ when absent: git init, then +# git fetch --depth 1 --filter=blob:limit=2m origin , then git checkout FETCH_HEAD. +# An existing clone is reused when HEAD is the pinned commit; at any other commit the app is +# refused (it is never reset). +# 2. Checks the tools its prep needs. A missing tool is reported with how to get it; prep is +# then skipped and the app is marked "degraded: missing ". Nothing is installed. +# 3. Runs prep inside the clone, logging to $EVAL_OUT/prep-.log, and records its exit code. +# 4. Type-checks the module dir with the app's env and GOTOOLCHAIN=$EVAL_GO: +# go list -e -export ./... . -export compiles every package, so a type error becomes that +# package's Error, and a package broken only by a dependency gets DepsErrors. (Plain +# go list -e -json reports load errors only: it shows 0 errors on dae for GOOS=darwin, where +# 2 packages fail to compile and 3 more are broken through them.) The status is "complete" +# (no package with Error or DepsErrors) or "degraded: N packages with errors"; the per-package +# list is in $EVAL_OUT/prepared/.typecheck. +# 5. Writes $EVAL_OUT/prepared/.env (module dir, env, commit, prep and type-check status), +# which the run phase reads. +# +# Run phase, for each app (manifest name or directory), with --eager (no cache) and the +# analyzer's stderr captured separately from the /usr/bin/time -l report: +# 0. cango -a 2 -j 1 (warm-up, discarded) +# 1. cango -a 2 -j 1 (wall time baseline) +# 2. cango -a 3 -j 1 +# 3. cango -a 3 -j N (N = CPU count) +# 4. cango -a 3 -j N (second run) +# and compares the three -a 3 analysis.json files byte for byte. It records wall time, peak +# RSS (macOS /usr/bin/time -l), module/callable/body-node/edge counts, external_symbols, the +# call_graph split, the prepare-time type-check status, the number of type-error warnings +# ("package ..." stderr lines), the analyzer's too-old-toolchain warning, and the number of +# "L3 skipped" stderr lines (caught per-callable panics). A manifest app runs with its env +# (e.g. GOOS=linux for dae) on top of the inherited environment; every run sets +# GOTOOLCHAIN=$EVAL_GO unless GOTOOLCHAIN is already set. +# +# Environment: +# EVAL_SRC clones (default /tmp/eval) +# EVAL_OUT raw outputs and logs (default /tmp/eval-out) +# EVAL_GO Go toolchain for the analyzer build, type-check and runs (default go1.27.3). It +# must be at least the newest `go` line in the corpus (go 1.27.x today), or the +# analyzer type-checks those packages only partially (one consolidated WARN). +# CANGO analyzer binary. When unset, the run phase builds one from this checkout with +# GOTOOLCHAIN=$EVAL_GO go build -o $EVAL_OUT/cango ./cmd/codeanalyzer +# EVAL_NOTE markdown results file (default $EVAL_OUT/l3-.md) +# EVAL_JOBS N for the -j N runs (default: CPU count) +# RUN_TIMEOUT per-run limit, seconds (default 3600); on expiry the run is killed, the app +# is marked TIMEOUT and its remaining runs are skipped +# EVAL_PRUNE 1 = after the comparison, delete the warm-up and both -j N analysis.json files +# (keeps a2j1 and a3j1, and every .err/.time). For very large apps. +# Any other environment (GOOS, GOFLAGS, ...) is inherited by the analyzer unchanged. +# +# Requires: git, go, /usr/bin/time -l (BSD/macOS), cmp. jq is used for counts; without it the +# count columns are reported as "n/a" and everything else still works. +set -u + +EVAL_SRC=${EVAL_SRC:-/tmp/eval} +EVAL_OUT=${EVAL_OUT:-/tmp/eval-out} +EVAL_GO=${EVAL_GO:-go1.27.3} +DATE=$(date +%Y-%m-%d) +EVAL_NOTE=${EVAL_NOTE:-$EVAL_OUT/l3-$DATE.md} +RUN_TIMEOUT=${RUN_TIMEOUT:-3600} +EVAL_PRUNE=${EVAL_PRUNE:-0} +JOBS=${EVAL_JOBS:-$( (sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4) | head -1)} +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +PREPARED=$EVAL_OUT/prepared + +# Homebrew's keg-only llvm@18 provides clang and llvm-strip for dae's `make ebpf`. +if command -v brew >/dev/null 2>&1; then + LLVM18=$(brew --prefix llvm@18 2>/dev/null || true) + if [ -n "$LLVM18" ] && [ -d "$LLVM18/bin" ]; then PATH=$LLVM18/bin:$PATH; export PATH; fi +fi + +# MANIFEST: name|repo|commit|subdir|env|prep|tools|role ("-" = none) +# subdir module dir inside the clone, analyzed with -i ("." = clone root) +# env extra environment for go list and the analyzer (space-separated K=V) +# prep shell snippet run inside the clone after checkout (no "|" allowed) +# tools tools prep needs, space-separated; "a/b" means either a or b +# The clone dir is $EVAL_SRC/, so entries may share one clone. +MANIFEST=$(cat <<'EOF' +cobra|https://github.com/spf13/cobra|adbc8813901bba65827259daa8e22ff94ec1f30e|.|-|-|-|smoke test; CLI library, no concurrency +listmonk|https://github.com/knadh/listmonk|8216c4f15fa918ffe7e1a9e692701df5134a5f4b|.|-|-|-|small HTTP-to-SQL app: echo handlers into sqlx +memos|https://github.com/usememos/memos|28719a925fb28432d7b253863138ab388ec4012a|.|-|-|-|HTTP to SQL through an interface (store.Driver) +nats-server|https://github.com/nats-io/nats-server|f8f6db894528640b27311353e874ac62c3a47af5|.|-|-|-|CFG stress: select, channels, defer, cyclomatic 453 +openbao|https://github.com/openbao/openbao|1d58653b0c00e53484de5af18f20679d78b9fb28|.|-|-|-|large app; many defers +dae|https://github.com/daeuniverse/dae|e3fee8fbc68a65167af13b685ab0b958757e20ee|.|GOOS=linux|git submodule update --init --depth 1 && make ebpf|clang llvm-strip|eBPF proxy, heavy select; Linux-only, bpf2go code generated +kubernetes|https://github.com/kubernetes/kubernetes|f78e722310e50bcaca9276be22276d9e91d91308|.|-|-|-|scale benchmark: root module k8s.io/kubernetes (v1.37.1), fully type-checked +k8s-api|https://github.com/kubernetes/kubernetes|f78e722310e50bcaca9276be22276d9e91d91308|staging/src/k8s.io/api|-|-|-|generated protobuf marshallers (k8s.io/api, v1.37.1): DDG volume on dAtA[] code +cockroach|https://github.com/cockroachdb/cockroach|d30c905fff79ef825adc96bcc647f1872a90f2ff|.|-|./dev generate go|bazel/bazelisk|very large, partly type-checked without Bazel; not a published benchmark (proprietary license) +EOF +) + +usage() { sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; } + +# manifest_get NAME -> sets M_NAME M_REPO M_COMMIT M_SUBDIR M_ENV M_PREP M_TOOLS M_ROLE; 1 if absent +manifest_get() { + local line + line=$(printf '%s\n' "$MANIFEST" | awk -F'|' -v n="$1" '$1 == n { print; exit }') + [ -n "$line" ] || return 1 + IFS='|' read -r M_NAME M_REPO M_COMMIT M_SUBDIR M_ENV M_PREP M_TOOLS M_ROLE <&2; exit 2 ;; esac + for n in $(manifest_names) $(ls "$EVAL_OUT" 2>/dev/null | sed -n 's/-a3j1$//p'); do + for t in warm a2j1 a3j1 a3jN a3jN2; do + rm -rf "$EVAL_OUT/$n-$t" "$EVAL_OUT/$n-$t.err" "$EVAL_OUT/$n-$t.time" "$EVAL_OUT/$n-$t.timeout" + done + rm -rf "$EVAL_OUT/cache-$n" + done + rm -f "$EVAL_OUT"/l3-*.md "$EVAL_OUT/cango" + echo "cleaned run outputs in $EVAL_OUT (kept $PREPARED, prep logs and clones in $EVAL_SRC)" >&2 + exit 0 +fi + +# ------------------------------------------------------------------------------- --prepare +# prepare_one NAME -> 0 prepared (complete or degraded), 1 not preparable +prepare_one() { + local name=$1 + manifest_get "$name" || { echo "unknown app: $name (see --list)" >&2; return 1; } + echo "== prepare $name ($M_CLONE @ ${M_COMMIT:0:12})" >&2 + mkdir -p "$EVAL_SRC" "$PREPARED" + rm -f "$PREPARED/$name.env" + + # 1. Clone or reuse. + if [ ! -e "$M_CLONE" ]; then + mkdir -p "$M_CLONE" + if ! (cd "$M_CLONE" && git init -q && git remote add origin "$M_REPO" && + git fetch --depth 1 --filter=blob:limit=2m origin "$M_COMMIT" && + git checkout -q FETCH_HEAD); then + echo " clone failed: $M_REPO $M_COMMIT (left at $M_CLONE; remove it to retry)" >&2 + return 1 + fi + fi + local head + head=$(git -C "$M_CLONE" rev-parse HEAD 2>/dev/null) || { + echo " $M_CLONE exists but is not a git checkout; refusing" >&2; return 1; } + if [ "$head" != "$M_COMMIT" ]; then + echo " $M_CLONE is at $head, not the pinned $M_COMMIT; refusing (not resetting it)." >&2 + echo " Check it out yourself, or move it aside to let --prepare clone afresh." >&2 + return 1 + fi + [ -d "$M_MODDIR" ] || { echo " module dir missing: $M_MODDIR" >&2; return 1; } + + # 2. Tools. + local missing="" t alt found + for t in $M_TOOLS; do + found="" + for alt in $(echo "$t" | tr '/' ' '); do command -v "$alt" >/dev/null 2>&1 && found=$alt && break; done + if [ -z "$found" ]; then + missing="$missing ${t//\// or }" + echo " missing tool: ${t//\// or } -> $(tool_hint "${t%%/*}")" >&2 + fi + done + missing=${missing# } + + # 3. Prep. + local prep_status=none prep_rc=- + local log=$EVAL_OUT/prep-$name.log + if [ -n "$M_PREP" ]; then + if [ -n "$missing" ]; then + prep_status="skipped: missing $missing" + echo "prep skipped: missing $missing" >"$log" + else + echo " prep: $M_PREP (log: $log)" >&2 + (cd "$M_CLONE" && GOTOOLCHAIN=${GOTOOLCHAIN:-$EVAL_GO} bash -c "$M_PREP") >"$log" 2>&1 + prep_rc=$? + if [ "$prep_rc" -eq 0 ]; then prep_status=ok; else prep_status="failed (rc $prep_rc)"; fi + echo " prep: $prep_status" >&2 + fi + fi + + # 4. Type-check status. + local tc=$PREPARED/$name.typecheck tc_rc nerr nown ndep tc_status + # shellcheck disable=SC2086 + (cd "$M_MODDIR" && env $M_ENV GOTOOLCHAIN=${GOTOOLCHAIN:-$EVAL_GO} go list -e -export \ + -f '{{.ImportPath}} {{if .Error}}E{{else}}-{{end}}{{if .DepsErrors}}D{{else}}-{{end}} {{if .Error}}{{printf "%q" .Error.Err}}{{end}}' \ + ./...) >"$tc" 2>"$tc.err" + tc_rc=$? + nerr=$(awk -F'\t' '$2 != "--"' "$tc" | wc -l | tr -d ' ') + nown=$(awk -F'\t' '$2 ~ /^E/' "$tc" | wc -l | tr -d ' ') + ndep=$(awk -F'\t' '$2 == "-D"' "$tc" | wc -l | tr -d ' ') + local npkg + npkg=$(wc -l <"$tc" | tr -d ' ') + if [ "$tc_rc" -ne 0 ] || [ "$npkg" -eq 0 ]; then + tc_status="unknown (go list rc $tc_rc; see $tc.err)" + elif [ "$nerr" -eq 0 ]; then + tc_status=complete + else + tc_status="degraded: $nerr packages with errors" + fi + echo " type-check: $tc_status ($npkg packages; $nown with errors, $ndep with dependency errors only)" >&2 + + # Overall. + local status=complete reasons="" + [ -n "$missing" ] && reasons="missing $missing" + case $prep_status in failed*) reasons="${reasons:+$reasons; }prep $prep_status" ;; esac + [ "$tc_status" != complete ] && reasons="${reasons:+$reasons; }${tc_status#degraded: }" + [ -n "$reasons" ] && status="degraded: $reasons" + + # 5. Prepared entry. + { + printf 'P_NAME=%q\n' "$name" + printf 'P_REPO=%q\n' "$M_REPO" + printf 'P_COMMIT=%q\n' "$M_COMMIT" + printf 'P_MODDIR=%q\n' "$M_MODDIR" + printf 'P_ENV=%q\n' "$M_ENV" + printf 'P_PREP=%q\n' "$prep_status" + printf 'P_PREP_RC=%q\n' "$prep_rc" + printf 'P_TC_PKGS=%q\n' "$npkg" + printf 'P_TC_ERR=%q\n' "$nerr" + printf 'P_TC_OWN=%q\n' "$nown" + printf 'P_TC_DEPS=%q\n' "$ndep" + printf 'P_TYPECHECK=%q\n' "$tc_status" + printf 'P_STATUS=%q\n' "$status" + printf 'P_GO=%q\n' "${GOTOOLCHAIN:-$EVAL_GO}" + printf 'P_DATE=%q\n' "$DATE" + } >"$PREPARED/$name.env" + echo " status: $status" >&2 + return 0 +} + +if [ "${1:-}" = "--prepare" ]; then + shift + mkdir -p "$EVAL_OUT" + [ $# -gt 0 ] || set -- $(manifest_names) + BAD="" + for n in "$@"; do prepare_one "$n" || BAD="$BAD $n"; done + echo >&2 + printf '%-12s %s\n' app status >&2 + for n in "$@"; do + if [ -f "$PREPARED/$n.env" ]; then + ( . "$PREPARED/$n.env"; printf '%-12s %s\n' "$n" "$P_STATUS" ) >&2 + else + printf '%-12s %s\n' "$n" "NOT PREPARED" >&2 + fi + done + [ -z "$BAD" ] || { echo "not prepared:$BAD" >&2; exit 1; } + exit 0 +fi + +# ------------------------------------------------------------------------------- run phase +[ $# -gt 0 ] || usage +case $1 in -h|--help) usage ;; esac +if [ "$1" = "--all" ]; then set -- $(manifest_names); fi + +# Resolve every argument before running anything. +TARGETS="" +for a in "$@"; do + if manifest_get "$a"; then + [ -f "$PREPARED/$a.env" ] || { echo "$a is not prepared: run $0 --prepare $a" >&2; exit 2; } + TARGETS="$TARGETS name:$a" + elif [ -d "$a" ]; then + TARGETS="$TARGETS dir:${a%/}" + else + echo "not a manifest app or a directory: $a (see --list)" >&2; exit 2 + fi +done + +mkdir -p "$EVAL_OUT" +if [ -z "${CANGO:-}" ]; then + CANGO=$EVAL_OUT/cango + echo "== building $CANGO with $EVAL_GO" >&2 + (cd "$REPO_ROOT" && GOTOOLCHAIN=$EVAL_GO go build -o "$CANGO" ./cmd/codeanalyzer) || + { echo "analyzer build failed" >&2; exit 2; } +fi +[ -x "$CANGO" ] || { echo "analyzer binary not found: $CANGO" >&2; exit 2; } +RUN_GO=${GOTOOLCHAIN:-$EVAL_GO} + +HAVE_JQ=0 +command -v jq >/dev/null 2>&1 && HAVE_JQ=1 + +JQ_COUNTS=' +(.application) as $a +| [$a.symbol_table[]|..|objects|select(has("signature") and has("parameters"))] as $c +| [ ($a.symbol_table|length), ($c|length), + ([$c[]|(.body//{})|length]|add//0), + ([$c[]|(.cfg//[])|length]|add//0), + ([$c[]|(.cdg//[])|length]|add//0), + ([$c[]|(.ddg//[])|length]|add//0), + (($a.external_symbols//{})|length), + ([$a.call_graph[]|select(.dst|contains("/@external/")|not)]|length), + ([$a.call_graph[]|select(.dst|contains("/@external/"))]|length) ] | @tsv' + +# run_one NAME TAG LEVEL JOBS -> sets RC WALL RSS; output in $EVAL_OUT/NAME-TAG/analysis.json +run_one() { + local name=$1 tag=$2 level=$3 jobs=$4 + local dir=$EVAL_OUT/$name-$tag + rm -rf "$dir"; mkdir -p "$dir" + # shellcheck disable=SC2086 + env $APP_ENV GOTOOLCHAIN="$RUN_GO" /usr/bin/time -l -o "$dir.time" \ + "$CANGO" -i "$APP" --app-name "$name" -a "$level" -j "$jobs" \ + --eager -c "$EVAL_OUT/cache-$name" -o "$dir" >/dev/null 2>"$dir.err" & + local pid=$! waited=0 + while kill -0 "$pid" 2>/dev/null; do + if [ "$waited" -ge "$RUN_TIMEOUT" ]; then + touch "$dir.timeout" + pkill -TERM -P "$pid" 2>/dev/null; kill -TERM "$pid" 2>/dev/null + break + fi + sleep 1; waited=$((waited + 1)) + done + wait "$pid" + RC=$? + if [ -e "$dir.timeout" ]; then RC=124; fi + WALL=$(awk '/ real /{print $1}' "$dir.time") + RSS=$(awk '/maximum resident set size/{printf "%d", $1/1048576}' "$dir.time") + [ -n "$WALL" ] || WALL=n/a + [ -n "$RSS" ] || RSS=n/a +} + +BUILD_INFO=$(go version -m "$CANGO" 2>/dev/null) +CANGO_GO=$(printf '%s\n' "$BUILD_INFO" | awk 'NR==1{print $2}') +CANGO_REV=$(printf '%s\n' "$BUILD_INFO" | awk -F'=' '/vcs.revision=/{print substr($2,1,7)}') +CANGO_DIRTY=$(printf '%s\n' "$BUILD_INFO" | awk -F'=' '/vcs.modified=/{print $2}') +[ "$CANGO_DIRTY" = true ] && CANGO_REV="$CANGO_REV+dirty" +{ + echo "# L3 real-app evaluation ($DATE)" + echo + echo "Analyzer: $("$CANGO" --version 2>&1 | head -1), built with ${CANGO_GO:-unknown} (runtime.Version), rev ${CANGO_REV:-unknown}; runs use GOTOOLCHAIN=$RUN_GO; host: $(uname -sm); j=$JOBS; all runs --eager." + echo + echo "| app | status | type-check | wall a2 j1 (s) | wall a3 j1 (s) | wall a3 jN (s) | RSS a3 j1 (MB) | modules | callables | body nodes | cfg | cdg | ddg | ext symbols | cg in-project | cg external | -j identical | L3 skips | type errors | toolchain warn |" + echo "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|" +} >"$EVAL_NOTE" + +FAILS="" +for T in $TARGETS; do + APP_ENV=""; TYPECHECK="n/a (not prepared)"; label="" + case $T in + name:*) + name=${T#name:} + # shellcheck disable=SC1090 + . "$PREPARED/$name.env" + APP=$P_MODDIR; APP_ENV=$P_ENV + if [ "$P_TYPECHECK" = complete ]; then TYPECHECK=complete + else TYPECHECK="degraded ${P_TC_ERR}"; fi + case $P_PREP in skipped*|failed*) TYPECHECK="$TYPECHECK; prep $P_PREP" ;; esac + label="$name \`${P_COMMIT:0:7}\`${APP_ENV:+ ($APP_ENV)}" + ;; + dir:*) + APP=${T#dir:}; name=$(basename "$APP"); label=$name ;; + esac + echo "== $name" >&2 + status=ok + + rm -f "$EVAL_OUT/$name"-*.timeout + W3=n/a; WN=n/a; R3=n/a; RC3=0; RCN=0; RCN2=0 + # Warm-up (discarded): populates the Go build/module cache and the OS file cache so the + # timed -a 2 and -a 3 runs are comparable. + run_one "$name" warm 2 1; RCW=$RC + run_one "$name" a2j1 2 1; W2=$WALL; RC2=$RC + if [ "$RCW" -ne 124 ] && [ "$RC" -ne 124 ]; then run_one "$name" a3j1 3 1; W3=$WALL; R3=$RSS; RC3=$RC; fi + if [ "$RC3" -ne 124 ] && [ "$RC2" -ne 124 ]; then + run_one "$name" a3jN 3 "$JOBS"; WN=$WALL; RCN=$RC + if [ "$RCN" -ne 124 ]; then run_one "$name" a3jN2 3 "$JOBS"; RCN2=$RC; fi + fi + + ident=n/a + if [ "$RC2" -eq 124 ] || [ "$RC3" -eq 124 ] || [ "$RCN" -eq 124 ] || [ "$RCN2" -eq 124 ]; then + status="TIMEOUT(>${RUN_TIMEOUT}s; remaining runs skipped)" + FAILS="$FAILS $name" + elif [ "$RC3" -ne 0 ] || [ "$RCN" -ne 0 ] || [ "$RCN2" -ne 0 ] || [ "$RC2" -ne 0 ]; then + status="FAIL(rc a2=$RC2 j1=$RC3 jN=$RCN jN2=$RCN2)" + FAILS="$FAILS $name" + else + a=$EVAL_OUT/$name-a3j1/analysis.json + if cmp -s "$a" "$EVAL_OUT/$name-a3jN/analysis.json" && cmp -s "$a" "$EVAL_OUT/$name-a3jN2/analysis.json"; then + ident=yes + else + ident=NO; status="FAIL(-j differs)"; FAILS="$FAILS $name" + fi + fi + + skips=$(grep -c 'L3 skipped' "$EVAL_OUT/$name-a3j1.err" 2>/dev/null || true) + terrs=$(grep -c 'WARN\] package ' "$EVAL_OUT/$name-a3j1.err" 2>/dev/null || true) + twarn=$(grep -c 'require a newer Go version' "$EVAL_OUT/$name-a3j1.err" 2>/dev/null || true) + counts="n/a n/a n/a n/a n/a n/a n/a n/a n/a" + if [ "$HAVE_JQ" -eq 1 ] && [ "$RC3" -eq 0 ]; then + counts=$(jq -r "$JQ_COUNTS" "$EVAL_OUT/$name-a3j1/analysis.json" 2>/dev/null || echo "$counts") + fi + IFS=$'\t' read -r c1 c2 c3 c4 c5 c6 c7 c8 c9 <>"$EVAL_NOTE" + + if [ "$EVAL_PRUNE" = 1 ] && [ "$ident" = yes ]; then + rm -f "$EVAL_OUT/$name-warm/analysis.json" "$EVAL_OUT/$name-a3jN/analysis.json" \ + "$EVAL_OUT/$name-a3jN2/analysis.json" + fi +done + +echo >>"$EVAL_NOTE" +echo "Raw outputs and logs: $EVAL_OUT/-{/analysis.json,.err,.time}; preparation: $PREPARED/.{env,typecheck}" >>"$EVAL_NOTE" +echo "wrote $EVAL_NOTE" >&2 +[ -z "$FAILS" ] || { echo "failures:$FAILS" >&2; exit 1; } diff --git a/scripts/l3_metrics.py b/scripts/l3_metrics.py new file mode 100644 index 0000000..4394fe9 --- /dev/null +++ b/scripts/l3_metrics.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""l3_metrics.py - L3 (body / cfg / cdg / ddg) metrics and consistency checks. + +Usage: + python3 -I scripts/l3_metrics.py LABEL=PATH/analysis.json [LABEL=PATH ...] > metrics.jsonl + +Prints one JSON object per input: structure counts, body-node and cfg-edge kind breakdowns, +switch / select / type-switch fan-out, defer-chain and exception-edge shape, the per-callable +DDG distribution and its top callables, `var` classes (wildcard, `:` pseudo path), DDG +volume by generated-file class and by callable name (e.g. protobuf `Marshal*` / `Unmarshal`), approximate bytes per section, and the internal-consistency +checks. The figures in docs/design/notes/l3-dataflow-metrics.md come from this script. It loads +each file whole: a 1.1 GB analysis.json needs about 11 GB of RAM. +""" +import json +import os +import re +import sys +from collections import Counter, defaultdict, deque + +CFG_KINDS = ["fallthrough", "true", "false", "switch_case", "loop_back", "exception", + "return", "break", "continue", "defer_resume"] +NODE_KINDS = ["call", "statement", "branch", "loop", "switch", "return", "entry", "exit"] +IF_ERR = re.compile(rb"^if\s+(?:[^{]*;\s*)?err\s*!=\s*nil\s*\{") + + +def callables(st): + """Yields (callable, is_closure, module, file its spans index). A closure inside a + cross-file method carries no source_file of its own; it inherits the method's.""" + def nested(c, f, closure): + for x in (c.get("callables") or {}).values(): + g = x.get("source_file") or f + yield x, closure, g + yield from nested(x, g, True) + for mpath, mod in st.items(): + for fn in (mod.get("functions") or {}).values(): + f = fn.get("source_file") or mpath + yield fn, False, mpath, f + for x, cl, g in nested(fn, f, True): + yield x, cl, mpath, g + for t in (mod.get("types") or {}).values(): + for x, cl, g in nested(t, mpath, False): + yield x, cl, mpath, g + + +def file_class(path): + if path.endswith(".pb.go") or ".pb." in path: + return "protobuf" + if path.endswith(".eg.go"): + return "execgen" + if ("easyjson" in path or path.endswith("_gen.go") or "/gen/" in path + or os.path.basename(path).startswith("zz_generated")): + return "other-generated" + return "hand-written" + + +def var_class(v): + if v.startswith(":"): + return "defer" + if v.startswith("<"): + return "other-pseudo" + if "[*]" in v and v.endswith(".*"): + return "elem+deref" + if "[*]" in v: + return "elem" + if v.endswith(".*"): + return "deref/k-limit" + return "plain" + + +def pct(xs, q): + return xs[min(len(xs) - 1, int(q * (len(xs) - 1) + 0.5))] if xs else 0 + + +def analyze(label, path): + with open(path, "rb") as fh: + doc = json.load(fh) + st = doc["application"]["symbol_table"] + C = Counter() # scalar counters + K = defaultdict(Counter) # keyed counters + chk = Counter() # consistency-check failures + fanout = defaultdict(list) + ddg_per, top = [], [] + src_cache = {} + + def src(f): + if f not in src_cache: + src_cache[f] = (st.get(f, {}).get("source") or "").encode("utf-8") + return src_cache[f] + + C["source_bytes"] = sum(len((mod.get("source") or "").encode("utf-8")) for mod in st.values()) + for mod in st.values(): + for t in (mod.get("types") or {}).values(): + K["types_by_kind"][t.get("kind")] += 1 + + for c, closure, mpath, sfile in callables(st): + K["callables_by_kind"]["closure" if closure else c.get("kind")] += 1 + # A closure inside a cross-file method: its spans index the method's declaring file, + # but it carries no source_file, so the text-recovery rule would slice the wrong file. + C["closures_spans_in_other_file_without_source_file"] += ( + closure and sfile != mpath and not c.get("source_file")) + if "cfg" not in c: + K["no_cfg"][c.get("kind") + (" with body" if c.get("body") else " no body")] += 1 + continue + C["with_cfg"] += 1 + body = c.get("body") or {} + cfg, cdg, ddg = c.get("cfg") or [], c.get("cdg") or [], c.get("ddg") or [] + if len(body) == 2: + C["entry_exit_only_bodies"] += 1 + + sats = {k for k, n in body.items() if "parent" in n} + for k, n in body.items(): + K["node_kinds"][n.get("kind")] += 1 + C["deferred_calls"] += bool(n.get("is_deferred")) + C["goroutine_calls"] += bool(n.get("is_goroutine")) + C["satellite_calls"] += len(sats) + nd = sum(1 for n in body.values() if n.get("is_deferred")) + C["max_deferred_in_callable"] = max(C["max_deferred_in_callable"], nd) + # Approximate bytes of each section, re-serialized compactly as cango writes it. + for sec in ("body", "cfg", "cdg", "ddg"): + C["bytes_" + sec] += len(json.dumps(c.get(sec), separators=(",", ":"), ensure_ascii=False).encode()) + C["bytes_body_node_ids"] += sum(len(n.get("id", "")) + 7 for n in body.values()) # "id":"…", + # Call nodes exist (with their id) from L1; the other kinds are added by L3. + C["bytes_body_node_ids_l3_added"] += sum(len(n.get("id", "")) + 7 for n in body.values() + if n.get("kind") != "call") + if (sum(1 for n in body.values() if n.get("kind") == "entry") != 1 + or sum(1 for n in body.values() if n.get("kind") == "exit") != 1 + or "@entry" not in body or "@exit" not in body): + chk["entry_exit_not_exactly_one"] += 1 + + keys = set(body) + succ, pred = defaultdict(list), defaultdict(list) + seen = set() + for e in cfg: + s, t, kd = e["src"], e["dst"], e.get("kind") + K["cfg_kinds"][kd] += 1 + chk["cfg_duplicate_edge"] += (s, t, kd) in seen + seen.add((s, t, kd)) + chk["cfg_endpoint_not_body_key"] += s not in keys or t not in keys + chk["cfg_endpoint_is_satellite"] += s in sats or t in sats + succ[s].append((t, kd)) + pred[t].append(s) + sk = body.get(s, {}).get("kind") + tdef = bool(body.get(t, {}).get("is_deferred")) + if kd == "exception": + K["exception_src_kind"][sk] += 1 + C["exception_to_exit"] += t == "@exit" + if tdef: + C["exception_chain_internal" if body.get(s, {}).get("is_deferred") + else "exception_into_defer_chain"] += 1 + elif kd == "return": + K["return_edge_src_kind"][sk] += 1 + C["return_into_defer_chain"] += tdef + C["callables_with_defer_chain"] += any(e.get("kind") == "defer_resume" for e in cfg) + + for k, n in body.items(): + kd = n.get("kind") + outs = succ.get(k, []) + if kd == "return": + chk["return_node_without_one_return_edge"] += sum(1 for x in outs if x[1] == "return") != 1 + if kd in ("branch", "loop"): + K[kd + "_out_pattern"][",".join(sorted({x[1] for x in outs if x[1] != "exception"}))] += 1 + if kd == "branch": + b = n["span"]["bytes"] + C["if_err_nil_branches"] += bool(IF_ERR.match(src(sfile)[b[0]:b[0] + 400])) + if kd == "switch": + b = n["span"]["bytes"] + head = src(sfile)[b[0]:b[1]] + word = head[:6].decode("ascii", "replace") + if word == "switch" and b".(type)" in head.split(b"{", 1)[0]: + word = "type-switch" + fanout[word].append(sum(1 for x in outs if x[1] == "switch_case")) + if kd not in ("entry", "exit") and k not in sats: + C["cfg_nodes"] += 1 + C["cfg_nodes_with_exception_edge"] += any(x[1] == "exception" for x in outs) + + # Reachability: every non-satellite node from @entry, and @exit from every node. + def closure_of(start, adj): + seen_, q = {start}, deque([start]) + while q: + for v in adj(q.popleft()): + if v not in seen_: + seen_.add(v) + q.append(v) + return seen_ + nodes = keys - sats + fw = closure_of("@entry", lambda u: (v for v, _ in succ.get(u, []))) + bw = closure_of("@exit", lambda u: pred.get(u, [])) + C["reach_nodes_checked"] += len(nodes) + chk["node_unreachable_from_entry"] += len(nodes - fw) + chk["node_not_reaching_exit"] += len(nodes - bw) + + for e in cdg: + s, t = e["src"], e["dst"] + chk["cdg_endpoint_not_body_key"] += s not in keys or t not in keys + chk["cdg_endpoint_is_satellite"] += s in sats or t in sats + chk["cdg_into_exit"] += t == "@exit" + chk["cdg_self_edge"] += s == t + C["cdg_from_entry"] += s == "@entry" + kinds = {x[1] for x in succ.get(s, [])} + if s != "@entry" and not kinds & {"true", "false", "switch_case"}: + C["cdg_from_exception_only_controller"] += 1 + C["cdg"] += len(cdg) + + fc = file_class(sfile) + for e in ddg: + s, t, v = e["src"], e["dst"], e.get("var", "") + chk["ddg_endpoint_not_body_key"] += s not in keys or t not in keys + chk["ddg_endpoint_is_satellite"] += s in sats or t in sats + chk["ddg_prov_not_ssa"] += e.get("prov") != ["ssa"] + K["ddg_var_class"][var_class(v)] += 1 + # A bare base (`c`, not `c.f`) matches every reaching def under that base. + C["ddg_var_bare_base"] += not v.startswith("<") and "." not in v and "[" not in v + # Loop-carried (approximate): the def's position is at or after the use's. + if not s.startswith("@") and not t.startswith("@"): + ps = tuple(map(int, s.split("/")[0].split(":"))) + pt = tuple(map(int, t.split("/")[0].split(":"))) + C["ddg_def_at_or_after_use"] += ps >= pt + K["ddg_src_kind"][body.get(s, {}).get("kind")] += 1 + K["ddg_by_file_class"][fc] += len(ddg) + K["body_by_file_class"][fc] += len(body) + K["callables_by_file_class"][fc] += 1 + # Group by the callable's own name (last signature segment, parameters dropped), e.g. + # MarshalToSizedBuffer / Unmarshal / Size for generated protobuf methods. + nm = (c.get("signature") or "").split("(", 1)[0].rsplit(".", 1)[-1] + K["ddg_by_name"][nm] += len(ddg) + K["callables_by_name"][nm] += 1 + C["ddg"] += len(ddg) + ddg_per.append(len(ddg)) + if len(ddg) > 1000: + C["callables_ddg_gt_1000"] += 1 + C["ddg_in_callables_gt_1000"] += len(ddg) + top.append((len(ddg), c.get("id"), c)) + if len(top) > 40: + top.sort(key=lambda x: -x[0]) + del top[5:] + + ddg_per.sort() + top.sort(key=lambda x: -x[0]) + top_detail = [] + for n, _, c in top[:5]: + vc = Counter(e["var"] for e in c["ddg"]) + defs, uses = defaultdict(set), defaultdict(set) + for e in c["ddg"]: + defs[e["var"]].add(e["src"]) + uses[e["var"]].add(e["dst"]) + top_detail.append({ + "signature": c.get("signature"), "ddg": n, "body": len(c["body"]), "cfg": len(c["cfg"]), + "cyclomatic": (c.get("metrics") or {}).get("cyclomatic"), + "node_kinds": dict(Counter(b.get("kind") for b in c["body"].values())), + "distinct_vars": len(vc), + "top_vars": [[v, k, len(defs[v]), len(uses[v])] for v, k in vc.most_common(4)]}) + k1 = max(1, len(ddg_per) // 100) + return { + "app": label, "bytes": os.path.getsize(path), "modules": len(st), + "types": sum(K["types_by_kind"].values()), + "callables": sum(K["callables_by_kind"].values()), + "body_nodes": sum(K["node_kinds"].values()), + "cfg": sum(K["cfg_kinds"].values()), + **dict(C), **{k: dict(v) for k, v in K.items() + if k not in ("ddg_by_name", "callables_by_name")}, + "ddg_by_name_top": [[nm, n, K["callables_by_name"][nm]] + for nm, n in K["ddg_by_name"].most_common(10)], + "switch_fanout": {w: {"n": len(v), "mean": round(sum(v) / len(v), 2), "max": max(v)} + for w, v in fanout.items()}, + "ddg_per_callable": {"median": pct(ddg_per, .5), "p90": pct(ddg_per, .9), + "p99": pct(ddg_per, .99), "max": ddg_per[-1] if ddg_per else 0, + "zero": sum(1 for x in ddg_per if x == 0), + "top1pct_share": round(sum(ddg_per[-k1:]) / max(1, C["ddg"]), 4)}, + "ddg_top": top_detail, + "check_failures": {k: v for k, v in chk.items() if v}, + } + + +if __name__ == "__main__": + for arg in sys.argv[1:]: + label, path = arg.split("=", 1) + print(json.dumps(analyze(label, path)), flush=True) diff --git a/scripts/load_and_query.sh b/scripts/load_and_query.sh index 80a2767..07d77d4 100755 --- a/scripts/load_and_query.sh +++ b/scripts/load_and_query.sh @@ -94,7 +94,7 @@ queries() { q "relationship counts by type" \ "MATCH ()-[r]->() RETURN type(r) AS rel, count(*) AS n ORDER BY n DESC" q "everything scoped to the cobra app (prefix seek)" \ - "MATCH (n:GoCanNode) WHERE n.id STARTS WITH 'can://go/cobra' RETURN count(n) AS cobra_nodes" + "MATCH (n:GoCanNode) WHERE n.id STARTS WITH 'can://cobra/' RETURN count(n) AS cobra_nodes" q "sample call graph edges" \ "MATCH (a:GoCallable)-[:GO_CALLS]->(b) RETURN a.signature AS caller, b.signature AS callee LIMIT 15" q "types and how many methods each has" \ diff --git a/testdata/dataflow/flow/flow.go b/testdata/dataflow/flow/flow.go new file mode 100644 index 0000000..5c392f8 --- /dev/null +++ b/testdata/dataflow/flow/flow.go @@ -0,0 +1,559 @@ +// Package flow is the L3 fixture for the dataflow train: one named function per +// control-flow / dataflow construct. Tests pin body-node keys by line:col, so +// keep each statement on its own line and append new functions at the end. +package flow + +func f(x int) int { return x } + +func g(x int) int { return x } + +func cleanup() {} + +func work(x int) { _ = x } + +// IfElse is an `if` with an init statement. +func IfElse(a int) int { + if b := a * 2; b > 10 { + return b + } else { + a = b + } + return a +} + +// ForLoop is a 3-clause `for`. +func ForLoop(n int) int { + s := 0 + for i := 0; i < n; i++ { + s += i + } + return s +} + +// Infinite is a `for {}` left only by a return. +func Infinite(ch chan int) { + for { + v := <-ch + if v < 0 { + return + } + } +} + +// Range is a `for … range`. +func Range(xs []int) int { + s := 0 + for _, x := range xs { + s += x + } + return s +} + +// Switch has a `fallthrough`. +func Switch(a int) int { + r := 0 + switch a { + case 1: + r = 1 + fallthrough + case 2: + r += 2 + default: + r = -1 + } + return r +} + +// TypeSwitch is a type switch. +func TypeSwitch(v any) string { + switch t := v.(type) { + case int: + _ = t + return "int" + case string: + return t + default: + return "" + } +} + +// Select2 selects over two channels. +func Select2(a, b chan int) int { + select { + case x := <-a: + return x + case b <- 1: + return 0 + } +} + +// Labeled has a labeled `break` and `continue`. +func Labeled(m [][]int) int { + s := 0 +outer: + for _, row := range m { + for _, v := range row { + if v < 0 { + continue outer + } + if v == 0 { + break outer + } + s += v + } + } + return s +} + +// GotoBack is a backward `goto`. +func GotoBack(n int) int { + i := 0 +loop: + i++ + if i < n { + goto loop + } + return i +} + +// EmptySelect blocks forever. +func EmptySelect() { + select {} +} + +// EarlyReturnDefer has two defers and an early return. +func EarlyReturnDefer(a int) int { + defer cleanup() + defer work(g(a)) + if a < 0 { + return -1 + } + cleanup() + return a +} + +// Panics has an explicit panic and a slice index. +func Panics(xs []int, i int) int { + if i < 0 { + panic("negative") + } + v := xs[i] + return v +} + +// Bump has the loop-carried y = y + 1. +func Bump(n int) int { + y := 0 + for i := 0; i < n; i++ { + y = y + 1 + } + return y +} + +// Shadow shadows x in an inner block. +func Shadow(a int) int { + x := a + if a > 0 { + x := a * 2 + _ = x + } + return x +} + +type H struct{ i int } + +type G struct { + h H + x int +} + +type F struct{ g G } + +type P struct{ f F } + +// Fields writes a deep access path, then reads a sibling path. +func Fields(p *P, v int) int { + p.f.g.h.i = v + return p.f.g.x +} + +// Chan sends and receives on a channel. +func Chan(ch chan int, v int) int { + ch <- v + w := <-ch + return w +} + +// Unreachable has statements after a return. +func Unreachable(a int) int { + return a + a++ + return a + 1 +} + +// Nested is a call nested in a call inside an assignment. +func Nested(y int) int { + x := f(g(y)) + return x +} + +// Spawn launches a goroutine. +func Spawn(x int) { + go work(f(x)) +} + +// get returns a *P so a statement can start with a call. +func get() *P { return &P{} } + +// CallLHS is a statement whose first token starts a call: the assignment and +// the call share a position. +func CallLHS(v int) { + get().f.g.x = v +} + +// Closure has a function literal whose body is its own callable. +func Closure(x int) int { + h := func() int { + return f(x) + } + return h() +} + +// S has chainable methods. +type S struct{} + +// A returns s so calls can chain. +func (s *S) A() *S { return s } + +// B ends the chain. +func (s *S) B() {} + +// Chain is a bare call statement whose call is co-positioned with the call +// nested in it: s.A().B() is "l:c" (the statement's node), s.A() is "l:c/2". +func Chain(s *S) { + s.A().B() +} + +// SwitchNoDefault has no default; its fallthrough enters an empty clause. +func SwitchNoDefault(a int) int { + switch a { + case 1: + a = 2 + fallthrough + case 2: + } + return a +} + +// BreakContinue has an unlabeled break and continue in a cond-only for. +func BreakContinue(n int) int { + for n > 0 { + n-- + if n == 5 { + continue + } + if n == 2 { + break + } + } + return n +} + +// GotoBlock jumps to a labeled block (target: its first statement) and to a +// labeled empty block (target: the block's continuation). +func GotoBlock(a int) int { + if a > 0 { + goto M + } + a-- +M: + { + a++ + } + if a > 5 { + goto E + } + a = 0 +E: + { + } + return a +} + +// GotoEnd jumps to a label on the implicit empty statement before `}`. +func GotoEnd(a int) { + if a > 0 { + goto L + } + a++ +L: +} + +// Spin has loops with empty bodies. +func Spin(n int) { + for n > 0 { + } + for { + } +} + +// LabeledSwitchBreak leaves a labeled switch with `break Sw` from a nested for. +func LabeledSwitchBreak(a, n int) int { +Sw: + switch a { + case 1: + for n > 0 { + if n == 3 { + break Sw + } + n-- + } + a = n + } + return a +} + +// LabeledSelectBreak leaves a labeled select with `break Sel`. +func LabeledSelectBreak(ch chan int, a int) int { +Sel: + select { + case v := <-ch: + if v > 0 { + break Sel + } + a = v + } + return a +} + +// ContinueInSwitch continues the enclosing for from inside a switch. +func ContinueInSwitch(n int) int { + for n > 0 { + n-- + switch n { + case 2: + continue + } + n-- + } + return n +} + +// BreakInSelect breaks out of a select, not out of the enclosing for. +func BreakInSelect(ch chan int, n int) int { + for n > 0 { + select { + case v := <-ch: + if v == 0 { + break + } + n = v + } + n-- + } + return n +} + +// DefaultMiddle has its default between two cases, with fallthrough across it. +func DefaultMiddle(a int) int { + r := 0 + switch a { + case 1: + r = 1 + fallthrough + default: + r += 2 + fallthrough + case 3: + r += 3 + } + return r +} + +// TypeSwitchNil has a `case nil:` clause. +func TypeSwitchNil(v any) int { + switch v.(type) { + case nil: + return 0 + case int: + return 1 + } + return 2 +} + +// GotoInLoop jumps backward inside a loop body. +func GotoInLoop(n int) int { + for i := 0; i < n; i++ { + again: + n-- + if n > 10 { + goto again + } + } + return n +} + +// GotoForever loops by a backward goto with no way out, under a defer. +func GotoForever(n int) { + defer cleanup() +L: + n++ + goto L +} + +// SelectForever blocks in select {} under a defer. +func SelectForever() { + defer cleanup() + select {} +} + +// PanicLast ends in a panic, so nothing falls off the end. +func PanicLast(xs []int, i int) { + xs[i] = 1 + panic(i) +} + +// ShadowPanic calls a local named panic: an ordinary call, not the builtin. +func ShadowPanic() int { + panic := func(int) {} + panic(1) + return 2 +} + +// DeadDefer has a defer after a return: it is never registered. +func DeadDefer() { + return + defer cleanup() +} + +// CondDefer registers a defer on one branch only. +func CondDefer(a int) int { + if a > 0 { + defer cleanup() + } + return a +} + +// LoopDefer registers a defer on every iteration. +func LoopDefer(n int) { + for i := 0; i < n; i++ { + defer work(i) + } +} + +// LoopPanic panics in a nested block inside a loop. +func LoopPanic(xs []int) int { + s := 0 + for _, x := range xs { + if x < 0 { + { + panic(x) + } + } + s += x + } + return s +} + +// AfterPanic has statements, and a defer, after a panic. +func AfterPanic(a int) int { + defer cleanup() + panic(a) + a++ + defer work(a) + return a +} + +// Guard returns r, which depends on c only through control: a backward slice +// from the return reaches the if header by the CDG alone. +func Guard(c, a int) int { + r := 0 + d := a + 1 + if c > 0 { + r = a + } + _ = d + return r +} + +// reader is a local io.Reader (the fixture imports nothing). +type reader interface{ Read(p []byte) (int, error) } + +func readFull(r reader, b []byte) { _, _ = r.Read(b) } + +// SubsliceRead reads into buf through the subslice buf[:n]. +func SubsliceRead(r reader, n int) byte { + buf := make([]byte, 16) + r.Read(buf[:n]) + return buf[0] +} + +// ArraySliceRead reads into an array through arr[:]. +func ArraySliceRead(r reader) byte { + var arr [8]byte + readFull(r, arr[:]) + return arr[0] +} + +// CopyInto copies into dst through the subslice dst[i:]. +func CopyInto(dst, src []byte, i int) byte { + dst = dst[:len(dst):len(dst)] + copy(dst[i:], src) + return dst[0] +} + +// ClearAll clears a map and a subslice. +func ClearAll(m map[int]int, s []int, n int) int { + clear(m) + clear(s[:n]) + return m[0] + s[0] +} + +type flusher struct{ buf []byte } + +func (f *flusher) Flush() {} + +func fill(b []byte) { _ = b } + +// DeferFlush has deferred calls that read, at exit, state written after +// their registration: w.buf through the receiver, b's elements through the +// argument. +func DeferFlush(w *flusher, b []byte) { + defer w.Flush() + defer fill(b[1:]) + w.buf = nil + b[0] = 1 +} + +func h() int { return 0 } + +// GoArgs launches goroutines whose arguments hold calls: g(x) and h() run at +// the go statement, in the launching goroutine; only the launched call is a +// goroutine. +func GoArgs(x int) { + go f(g(x)) + go func(y int) { _ = y }(h()) +} + +// AppendReuse appends into dst's backing array through dst[:0]: the append +// writes dst's elements, which the later read sees. +func AppendReuse(dst []int, x int) []int { + out := append(dst[:0], x) + _ = dst[0] + return out +} + +func wrapErr(err error) error { return err } + +func observe(err error) { _ = err } + +// DeferWrap has a deferred closure that rewrites the named result err (R11). +// The chain runs the rewrite, then the observer, then @exit. Deferred nodes +// never strong-define, and L3 records no def for the closure's write to its +// captured err, so the earlier def of err still reaches the observer. +func DeferWrap() (err error) { + err = wrapErr(nil) + defer func() { observe(err) }() + defer func() { err = wrapErr(err) }() + return +} diff --git a/testdata/dataflow/generic/a.go b/testdata/dataflow/generic/a.go new file mode 100644 index 0000000..75d1818 --- /dev/null +++ b/testdata/dataflow/generic/a.go @@ -0,0 +1,10 @@ +package generic + +import "sync/atomic" + +// A and B are two distinct instantiation arguments for atomic.Pointer. +type A struct{ n int } + +var pa atomic.Pointer[A] + +func StoreA() { pa.Store(nil) } diff --git a/testdata/dataflow/generic/b.go b/testdata/dataflow/generic/b.go new file mode 100644 index 0000000..713f17b --- /dev/null +++ b/testdata/dataflow/generic/b.go @@ -0,0 +1,9 @@ +package generic + +import "sync/atomic" + +type B struct{ s string } + +var pb atomic.Pointer[B] + +func StoreB() { pb.Store(nil) } diff --git a/testdata/dataflow/go.mod b/testdata/dataflow/go.mod new file mode 100644 index 0000000..f291d31 --- /dev/null +++ b/testdata/dataflow/go.mod @@ -0,0 +1,5 @@ +module example.com/dataflow + +go 1.21 + +require example.com/vlib v0.0.0 diff --git a/testdata/dataflow/inits/a.go b/testdata/dataflow/inits/a.go new file mode 100644 index 0000000..33733b0 --- /dev/null +++ b/testdata/dataflow/inits/a.go @@ -0,0 +1,14 @@ +// Package inits has a func init() in more than one file. Every init shares the +// signature example.com/dataflow/inits.init, but each is its own callable. +package inits + +import "os" + +var mode string + +func init() { + setup() + mode = os.Getenv("INITS_MODE") +} + +func setup() { mode = "default" } diff --git a/testdata/dataflow/inits/b.go b/testdata/dataflow/inits/b.go new file mode 100644 index 0000000..94b3137 --- /dev/null +++ b/testdata/dataflow/inits/b.go @@ -0,0 +1,8 @@ +package inits + +import "os" + +func init() { + setup() + _ = os.Getenv("INITS_DEBUG") +} diff --git a/testdata/dataflow/l1/l1.go b/testdata/dataflow/l1/l1.go new file mode 100644 index 0000000..c3b2640 --- /dev/null +++ b/testdata/dataflow/l1/l1.go @@ -0,0 +1,45 @@ +// Package l1 is the L1/L2 fixture for the dataflow train: type kinds, defer/go +// markers, and co-positioned (chained) calls. +package l1 + +// Celsius is a defined type over a non-struct. +type Celsius float64 + +// Alias is a type alias. +type Alias = Celsius + +// Base is a plain struct. +type Base struct{ N int } + +// MyErr is defined over a struct, so its kind is struct (underlying type). +type MyErr Base + +// I1 is an interface. +type I1 interface{ M() } + +// I2 is defined over an interface, so its kind is interface (underlying type). +type I2 I1 + +// S is a struct whose methods chain. +type S struct{} + +// A returns a *S so calls can be chained. +func (s *S) A() *S { return s } + +// B terminates the chain. +func (s *S) B() {} + +func cleanup() {} + +func work() {} + +// D exercises defer and go call markers. +func D() { + defer cleanup() + go work() +} + +// Chain has two calls that start at the same line and column. +func Chain(s *S) { + s.A().B() +} diff --git a/testdata/dataflow/linedir/linedir.go b/testdata/dataflow/linedir/linedir.go new file mode 100644 index 0000000..aa68c34 --- /dev/null +++ b/testdata/dataflow/linedir/linedir.go @@ -0,0 +1,16 @@ +// Package linedir has calls under a //line directive, the shape of generated +// code (goyacc, templ): L1 records them at their virtual positions, far past +// the file's real line count, and L2 must still resolve them. +package linedir + +import "os" + +func helper() string { return "" } + +// Caller's calls sit on virtual lines of virtual.y. +func Caller() { +//line virtual.y:900:1 + os.Getenv(helper()) +//line linedir.go:15:1 + helper() +} diff --git a/testdata/dataflow/vend/vend.go b/testdata/dataflow/vend/vend.go new file mode 100644 index 0000000..a4bbc8e --- /dev/null +++ b/testdata/dataflow/vend/vend.go @@ -0,0 +1,8 @@ +// Package vend calls into the vendored dependency vlib. +package vend + +import "example.com/vlib" + +func CallVendored() int { return vlib.F() } + +func CallUnnamed() bool { return vlib.G(1, "a") && vlib.H(2, "b") != "" } diff --git a/testdata/dataflow/vendor/example.com/vlib/vlib.go b/testdata/dataflow/vendor/example.com/vlib/vlib.go new file mode 100644 index 0000000..9db85eb --- /dev/null +++ b/testdata/dataflow/vendor/example.com/vlib/vlib.go @@ -0,0 +1,11 @@ +// Package vlib is a vendored dependency of the dataflow fixture. +package vlib + +// F is a library function reached through the vendor directory. +func F() int { return 1 } + +// G has unnamed parameters: its external symbol names them _0, _1. +func G(int, string) bool { return true } + +// H has a blank parameter: it is named by position too (_0), n keeps its name. +func H(_ int, n string) string { return n } diff --git a/testdata/dataflow/vendor/modules.txt b/testdata/dataflow/vendor/modules.txt new file mode 100644 index 0000000..06b192c --- /dev/null +++ b/testdata/dataflow/vendor/modules.txt @@ -0,0 +1,3 @@ +# example.com/vlib v0.0.0 +## explicit +example.com/vlib diff --git a/testdata/dataflow/xfile/a.go b/testdata/dataflow/xfile/a.go new file mode 100644 index 0000000..04a5865 --- /dev/null +++ b/testdata/dataflow/xfile/a.go @@ -0,0 +1,6 @@ +// Package xfile declares a type in a.go and a method on it in b.go; the +// method holds a closure and a nested closure (source_file, R19). +package xfile + +// T is declared here; its method Run is declared in b.go. +type T struct{ n int } diff --git a/testdata/dataflow/xfile/b.go b/testdata/dataflow/xfile/b.go new file mode 100644 index 0000000..d365bd9 --- /dev/null +++ b/testdata/dataflow/xfile/b.go @@ -0,0 +1,14 @@ +package xfile + +// Run is declared apart from its receiver type T (a.go). +func (t *T) Run() int { + add := func(d int) int { + t.n += d + inc := func() { + t.n++ + } + inc() + return t.n + } + return add(1) +} diff --git a/testdata/futurego/a/a.go b/testdata/futurego/a/a.go new file mode 100644 index 0000000..1039f25 --- /dev/null +++ b/testdata/futurego/a/a.go @@ -0,0 +1,10 @@ +//go:build go1.99 + +// Package a needs a Go newer than any toolchain. The go command excludes it +// unless the go1.99 tag is set (the test sets it through GOFLAGS); then +// go/types reports "file requires newer Go version go1.99", the error an +// analyzer built with an older Go hits on a module with a newer go line (a +// go.mod line of 1.99 itself is refused by the go command before any load). +package a + +func A() int { return 1 } diff --git a/testdata/futurego/b/b.go b/testdata/futurego/b/b.go new file mode 100644 index 0000000..dbfe179 --- /dev/null +++ b/testdata/futurego/b/b.go @@ -0,0 +1,6 @@ +//go:build go1.98 + +// Package b needs go1.98: a second affected package, older than a's. +package b + +func B() int { return 2 } diff --git a/testdata/futurego/c/c.go b/testdata/futurego/c/c.go new file mode 100644 index 0000000..3bb479e --- /dev/null +++ b/testdata/futurego/c/c.go @@ -0,0 +1,4 @@ +// Package c needs nothing new: it is not affected. +package c + +func C() int { return 3 } diff --git a/testdata/futurego/go.mod b/testdata/futurego/go.mod new file mode 100644 index 0000000..de3b405 --- /dev/null +++ b/testdata/futurego/go.mod @@ -0,0 +1,3 @@ +module example.com/futurego + +go 1.21 diff --git a/testdata/malformed/case.go b/testdata/malformed/case.go new file mode 100644 index 0000000..f1f8541 --- /dev/null +++ b/testdata/malformed/case.go @@ -0,0 +1,10 @@ +package malformed + +// GoodCase is well formed. +func GoodCase(x int) int { return x + 1 } + +func badCase(x int) { + switch x { + case 1, : + } +} diff --git a/testdata/malformed/doc.go b/testdata/malformed/doc.go new file mode 100644 index 0000000..232f328 --- /dev/null +++ b/testdata/malformed/doc.go @@ -0,0 +1,7 @@ +// Package malformed holds syntax errors the parser recovers from with a +// function node whose End lies one past the end of its file: in the FileSet +// that position has no token.File (the last file) or belongs to the next +// file. Each file holds one reproducer after a well-formed function, which +// the analyzer must still emit; recovery from the error swallows the rest of +// the file, so each reproducer gets its own file. +package malformed diff --git a/testdata/malformed/go.mod b/testdata/malformed/go.mod new file mode 100644 index 0000000..d2faac2 --- /dev/null +++ b/testdata/malformed/go.mod @@ -0,0 +1,3 @@ +module example.com/malformed + +go 1.21 diff --git a/testdata/malformed/if.go b/testdata/malformed/if.go new file mode 100644 index 0000000..a31550a --- /dev/null +++ b/testdata/malformed/if.go @@ -0,0 +1,8 @@ +package malformed + +// GoodIf is well formed. +func GoodIf(x int) int { return x + 1 } + +func badIf() { + if := 1 +} diff --git a/testdata/malformed/range.go b/testdata/malformed/range.go new file mode 100644 index 0000000..1e7f651 --- /dev/null +++ b/testdata/malformed/range.go @@ -0,0 +1,9 @@ +package malformed + +// GoodRange is well formed. +func GoodRange(x int) int { return x + 1 } + +func badRange() { + for range { + } +} diff --git a/testdata/malformed/recv.go b/testdata/malformed/recv.go new file mode 100644 index 0000000..f089770 --- /dev/null +++ b/testdata/malformed/recv.go @@ -0,0 +1,10 @@ +package malformed + +// GoodRecv is well formed. +func GoodRecv(x int) int { return x + 1 } + +func badRecv(c chan int) { + select { + case <-: + } +} diff --git a/testdata/typeerr/go.mod b/testdata/typeerr/go.mod new file mode 100644 index 0000000..f70c6d5 --- /dev/null +++ b/testdata/typeerr/go.mod @@ -0,0 +1,3 @@ +module example.com/typeerr + +go 1.21 diff --git a/testdata/typeerr/typeerr.go b/testdata/typeerr/typeerr.go new file mode 100644 index 0000000..fd2949f --- /dev/null +++ b/testdata/typeerr/typeerr.go @@ -0,0 +1,32 @@ +// Package typeerr does not type-check: it imports a package that does not +// exist, so missing.Req is an invalid type and go/types records no field +// selections on it. L3 must still see the writes to its fields. +package typeerr + +import "example.com/typeerr/missing" + +// Forward writes a field of a value of the unresolved type and reads it back. +func Forward(src string) string { + var req missing.Req + req.Body = src + return req.Body +} + +// Undeclared writes a field of a name that is declared nowhere: the base +// itself cannot be resolved. +func Undeclared(src string) string { + undeclared.Body = src + return undeclared.Body +} + +// Qualified writes a field of a variable of the missing package. +func Qualified(src string) string { + missing.Cfg.Body = src + return missing.Cfg.Body +} + +// Asserted writes through a type assertion, a target pathOf cannot walk. +func Asserted(x any, src string) any { + x.(*missing.Req).Body = src + return x +}