From cf683d584faeee9232bcd26dd10087f999b6ffbb Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Tue, 20 Oct 2026 12:29:21 +0200 Subject: [PATCH 1/2] feat(trace): e2e adapter for Pi Add the Pi harness to the Trace end-to-end suite and a Pi column to the feature tables. Trace does not support Pi yet, so each Pi cell is No and the suite skips it until Pi support is added. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/internal/trace/README.md | 96 ++++++------- app/cli/internal/trace/e2e/checks_more.go | 8 +- app/cli/internal/trace/e2e/claude.go | 4 +- app/cli/internal/trace/e2e/config.yaml | 4 + app/cli/internal/trace/e2e/e2e_test.go | 1 + app/cli/internal/trace/e2e/harness.go | 7 + app/cli/internal/trace/e2e/opencode.go | 4 +- app/cli/internal/trace/e2e/pi.go | 157 ++++++++++++++++++++++ 8 files changed, 229 insertions(+), 52 deletions(-) create mode 100644 app/cli/internal/trace/e2e/pi.go diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 906ca71ba..d7fc491ef 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -8,15 +8,17 @@ In the tables, **Yes** means supported, **Partial** means supported with limits, Claude Code has the most complete support. +Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and `trace run --pi` do not exist, so no Pi session is recorded. Each Pi cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Pi support is added, set the Pi cells to **Unknown** and run the end-to-end suite to find the values. + ## Setup How Trace is added to a repository and to an agent. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | -|---------|---------------|-------------|--------|--------------|------------| -| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Yes | -| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Yes | -| More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | +|---------|---------------|-------------|--------|--------------|------------|---| +| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Yes | No | +| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Yes | No | +| More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | No | Why not Yes: @@ -27,13 +29,13 @@ Why not Yes: What the session evidence holds about the conversation. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | -|---------|---------------|-------------|--------|--------------|------------| -| Transcript | The evidence holds the conversation (`raw_session`): the messages and the tool calls of the agent. | Yes | Yes | Partial | Partial | -| Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | -| Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | -| Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | -| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | +|---------|---------------|-------------|--------|--------------|------------|---| +| Transcript | The evidence holds the conversation (`raw_session`): the messages and the tool calls of the agent. | Yes | Yes | Partial | Partial | No | +| Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | No | +| Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | No | +| Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | No | +| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | No | Why not Yes: @@ -48,15 +50,15 @@ Why not Yes: Which lines of a commit the AI wrote. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | -|---------|---------------|-------------|--------|--------------|------------| -| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | -| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | -| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence does not report them yet (spec issue-3639). | Yes | Partial | Yes | Yes | -| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | -| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the sum of the removals of the session's commits (spec issue-3639). | Partial | Partial | Partial | Partial | -| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | -| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | +|---------|---------------|-------------|--------|--------------|------------|---| +| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | +| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | +| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence does not report them yet (spec issue-3639). | Yes | Partial | Yes | Yes | No | +| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | +| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the sum of the removals of the session's commits (spec issue-3639). | Partial | Partial | Partial | Partial | No | +| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | +| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial | No | Why not Yes: @@ -71,11 +73,11 @@ Why not Yes: What the user sees in the agent. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | -|---------|---------------|-------------|--------|--------------|------------| -| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | -| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | -| Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | +|---------|---------------|-------------|--------|--------------|------------|---| +| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | +| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | +| Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | No | Why not Yes: @@ -88,15 +90,15 @@ Why not Yes: What the session was asked to build, and which skills it used. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | -|---------|---------------|-------------|--------|--------------|------------| -| Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | -| Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | -| Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | -| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Unknown | Unknown | Yes | -| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | -| Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | -| Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | +|---------|---------------|-------------|--------|--------------|------------|---| +| Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | No | +| Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | +| Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | +| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Unknown | Unknown | Yes | No | +| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | +| Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | +| Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | Why not Yes: @@ -111,11 +113,11 @@ Why not Yes: How the evidence is protected before it leaves the machine. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | -|---------|---------------|-------------|--------|--------------|------------| -| Secret redaction | Secrets in the session evidence and in the spec materials are replaced with a placeholder before upload. | Yes | Yes | Yes | Yes | -| Media skipped by redaction | The secret redaction does not scan base64 media (images, PDFs) in the transcript, so it is faster and does not break the media. | Yes | Unknown | Not applicable | Not applicable | -| Allow markers in redaction | A secret on a line that carries an inline `gitleaks:allow` or `betterleaks:allow` marker, such as a test fixture the agent read, is not redacted. The marker covers only its own line. | Yes | Yes | Yes | Yes | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | +|---------|---------------|-------------|--------|--------------|------------|---| +| Secret redaction | Secrets in the session evidence and in the spec materials are replaced with a placeholder before upload. | Yes | Yes | Yes | Yes | No | +| Media skipped by redaction | The secret redaction does not scan base64 media (images, PDFs) in the transcript, so it is faster and does not break the media. | Yes | Unknown | Not applicable | Not applicable | No | +| Allow markers in redaction | A secret on a line that carries an inline `gitleaks:allow` or `betterleaks:allow` marker, such as a test fixture the agent read, is not redacted. The marker covers only its own line. | Yes | Yes | Yes | Yes | No | Why not Yes: @@ -126,12 +128,12 @@ Why not Yes: How the sessions are linked to the commits and attested. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | -|---------|---------------|-------------|--------|--------------|------------| -| Commit trailer | Each commit gets a `Chainloop-Trace-Sessions` trailer with the sessions that contributed to it. | Yes | Yes | Yes | Yes | -| Attestation at push | In the default control-plane storage mode, the `pre-push` hook attests the sessions of the pushed commits to the control plane. | Yes | Yes | Yes | Yes | -| Local storage at push | In a repository with `traceStorage: LOCAL` in `.chainloop.yml`, the `pre-push` hook runs the same assembly as an attestation. It writes the session evidence set to the `local-sessions` folder of the trace directory and sends no attestation. It needs no control plane, login, or network. The stored set is equal to the upload of a push, except for the signature. A `statement.json` file indexes the set: the in-toto statement that a push would sign, with the `chainloop.dev/attestation/v0.2` predicate. It has no envelope and no signature, so it is not an attestation. `trace init --local` sets up the mode. `trace run --local` stores one wrapped session the same way, with no `.chainloop.yml` setting. Redaction is on by default. Set `CHAINLOOP_TRACE_NO_REDACT` to turn it off, for a trusted local store only (spec issue-3582). | Yes | Yes | Yes | Yes | -| Shell commands in another checkout | A session can work in a checkout other than the one it started in through shell commands only, for example in a fresh clone. The checkout is found from the command text: the target of `cd` or `pushd`, the path of `git -C`, and the folder that `git clone` or `gh repo clone` creates. The git hooks are installed there and the session is recorded there, the same as for a file tool edit in that checkout. A clone gets the git hooks only after the command that created it, so a commit in that same command does not carry the session. For OpenCode, a plugin installed by an earlier version does not send the command text: install the hooks again to update it. | Yes | Partial | Yes | Yes | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | +|---------|---------------|-------------|--------|--------------|------------|---| +| Commit trailer | Each commit gets a `Chainloop-Trace-Sessions` trailer with the sessions that contributed to it. | Yes | Yes | Yes | Yes | No | +| Attestation at push | In the default control-plane storage mode, the `pre-push` hook attests the sessions of the pushed commits to the control plane. | Yes | Yes | Yes | Yes | No | +| Local storage at push | In a repository with `traceStorage: LOCAL` in `.chainloop.yml`, the `pre-push` hook runs the same assembly as an attestation. It writes the session evidence set to the `local-sessions` folder of the trace directory and sends no attestation. It needs no control plane, login, or network. The stored set is equal to the upload of a push, except for the signature. A `statement.json` file indexes the set: the in-toto statement that a push would sign, with the `chainloop.dev/attestation/v0.2` predicate. It has no envelope and no signature, so it is not an attestation. `trace init --local` sets up the mode. `trace run --local` stores one wrapped session the same way, with no `.chainloop.yml` setting. Redaction is on by default. Set `CHAINLOOP_TRACE_NO_REDACT` to turn it off, for a trusted local store only (spec issue-3582). | Yes | Yes | Yes | Yes | No | +| Shell commands in another checkout | A session can work in a checkout other than the one it started in through shell commands only, for example in a fresh clone. The checkout is found from the command text: the target of `cd` or `pushd`, the path of `git -C`, and the folder that `git clone` or `gh repo clone` creates. The git hooks are installed there and the session is recorded there, the same as for a file tool edit in that checkout. A clone gets the git hooks only after the command that created it, so a commit in that same command does not carry the session. For OpenCode, a plugin installed by an earlier version does not send the command text: install the hooks again to update it. | Yes | Partial | Yes | Yes | No | ### Switching storage modes @@ -159,7 +161,7 @@ All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. ## End-to-end tests -The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code and OpenCode 2 columns. OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace, because OpenCode takes its project folder from `PWD`. The OpenCode 2 model in `e2e/config.yaml` is an OpenRouter model, so the run needs `OPENROUTER_API_KEY`. +The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code, OpenCode 2, and Pi columns. The Pi cells are **No**, so the suite skips them until Pi support is added. OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace, because OpenCode takes its project folder from `PWD`. Pi runs with `pi -p --mode json --approve`, with its configuration and session folders next to the workspace. The OpenCode 2 and Pi models in `e2e/config.yaml` are OpenRouter models, so the run needs `OPENROUTER_API_KEY`. A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows: diff --git a/app/cli/internal/trace/e2e/checks_more.go b/app/cli/internal/trace/e2e/checks_more.go index ebcb8c5fb..c368582a1 100644 --- a/app/cli/internal/trace/e2e/checks_more.go +++ b/app/cli/internal/trace/e2e/checks_more.go @@ -143,8 +143,14 @@ func checkGenericSessionStored(s *Session) error { // checkNoSessionStartHook makes sure that the session-start hook did not // fire, so that the scenario tests a missed session start. func checkNoSessionStartHook(s *Session) error { - if _, err := hookMessage(s, "SessionStart"); err == nil { + _, err := hookMessage(s, "SessionStart") + switch { + case err == nil: return errors.New("the session-start hook fired, so the scenario does not test a missed start") + case errors.Is(err, errNotCovered): + // A harness that reports no hook messages cannot show that the hook + // did not fire. + return err } return nil } diff --git a/app/cli/internal/trace/e2e/claude.go b/app/cli/internal/trace/e2e/claude.go index 1d1cbc8b1..1b3463518 100644 --- a/app/cli/internal/trace/e2e/claude.go +++ b/app/cli/internal/trace/e2e/claude.go @@ -74,7 +74,7 @@ func (c *Claude) Version(ctx context.Context) (string, error) { func (c *Claude) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { args := []string{ "-p", req.Prompt, - "--model", req.Model, + flagModel, req.Model, "--output-format", "stream-json", "--verbose", "--include-hook-events", @@ -181,7 +181,7 @@ func (c *Claude) HookConfig() HookConfig { } } `, - UserMarker: "e2e-user-hook", + UserMarker: userHookMarker, } } diff --git a/app/cli/internal/trace/e2e/config.yaml b/app/cli/internal/trace/e2e/config.yaml index 937becf5c..ebf721981 100644 --- a/app/cli/internal/trace/e2e/config.yaml +++ b/app/cli/internal/trace/e2e/config.yaml @@ -14,6 +14,10 @@ harnesses: version: "2.0.22" models: - openrouter/anthropic/claude-haiku-5.5 + Pi: + version: "1.3.0" + models: + - openrouter/anthropic/claude-haiku-5.5 # Runs and pass threshold of a cell whose feature depends on the model # following an instruction (R-006, D-004). A cell of any other feature runs diff --git a/app/cli/internal/trace/e2e/e2e_test.go b/app/cli/internal/trace/e2e/e2e_test.go index 5043e92a2..599e5d6db 100644 --- a/app/cli/internal/trace/e2e/e2e_test.go +++ b/app/cli/internal/trace/e2e/e2e_test.go @@ -130,6 +130,7 @@ func TestTraceMatrix(t *testing.T) { var harnesses = map[string]Harness{ HarnessClaudeCode: &Claude{}, "OpenCode 2": &OpenCode{Major: "2"}, + "Pi": &Pi{}, } // configuredHarnesses returns the adapters of the harnesses that the suite diff --git a/app/cli/internal/trace/e2e/harness.go b/app/cli/internal/trace/e2e/harness.go index ac88ceb3d..aa33a676c 100644 --- a/app/cli/internal/trace/e2e/harness.go +++ b/app/cli/internal/trace/e2e/harness.go @@ -20,6 +20,13 @@ import "context" // HarnessClaudeCode is the column name of Claude Code in the feature table. const HarnessClaudeCode = "Claude Code" +const ( + // flagModel is the model flag that all the harnesses share. + flagModel = "--model" + // userHookMarker is in the hook of the user that a scenario seeds. + userHookMarker = "e2e-user-hook" +) + // RunRequest is one headless session of a harness. type RunRequest struct { // Dir is the repository the session runs in. diff --git a/app/cli/internal/trace/e2e/opencode.go b/app/cli/internal/trace/e2e/opencode.go index f1750c80a..15550c09f 100644 --- a/app/cli/internal/trace/e2e/opencode.go +++ b/app/cli/internal/trace/e2e/opencode.go @@ -94,7 +94,7 @@ func (o *OpenCode) Version(ctx context.Context) (string, error) { // as synthetic messages, which the run output does not show. Run exports the // session after the run and reads them from the export. func (o *OpenCode) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { - args := []string{"run", "--standalone", "--format", "json", "--auto", "--model", req.Model} + args := []string{"run", "--standalone", "--format", "json", "--auto", flagModel, req.Model} // A pasted image goes in as an attached file, from outside the // repository, so that the agent cannot commit it. @@ -274,7 +274,7 @@ export default { async setup() {}, } `, - UserMarker: "e2e-user-hook", + UserMarker: userHookMarker, } } diff --git a/app/cli/internal/trace/e2e/pi.go b/app/cli/internal/trace/e2e/pi.go new file mode 100644 index 000000000..5a942d0b2 --- /dev/null +++ b/app/cli/internal/trace/e2e/pi.go @@ -0,0 +1,157 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" +) + +// piExtensionPath is the extension file where `trace init` installs the Pi +// hooks (spec issue-3520, D-002). +const piExtensionPath = ".pi/extensions/chainloop-trace.ts" + +var piVersionRe = regexp.MustCompile(`\d+\.\d+\.\d+`) + +// Pi is the harness adapter of the Pi coding agent. +type Pi struct { + // Bin is the pi executable. Empty means "pi" from PATH. + Bin string +} + +func (p *Pi) bin() string { + if p.Bin != "" { + return p.Bin + } + return "pi" +} + +// Column implements Harness. +func (p *Pi) Column() string { return "Pi" } + +// TraceFlag implements Harness. +func (p *Pi) TraceFlag() string { return "--pi" } + +// Version implements Harness. `pi --version` prints "0.79.6". +func (p *Pi) Version(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, p.bin(), "--version").Output() //nolint:gosec // the harness binary under test + if err != nil { + return "", fmt.Errorf("pi --version: %w", err) + } + v := piVersionRe.FindString(string(out)) + if v == "" { + return "", fmt.Errorf("pi --version: no version in %q", strings.TrimSpace(string(out))) + } + return v, nil +} + +// Run implements Harness. It runs `pi -p` with the JSON output. +// +// The configuration folder of Pi and its session folder are next to the +// workspace, so the extensions, the skills and the settings of the user who +// runs the suite do not load, and the sessions of the suite do not mix with +// the sessions of the user. --approve trusts the project files of the +// workspace, which hold the Chainloop extension. +// +// The Chainloop extension gives its instruction and reminder to the model as +// hidden context, which the run output does not show. Until the suite reads +// them from the session file, the hook messages are not reported. +func (p *Pi) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { + base := filepath.Dir(req.Dir) + args := []string{ + "-p", "--mode", "json", "--approve", + flagModel, req.Model, + "--session-dir", filepath.Join(base, "pi-sessions"), + } + + // A pasted image goes in as an attached file, from outside the + // repository, so that the agent cannot commit it. + if len(req.Images) > 0 { + dir, err := os.MkdirTemp("", "chainloop-trace-e2e-") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + for i, img := range req.Images { + path := filepath.Join(dir, fmt.Sprintf("pasted-%d.png", i+1)) + if err := os.WriteFile(path, img, 0o600); err != nil { + return nil, err + } + args = append(args, "@"+path) + } + } + args = append(args, req.Prompt) + + name := p.bin() + if len(req.Wrapper) > 0 { + name = req.Wrapper[0] + args = append(append(append([]string{}, req.Wrapper[1:]...), p.bin()), args...) + } + + cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario // nosemgrep + cmd.Dir = req.Dir + cmd.Env = append(slices.Clone(req.Env), + "PI_CODING_AGENT_DIR="+filepath.Join(base, "pi-agent"), + "PI_SKIP_VERSION_CHECK=1", + "PI_TELEMETRY=0", + ) + var stderr bytes.Buffer + cmd.Stderr = &stderr + + raw, err := cmd.Output() + if err != nil { + return &RunOutput{Raw: raw}, fmt.Errorf("pi -p: %w: %s", err, strings.TrimSpace(stderr.String())) + } + + return &RunOutput{Raw: raw}, nil +} + +// HookConfig implements Harness. The Chainloop extension is a file of its +// own, so the extension of the user is in another file. +func (p *Pi) HookConfig() HookConfig { + return HookConfig{ + Path: piExtensionPath, + ChainloopMarker: "chainloop", + UserPath: ".pi/extensions/e2e-user.ts", + UserConfig: `// e2e-user-hook +export default function () {} +`, + UserMarker: userHookMarker, + } +} + +// errPiSessionStart is the error of DisableSessionStart until the suite +// knows the shape of the Chainloop extension. +var errPiSessionStart = errors.New("pi: the suite cannot remove the session-start hook of the Chainloop extension yet") + +// DisableSessionStart implements Harness. The Chainloop extension of Pi does +// not exist yet, so there is no handler to change. +func (p *Pi) DisableSessionStart(string) error { + return errPiSessionStart +} + +// SkillPath implements Harness. Pi finds project skills in .agents/skills. +func (p *Pi) SkillPath(name string) string { + return ".agents/skills/" + name + "/SKILL.md" +} From 4d1379ba88709c420ab8f0904a3ffaa53afa5854 Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Tue, 20 Oct 2026 12:35:53 +0200 Subject: [PATCH 2/2] fix(trace): keep the e2e Pi version command out of SAST findings Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/internal/trace/e2e/pi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/cli/internal/trace/e2e/pi.go b/app/cli/internal/trace/e2e/pi.go index 5a942d0b2..a8b86e327 100644 --- a/app/cli/internal/trace/e2e/pi.go +++ b/app/cli/internal/trace/e2e/pi.go @@ -55,7 +55,7 @@ func (p *Pi) TraceFlag() string { return "--pi" } // Version implements Harness. `pi --version` prints "0.79.6". func (p *Pi) Version(ctx context.Context) (string, error) { - out, err := exec.CommandContext(ctx, p.bin(), "--version").Output() //nolint:gosec // the harness binary under test + out, err := exec.CommandContext(ctx, p.bin(), "--version").Output() //nolint:gosec // the harness binary under test // nosemgrep if err != nil { return "", fmt.Errorf("pi --version: %w", err) }