From 5817fb1f73e16544ae46d98c72f7eefd15d67424 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 20:01:34 +0200 Subject: [PATCH 1/4] feat(trace): capture Codex skills and spec materials Implement milestone 2 of #3657 with skill snapshots, spec-source pointers, pasted-image materials, and viewer-compatible transcript replacements. Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: 01a15f80-7cef-7100-a8fc-389f64109471 --- app/cli/internal/trace/PROVIDERS.md | 6 +- app/cli/internal/trace/README.md | 16 +- .../internal/trace/codex/extensions_test.go | 234 ++++++ app/cli/internal/trace/codex/materials.go | 735 ++++++++++++++++++ app/cli/internal/trace/codex/skills.go | 303 ++++++++ app/cli/internal/trace/codex/transcript.go | 55 +- app/cli/internal/trace/e2e/checks.go | 6 + app/cli/internal/trace/e2e/checks_codex.go | 170 ++++ .../internal/trace/e2e/checks_codex_test.go | 38 + app/cli/internal/trace/e2e/checks_more.go | 15 + app/cli/internal/trace/e2e/scenarios.go | 2 +- .../pkg/action/trace_codex_materials_test.go | 155 ++++ docs/specs/issue-3657-codex-trace-provider.md | 44 +- 13 files changed, 1756 insertions(+), 23 deletions(-) create mode 100644 app/cli/internal/trace/codex/extensions_test.go create mode 100644 app/cli/internal/trace/codex/materials.go create mode 100644 app/cli/internal/trace/codex/skills.go create mode 100644 app/cli/internal/trace/e2e/checks_codex.go create mode 100644 app/cli/internal/trace/e2e/checks_codex_test.go create mode 100644 app/cli/pkg/action/trace_codex_materials_test.go diff --git a/app/cli/internal/trace/PROVIDERS.md b/app/cli/internal/trace/PROVIDERS.md index 34a5f4425..30112d152 100644 --- a/app/cli/internal/trace/PROVIDERS.md +++ b/app/cli/internal/trace/PROVIDERS.md @@ -159,9 +159,9 @@ The handlers find these interfaces with a type assertion. A provider without one | Interface | Feature | Implemented by | What to do | |-----------|---------|----------------|------------| -| `trace.SkillTracker` | Skills tracking | Claude, OpenCode | `NewSkillLoads`: return the skill folders that the session loaded since the last call (from the hook input or the transcript). `SessionSkills`: return the skill uses from the copied session data, in the order of the first use. `SkillRoots`: return the plugin, organization, user, and project skill folders of the agent. | -| `trace.SpecCopyReplacer` | Spec source pointers | Claude | Replace each exact copy of a spec source in the raw session with `pointer.New(digest)`. Report the result per finder. | -| `trace.PastedImageFinder` | Pasted image pointers | Claude | Return each distinct pasted image of the raw session, with its digest, media type, and number. | +| `trace.SkillTracker` | Skills tracking | Claude, OpenCode, Codex | `NewSkillLoads`: return the skill folders that the session loaded since the last call (from the hook input or the transcript). `SessionSkills`: return the skill uses from the copied session data, in the order of the first use. `SkillRoots`: return the plugin, organization, user, and project skill folders of the agent. | +| `trace.SpecCopyReplacer` | Spec source pointers | Claude, Codex | Replace each exact copy of a spec source in the raw session with `pointer.New(digest)`. Report the result per finder. | +| `trace.PastedImageFinder` | Pasted image pointers | Claude, Codex | Return each distinct pasted image of the raw session, with its digest, media type, and number. | Add a compile-time check for each one, for example `var _ trace.SkillTracker = (*Provider)(nil)`. diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index a9fee17de..5602cef42 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -12,13 +12,15 @@ Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --local --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Installed hooks have a `Chainloop Trace` status label in Codex. Re-running init adds labels to existing Chainloop hooks. Changing the hooks requires a new trust review. -Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 22 feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts, skills tracking, spec source pointers, and pasted image pointers are not implemented. +Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 22 core feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts are not implemented. + +Milestone 2 live checks passed with the same CLI and model: skills in 3/3 sessions, spec pointers in 2/3 sessions (the suite requires 2/3), and pasted images in 1/1 session. The image scenario attaches the same image twice and requires one stored material. Transcript rendering and media redaction checks also passed. Provider and assembly regressions cover malformed images, storage failures, and reuse of an existing image material. Codex token usage comes from cumulative rollout counters. Cost is an estimate from the embedded standard API prices, with a warning for unknown models. Subscription billing and third-party provider prices can differ. Trace reads rollouts from `$CODEX_HOME/sessions` (default `~/.codex/sessions`) or `archived_sessions`. Their JSONL format is not a stable Codex interface. In a linked Git worktree, Codex 0.163 reads project hooks from the primary checkout. Install the Codex hooks in that checkout too, then review and trust them in Codex. -Codex evidence converts native messages, tool calls, and tool results to the message shape used by the existing session viewer. Text uses `message.content`, calls use `tool_use` blocks, and results use `tool_result` blocks with matching IDs. Each converted entry retains its original event in `source_event`; native metadata stays unchanged. Codex regression and E2E checks require renderable messages. When tools are used, they also require a completed exchange with readable tool output. A nonempty native transcript alone does not pass. These checks apply to Codex; `raw_session` remains open in the shared evidence schema. +Codex evidence converts native messages, tool calls, and tool results to the message shape used by the existing session viewer. Text uses `message.content`, calls use `tool_use` blocks, and results use `tool_result` blocks with matching IDs. Each converted entry retains its event in `source_event`. Spec and pasted-image pointers replace copies in viewer messages, retained events, and supported native timeline records. Other native metadata stays unchanged. Codex regression and E2E checks require renderable messages. When tools are used, they also require a completed exchange with readable tool output. A nonempty native transcript alone does not pass. These checks apply to Codex; `raw_session` remains open in the shared evidence schema. ## Setup @@ -111,16 +113,18 @@ What the session was asked to build, and which skills it used. | Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | Unknown | | Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | Yes | | Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Yes | Unknown | Yes | No | Yes | -| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | No | -| Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | No | -| Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | No | +| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | Yes | +| Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | Partial | +| Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | Yes | Why not Yes: - **Spec capture reminder, Codex:** Codex does not report hook messages in its headless output, so the suite cannot inspect the reminder. - **Spec capture reminder, Cursor:** Cursor has no hook at each user prompt. The agent gets the session-start instruction only. - **Spec source role, title, and description, OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). -- **Skills tracking, spec source pointers, and pasted image pointers, Codex:** these optional integrations are not implemented. +- **Skills tracking, Codex:** tracks filesystem skills loaded through explicit Codex skill instructions or successful shell reads of one identified `SKILL.md`. The definition and supporting files use the shared skill snapshot and packaging pipeline. Multiple distinct skill paths in one tool call cannot be attributed reliably and are not tracked. Skills without a filesystem folder and child sessions are not captured. A snapshot first made at push carries a warning. +- **Spec source pointers, Codex:** supports exact text results, native shell envelopes, recognized code-mode JSON envelopes, and complete file creation through native patches or literal patches in code mode. Stored source files read with `cat` can also match within combined command output and native file-change records. Mixed or computed patches, arbitrary output wrappers, transformed content, and other unsupported formats remain inline. One live session trimmed the source's trailing newline and produced no viewer pointer, so the table remains Partial. +- **Pasted image pointers, Codex:** extracts base64 image data URLs from user input, deduplicates their decoded bytes, and replaces stored copies in both viewer messages and retained events. Tool-result images are not pastes. Invalid data, remote URLs, and path-only image records remain inline. - **Skills tracking, Cursor:** the provider does not track skills. The evidence has no skill entries, which means "not recorded", not "no skill used". - **Skills tracking, OpenCode:** only the skills that the model starts with the `skill` tool are counted. Skills that the user starts, and skills used in subagents, are not. A plugin installed by an earlier version does not send the skill folder for OpenCode 2: install the hooks again to update it. - **Spec source pointers, Cursor and OpenCode:** the provider has no finder for the copies in its transcript yet. For OpenCode, the rebuilt transcript holds no tool outputs, so only the write copies would apply. diff --git a/app/cli/internal/trace/codex/extensions_test.go b/app/cli/internal/trace/codex/extensions_test.go new file mode 100644 index 000000000..49beabea9 --- /dev/null +++ b/app/cli/internal/trace/codex/extensions_test.go @@ -0,0 +1,234 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:goconst // Keep native protocol literals readable in regression fixtures. +package codex + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testReadCallID = "read" + testExtensionCallID = "call-1" + testSuccess = "Success" +) + +func extensionEvent(t *testing.T, kind string, payload any) json.RawMessage { + t.Helper() + b, err := json.Marshal(payload) + require.NoError(t, err) + raw, err := json.Marshal(rolloutEntry{Timestamp: "2026-10-21T10:00:00Z", Type: kind, Payload: b}) + require.NoError(t, err) + return raw +} + +func extensionMessage(t *testing.T, text string) json.RawMessage { + t.Helper() + return extensionEvent(t, responseItem, map[string]any{typeField: "message", "role": "user", contentField: []map[string]string{{typeField: "input_text", textField: text}}}) +} + +func normalizedExtension(t *testing.T, raw json.RawMessage) json.RawMessage { + t.Helper() + var e rolloutEntry + var payload rolloutPayload + require.NoError(t, json.Unmarshal(raw, &e)) + require.NoError(t, json.Unmarshal(e.Payload, &payload)) + out, err := transcriptEntry(e, payload, raw, testModel, 1) + require.NoError(t, err) + return out +} + +func TestCodexSpecCopies(t *testing.T) { + const source = "# Task\n\nBuild the requested feature.\n" + const material = "sha256:stored-redacted-source" + for _, tc := range []struct { + name, tool, input, output string + want bool + }{ + {name: "plain shell read", tool: toolExecCommand, input: `{"cmd":"cat ticket.md"}`, output: source, want: true}, + {name: "framed shell read", tool: toolExecCommand, input: `{"cmd":"cat ticket.md"}`, output: "Chunk ID: test\nWall time: 0.1 seconds\nProcess exited with code 0\nFinal output:\n" + source, want: true}, + {name: "partial read", tool: toolExecCommand, input: `{"cmd":"head ticket.md"}`, output: "# Task\n", want: false}, + {name: "trimmed source stays inline", tool: toolExecCommand, input: `{"cmd":"cat ticket.md"}`, output: strings.TrimSuffix(source, "\n"), want: false}, + {name: "failed shell", tool: toolExecCommand, input: `{"cmd":"cat ticket.md"}`, output: "Process exited with code 1\nFinal output:\n" + source, want: false}, + {name: "structured write", tool: "write_file", input: `{"path":"ticket.md","content":` + string(mustJSON(t, source)) + `}`, output: testSuccess, want: true}, + {name: "patch creation", tool: toolApplyPatch, input: "*** Begin Patch\n*** Add File: ticket.md\n+# Task\n+\n+Build the requested feature.\n*** End Patch", output: testSuccess, want: true}, + {name: "code mode patch creation", tool: "exec", input: "const patch = " + string(mustJSON(t, "*** Begin Patch\n*** Add File: ticket.md\n+# Task\n+\n+Build the requested feature.\n*** End Patch")) + "; text(await tools.apply_patch(patch));", output: "Script completed", want: true}, + {name: "failed write", tool: "write_file", input: `{"path":"ticket.md","content":` + string(mustJSON(t, source)) + `}`, output: "Error: permission denied", want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + call := map[string]string{typeField: functionCall, "name": tc.tool, callIDField: testExtensionCallID, "arguments": tc.input} + resultType := functionCallOutput + if tc.tool == toolApplyPatch || tc.tool == "exec" { + call = map[string]string{typeField: customToolCall, "name": tc.tool, callIDField: testExtensionCallID, "input": tc.input} + resultType = "custom_tool_call_output" + } + raw := map[string][]json.RawMessage{mainTranscriptKey: { + normalizedExtension(t, extensionEvent(t, responseItem, call)), + normalizedExtension(t, extensionEvent(t, responseItem, map[string]string{typeField: resultType, callIDField: testExtensionCallID, outputField: tc.output})), + }} + before := string(mustJSON(t, raw)) + report := New().ReplaceSpecCopies(raw, pointer.Sources{pointer.Digest([]byte(source)): material}) + after := string(mustJSON(t, raw)) + if !tc.want { + assert.Equal(t, before, after) + return + } + assert.Contains(t, after, pointer.Marker) + assert.Contains(t, after, material) + assert.NotContains(t, after, "Build the requested feature.") + assert.NotEmpty(t, report) + // Repeating assembly cannot replace a pointer or lose native metadata. + New().ReplaceSpecCopies(raw, pointer.Sources{pointer.Digest([]byte(source)): material}) + assert.Equal(t, after, string(mustJSON(t, raw))) + }) + } +} + +func mustJSON(t *testing.T, v any) json.RawMessage { + t.Helper() + raw, err := json.Marshal(v) + require.NoError(t, err) + return raw +} + +func TestCodexPastedImages(t *testing.T) { + data := []byte("test image bytes") + url := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data) + image := map[string]string{typeField: blockInputImage, "image_url": url} + prompt := extensionEvent(t, responseItem, map[string]any{typeField: "message", "role": "user", contentField: []any{image, image, map[string]string{typeField: blockInputImage, "image_url": "data:image/png;base64,!invalid"}}}) + tool := extensionEvent(t, responseItem, map[string]any{typeField: functionCallOutput, callIDField: "image-read", outputField: []any{image}}) + raw := map[string][]json.RawMessage{mainTranscriptKey: {normalizedExtension(t, prompt), normalizedExtension(t, tool)}} + images := New().PastedImages(raw) + require.Len(t, images, 1) + assert.Equal(t, data, images[0].Data) + assert.Equal(t, 1, images[0].Number) + assert.Equal(t, "image/png", images[0].MediaType) + // A storage failure leaves all copies intact. + before := string(mustJSON(t, raw)) + New().ReplaceSpecCopies(raw, nil) + assert.Equal(t, before, string(mustJSON(t, raw))) + New().ReplaceSpecCopies(raw, pointer.Sources{images[0].Digest: "sha256:stored-image"}) + assert.NotContains(t, string(raw[mainTranscriptKey][0]), url) + assert.Contains(t, string(raw[mainTranscriptKey][0]), "!invalid") + assert.Contains(t, string(raw[mainTranscriptKey][0]), "sha256:stored-image") + assert.Contains(t, string(raw[mainTranscriptKey][1]), url, "tool images remain inline") + var viewer trace.RawSessionEntry + require.NoError(t, json.Unmarshal(raw[mainTranscriptKey][0], &viewer)) + assert.Contains(t, string(viewer.Message.Content), `"type":"image"`) + var blocks []struct { + Source json.RawMessage `json:"source"` + } + require.NoError(t, json.Unmarshal(viewer.Message.Content, &blocks)) + assert.JSONEq(t, string(pointer.New("sha256:stored-image")), string(blocks[0].Source)) +} + +func TestCodexSessionSkills(t *testing.T) { + dir := t.TempDir() + skillDir := filepath.Join(dir, ".agents", "skills", "test-skill") + require.NoError(t, os.MkdirAll(skillDir, 0700)) + definition := "---\nname: test-skill\ndescription: Test skill.\n---\n\nDo the task.\n" + require.NoError(t, os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(definition), 0600)) + header := extensionEvent(t, nativeSessionMeta, map[string]string{"id": testSessionID, cwdField: dir}) + injected := extensionMessage(t, "\ntest-skill\n"+filepath.Join(skillDir, "SKILL.md")+"\n"+definition+"") + catalog := extensionMessage(t, "Available skills: test-skill (file: "+filepath.Join(skillDir, "SKILL.md")+")") + call := extensionEvent(t, responseItem, map[string]string{typeField: customToolCall, "name": "exec", callIDField: "read-skill", "input": "const path = " + string(mustJSON(t, filepath.Join(skillDir, "SKILL.md"))) + `; text(await tools.exec_command({cmd:"cat .agents/skills/test-skill/SKILL.md"}));`}) + result := extensionEvent(t, responseItem, map[string]string{typeField: customToolCallOutput, callIDField: "read-skill", outputField: definition}) + path := state.RawSessionPath(dir, testSessionID) + transcript := strings.Join([]string{string(header), string(catalog), string(injected), string(call), string(result)}, "\n") + "\n" + require.NoError(t, os.WriteFile(path, []byte(transcript), 0600)) + uses, warnings, err := New().SessionSkills(&trace.ParseOpts{SessionID: testSessionID, SessionDir: dir}) + require.NoError(t, err) + assert.Empty(t, warnings) + require.Len(t, uses, 1) + assert.Equal(t, skillDir, uses[0].Dir) + assert.Equal(t, 1, uses[0].ByUser) + assert.Equal(t, 1, uses[0].ByModel) + assert.Equal(t, "2026-10-21T10:00:00Z", uses[0].FirstUsedAt) +} + +func TestCodexCodeModeCopies(t *testing.T) { + const source = "# Task\n\nImplement this task.\n" + const digest = "sha256:redacted-material" + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "ticket.md"), []byte(source), 0600)) + for _, tc := range []struct { + name string + output any + }{ + {name: "promise envelope", output: []map[string]string{{typeField: "input_text", textField: string(mustJSON(t, map[string]any{"status": "fulfilled", "value": map[string]any{"exit_code": 0, outputField: source}}))}}}, + {name: "combined stdout", output: []map[string]string{{typeField: "input_text", textField: source + "## main...origin/main\n"}}}, + {name: "repeated combined stdout", output: []map[string]string{{typeField: "input_text", textField: source + "## main...origin/main\n" + source}}}, + } { + t.Run(tc.name, func(t *testing.T) { + call := extensionEvent(t, responseItem, map[string]string{typeField: customToolCall, "name": "exec", callIDField: testReadCallID, "input": `text(await tools.exec_command({cmd:"cat ticket.md && git status"}));`}) + result := extensionEvent(t, responseItem, map[string]any{typeField: customToolCallOutput, callIDField: testReadCallID, outputField: tc.output}) + timeline := extensionEvent(t, nativeEventMsg, map[string]any{typeField: "item_completed", "item": map[string]any{typeField: "CommandExecution", "id": testReadCallID, "command": "cat ticket.md && git status", cwdField: dir, "aggregated_output": source, "exit_code": 0}}) + fileChange := extensionEvent(t, nativeEventMsg, map[string]any{typeField: "item_completed", "item": map[string]any{typeField: "FileChange", "changes": map[string]any{"ticket.md": map[string]string{typeField: "add", contentField: "# Capture metadata\n" + source + "\n"}}}}) + raw := map[string][]json.RawMessage{mainTranscriptKey: {extensionEvent(t, nativeSessionMeta, map[string]string{"id": testSessionID, cwdField: dir}), normalizedExtension(t, call), normalizedExtension(t, result), timeline, fileChange}} + New().ReplaceSpecCopies(raw, pointer.Sources{pointer.Digest([]byte(source)): digest}) + for _, index := range []int{2, 3, 4} { + assert.NotContains(t, string(raw[mainTranscriptKey][index]), "Implement this task.") + assert.Contains(t, string(raw[mainTranscriptKey][index]), digest) + } + var viewer trace.RawSessionEntry + require.NoError(t, json.Unmarshal(raw[mainTranscriptKey][2], &viewer)) + var blocks []trace.RawSessionToolResultBlock + require.NoError(t, json.Unmarshal(viewer.Message.Content, &blocks)) + assert.Contains(t, string(blocks[0].Content), pointer.Marker) + }) + } +} + +func TestCodexLiveSkillLoads(t *testing.T) { + home := t.TempDir() + t.Setenv("CODEX_HOME", home) + store := state.NewOutOfTreeStore(t.TempDir()) + dir := filepath.Join(home, "sessions") + require.NoError(t, os.MkdirAll(dir, 0700)) + path := filepath.Join(dir, "rollout-"+testSessionID+".jsonl") + header := extensionEvent(t, nativeSessionMeta, map[string]string{"id": testSessionID}) + load := extensionMessage(t, "\none\n/skills/one/SKILL.md\nDo the task.\n") + input := &trace.HookInput{SessionID: testSessionID, TranscriptPath: path} + // An in-flight line remains for the next hook, even when its JSON parses. + require.NoError(t, os.WriteFile(path, append(append(header, '\n'), load...), 0600)) + assert.Empty(t, New().NewSkillLoads(store, input)) + require.NoError(t, os.WriteFile(path, append(append(append(header, '\n'), load...), '\n'), 0600)) + assert.Equal(t, []string{"/skills/one"}, New().NewSkillLoads(store, input)) + assert.Empty(t, New().NewSkillLoads(store, input)) + // Truncation resets the scan; a foreign session never yields skill paths. + require.NoError(t, os.WriteFile(path, append(header, '\n'), 0600)) + assert.Empty(t, New().NewSkillLoads(store, input)) + foreign := extensionEvent(t, nativeSessionMeta, map[string]string{"id": "other-session"}) + require.NoError(t, os.WriteFile(path, append(append(append(foreign, '\n'), load...), '\n'), 0600)) + assert.Empty(t, New().NewSkillLoads(store, input)) + input.TranscriptPath = filepath.Join(t.TempDir(), "foreign.jsonl") + require.NoError(t, os.WriteFile(input.TranscriptPath, append(append(append(header, '\n'), load...), '\n'), 0600)) + assert.Empty(t, New().NewSkillLoads(store, input)) + link := filepath.Join(dir, "escaped.jsonl") + require.NoError(t, os.Symlink(input.TranscriptPath, link)) + input.TranscriptPath = link + assert.Empty(t, New().NewSkillLoads(store, input)) +} diff --git a/app/cli/internal/trace/codex/materials.go b/app/cli/internal/trace/codex/materials.go new file mode 100644 index 000000000..0ce4ff421 --- /dev/null +++ b/app/cli/internal/trace/codex/materials.go @@ -0,0 +1,735 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" +) + +const ( + nativeTurnContext = "turn_context" + nativeEventMsg = "event_msg" + contentField = "content" + callIDField = "call_id" + cwdField = "cwd" + textField = "text" + nativeSessionMeta = "session_meta" + mainTranscriptKey = "main" + functionCallOutput = "function_call_output" + customToolCallOutput = "custom_tool_call_output" + blockInputImage = "input_image" + toolExecCommand = "exec_command" + toolApplyPatch = "apply_patch" + typeField = "type" + outputField = "output" +) + +var catPathPattern = regexp.MustCompile(`\bcat\s+(?:--\s+)?(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"';&|()]+))`) + +var codeStringPattern = regexp.MustCompile(`"(?:[^"\\]|\\.)*"`) + +type sourceCopies map[string]string + +// commandSources loads only explicitly read files, with the same text-size +// bound as local spec capture. Their full bytes must match a stored source. +func commandSources(input, cwd string, sources pointer.Sources) sourceCopies { + copies := sourceCopies{} + for _, match := range catPathPattern.FindAllStringSubmatch(input, -1) { + path := "" + for _, candidate := range match[1:] { + if candidate != "" { + path = candidate + } + } + if !filepath.IsAbs(path) { + if !filepath.IsAbs(cwd) { + continue + } + path = filepath.Join(cwd, path) + } + if info, err := os.Stat(path); err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 { + continue + } + f, err := os.Open(path) + if err != nil { + continue + } + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 { + _ = f.Close() + continue + } + data, err := io.ReadAll(io.LimitReader(f, (1<<20)+1)) + _ = f.Close() + if err != nil || len(data) == 0 || len(data) > 1<<20 { + continue + } + if material, ok := sources[pointer.Digest(data)]; ok { + copies[string(data)] = material + } + } + return copies +} + +func toolCommand(fields jsonObject) (string, string) { + if input := jsonString(fields["input"]); input != "" { + return input, "" + } + args := object(json.RawMessage(jsonString(fields["arguments"]))) + cmd := jsonString(args["cmd"]) + if cmd == "" { + cmd = jsonString(args["command"]) + } + return cmd, jsonString(args["workdir"]) +} + +var ( + _ trace.SpecCopyReplacer = (*Provider)(nil) + _ trace.PastedImageFinder = (*Provider)(nil) +) + +// jsonObject keeps unmodified and future fields without decoding them into any. +type jsonObject map[string]json.RawMessage + +func object(raw json.RawMessage) jsonObject { + var v jsonObject + _ = json.Unmarshal(raw, &v) + return v +} + +func jsonString(raw json.RawMessage) string { + var s string + _ = json.Unmarshal(raw, &s) + return s +} + +func encodeJSON(v any) json.RawMessage { + b, _ := json.Marshal(v) // all callers supply JSON values or typed structures + return b +} + +// nativeRecord selects exactly one representation of an event. Normalized +// messages also retain the event, which must not count as another use or paste. +func nativeRecord(raw json.RawMessage) (rolloutEntry, rolloutPayload, bool) { + var wrapper struct { + SourceEvent json.RawMessage `json:"source_event"` + } + _ = json.Unmarshal(raw, &wrapper) + if len(wrapper.SourceEvent) > 0 { + raw = wrapper.SourceEvent + } + var e rolloutEntry + var payload rolloutPayload + ok := json.Unmarshal(raw, &e) == nil && e.Type != "" && json.Unmarshal(e.Payload, &payload) == nil + return e, payload, ok +} + +// updateNative preserves wrapper metadata and rebuilds only viewer content +// from the replaced native payload. Neither representation keeps an old copy. +func updateNative(raw json.RawMessage, payload jsonObject) json.RawMessage { + fields := object(raw) + if fields == nil { + return raw + } + if source := fields["source_event"]; len(source) > 0 { + native := object(source) + if native == nil { + return raw + } + native["payload"] = encodeJSON(payload) + updated := encodeJSON(native) + e, p, ok := nativeRecord(updated) + if !ok { + return raw + } + var wrapper trace.RawSessionEntry + if json.Unmarshal(raw, &wrapper) != nil { + return raw + } + converted, err := transcriptEntry(e, p, updated, wrapper.Message.Model, 0) + if err != nil { + return raw + } + message := object(converted)["message"] + if len(message) == 0 { + return raw + } + fields["message"] = message + fields["source_event"] = updated + } else { + fields["payload"] = encodeJSON(payload) + } + return encodeJSON(fields) +} + +func orderedStreams(raw map[string][]json.RawMessage) []string { + var keys []string + for key := range raw { + if key != mainTranscriptKey { + keys = append(keys, key) + } + } + sort.Strings(keys) + if _, ok := raw[mainTranscriptKey]; ok { + keys = append([]string{mainTranscriptKey}, keys...) + } + return keys +} + +func decodeImageURL(url string) ([]byte, string, bool) { + header, encoded, ok := strings.Cut(url, ",") + if !ok || !strings.HasPrefix(header, "data:image/") || !strings.HasSuffix(header, ";base64") { + return nil, "", false + } + data, err := base64.StdEncoding.DecodeString(encoded) + return data, strings.TrimSuffix(strings.TrimPrefix(header, "data:"), ";base64"), err == nil && len(data) > 0 +} + +// promptImages visits user input only. Tool images are deliberately excluded. +func promptImages(e rolloutEntry, p rolloutPayload) ([]jsonObject, jsonObject) { + fields := object(e.Payload) + if fields == nil { + return nil, nil + } + if e.Type == responseItem && p.Type == messageItem && p.Role == trace.RoleUser { + var blocks []jsonObject + if json.Unmarshal(fields[contentField], &blocks) != nil { + return nil, nil + } + return blocks, fields + } + return nil, nil +} + +// PastedImages extracts each distinct user image from native or normalized +// rollouts. The digest identifies the model-visible bytes, not a file path. +func (p *Provider) PastedImages(raw map[string][]json.RawMessage) []trace.PastedImage { + var images []trace.PastedImage + seen := map[string]bool{} + position := 0 + for _, key := range orderedStreams(raw) { + for _, line := range raw[key] { + e, payload, ok := nativeRecord(line) + if !ok { + continue + } + blocks, _ := promptImages(e, payload) + for _, b := range blocks { + if jsonString(b[typeField]) != blockInputImage { + continue + } + position++ + data, media, valid := decodeImageURL(jsonString(b["image_url"])) + if !valid { + continue + } + digest := pointer.Digest(data) + if !seen[digest] { + seen[digest] = true + images = append(images, trace.PastedImage{Data: data, Digest: digest, MediaType: media, Number: position, Timestamp: eventTimestamp(e.Timestamp)}) + } + } + } + } + return images +} + +func eventTimestamp(timestamp string) string { + at, err := time.Parse(time.RFC3339Nano, timestamp) + if err != nil { + return "" + } + return at.UTC().Format(time.RFC3339) +} + +// ReplaceSpecCopies replaces exact source copies only after material storage. +// Results and native duplicates use the same digest mapping as viewer messages. +func (p *Provider) ReplaceSpecCopies(raw map[string][]json.RawMessage, sources pointer.Sources) pointer.Report { + report := pointer.Report{} + if len(sources) == 0 { + return report + } + for _, lines := range raw { + calls := map[string]rolloutPayload{} + results := map[string]json.RawMessage{} + copies := sourceCopies{} + cwd := "" + for _, line := range lines { + e, payload, ok := nativeRecord(line) + if !ok { + continue + } + fields := object(e.Payload) + if e.Type == nativeSessionMeta || e.Type == nativeTurnContext { + if dir := jsonString(fields[cwdField]); dir != "" { + cwd = dir + } + } + if e.Type == nativeEventMsg && payload.Type == "item_completed" { + item := object(fields["item"]) + if jsonString(item[typeField]) == "CommandExecution" { + addSourceCopies(copies, commandSources(jsonString(item["command"]), jsonString(item[cwdField]), sources)) + } + } + if e.Type != responseItem { + continue + } + switch payload.Type { + case functionCall, customToolCall: + calls[payload.CallID] = payload + cmd, workdir := toolCommand(fields) + if workdir == "" { + workdir = cwd + } + addSourceCopies(copies, commandSources(cmd, workdir, sources)) + case functionCallOutput, customToolCallOutput: + results[payload.CallID] = object(e.Payload)[outputField] + } + } + for i, line := range lines { + e, payload, ok := nativeRecord(line) + if !ok { + continue + } + fields := object(e.Payload) + changed := replacePromptImages(e, payload, fields, sources, report) + if e.Type == nativeEventMsg && payload.Type == "item_completed" { + changed = replaceTimelineItem(fields, sources, copies, report) || changed + } + if e.Type == responseItem { + switch payload.Type { + case functionCallOutput, customToolCallOutput: + call := calls[payload.CallID] + if isReadTool(call.Name) { + if value, n := replaceOutput(fields[outputField], sources, copies); n > 0 { + fields[outputField] = value + res := report.Finder(pointer.FinderFileRead) + res.Replaced += n + changed = true + } + } + case functionCall, customToolCall: + if result, found := results[payload.CallID]; found && !failedOutput(result) { + changed = replaceWrite(fields, payload.Name, sources, report) || changed + } + } + } + if changed { + lines[i] = updateNative(line, fields) + } + } + } + return report +} + +func addSourceCopies(dst, src sourceCopies) { + for text, material := range src { + dst[text] = material + } +} + +func replaceTimelineItem(fields jsonObject, sources pointer.Sources, copies sourceCopies, report pointer.Report) bool { + item := object(fields["item"]) + changed := false + switch jsonString(item[typeField]) { + case "CommandExecution": + if failedOutput(encodeJSON(item)) { + return false + } + if updated, n := replaceOutput(item["aggregated_output"], sources, copies); n > 0 { + delete(item, "aggregated_output") + item[pointer.Marker] = updated + report.Finder(pointer.FinderFileRead).Replaced += n + changed = true + } + case "FileChange": + if jsonString(item["status"]) == "failed" { + return false + } + changes := object(item["changes"]) + for path, raw := range changes { + change := object(raw) + if jsonString(change[typeField]) != "add" { + continue + } + if updated, n := replaceOutput(change[contentField], sources, copies); n > 0 { + delete(change, contentField) + change[pointer.Marker] = updated + changes[path] = encodeJSON(change) + report.Finder(pointer.FinderFileWrite).Replaced += n + changed = true + } + } + if changed { + item["changes"] = encodeJSON(changes) + } + } + if changed { + fields["item"] = encodeJSON(item) + } + return changed +} + +func isReadTool(name string) bool { + switch name { + case "exec", toolExecCommand, "shell", "shell_command", "Bash", "read_file", "read", "write_stdin": + return true + } + return false +} + +func replacePromptImages(e rolloutEntry, payload rolloutPayload, fields jsonObject, sources pointer.Sources, report pointer.Report) bool { + blocks, _ := promptImages(e, payload) + changed := false + for _, b := range blocks { + if jsonString(b[typeField]) != blockInputImage || len(b[pointer.Marker]) > 0 { + continue + } + data, _, ok := decodeImageURL(jsonString(b["image_url"])) + res := report.Finder(pointer.FinderPastedImage) + if !ok { + res.Skip(pointer.ReasonInvalidImage, 1) + continue + } + material, stored := sources[pointer.Digest(data)] + if !stored { + res.Skip(pointer.ReasonMaterialNotStored, 1) + continue + } + res.BytesRemoved += len(b["image_url"]) + delete(b, "image_url") + b[pointer.Marker] = pointer.New(material) + res.Replaced++ + changed = true + } + if changed { + fields[contentField] = encodeJSON(blocks) + } + return changed +} + +// stdoutText strips only the documented shell result envelope. It never +// trims the source text: trailing newlines participate in its digest. +func stdoutText(text string) (string, bool) { + if strings.Contains(text, "Process exited with code ") { + prefix, output, ok := strings.Cut(text, "\nFinal output:\n") + if !ok || !strings.HasSuffix(prefix, "Process exited with code 0") { + return "", false + } + return output, true + } + return text, true +} + +func failedOutput(raw json.RawMessage) bool { + text := jsonString(raw) + if text != "" { + for _, prefix := range []string{"Error:", "error:", "Failed to ", "apply_patch verification failed:", "Script failed"} { + if strings.HasPrefix(strings.TrimSpace(text), prefix) { + return true + } + } + _, ok := stdoutText(text) + return !ok + } + var blocks []struct{ Text string } + if json.Unmarshal(raw, &blocks) == nil { + for _, block := range blocks { + if failedOutput(encodeJSON(block.Text)) { + return true + } + } + } + var result struct { + ExitCode *int `json:"exit_code"` + IsError bool `json:"is_error"` + } + return json.Unmarshal(raw, &result) == nil && (result.IsError || result.ExitCode != nil && *result.ExitCode != 0) +} + +func replaceOutput(raw json.RawMessage, sources pointer.Sources, copies sourceCopies) (json.RawMessage, int) { + if failedOutput(raw) { + return raw, 0 + } + var text string + if json.Unmarshal(raw, &text) == nil { + stdout, ok := stdoutText(text) + if ok { + if material, found := sources[pointer.Digest([]byte(stdout))]; found { + return encodeJSON([]json.RawMessage{pointer.New(material)}), 1 + } + if updated, n := replaceTextCopies(stdout, copies); n > 0 { + return updated, n + } + } + // Code mode exposes nested exec_command results as JSON text blocks. + if nested := object(json.RawMessage(text)); nested != nil { + if updated, n := replaceOutput(json.RawMessage(text), sources, copies); n > 0 { + // Keep envelope metadata with the pointer while exposing a + // supported block type to the viewer. + var pointers []jsonObject + collectPointers(updated, &pointers) + if len(pointers) > 0 { + pointers[0]["codex_output"] = updated + return encodeJSON(pointers), n + } + } + } + return raw, 0 + } + if fields := object(raw); fields != nil { + for _, key := range []string{outputField, "value"} { + if value, n := replaceOutput(fields[key], sources, copies); n > 0 { + fields[key] = value + return encodeJSON(fields), n + } + } + return raw, 0 + } + var blocks []jsonObject + if json.Unmarshal(raw, &blocks) != nil { + return raw, 0 + } + n := 0 + var replaced []jsonObject + for _, block := range blocks { + if kind := jsonString(block[typeField]); kind != "input_text" && kind != "output_text" && kind != textField { + replaced = append(replaced, block) + continue + } + if updated, count := replaceOutput(block[textField], sources, copies); count > 0 { + var values []jsonObject + if json.Unmarshal(updated, &values) == nil && len(values) > 0 { + replaced = append(replaced, values...) + n += count + continue + } + } + replaced = append(replaced, block) + } + if n > 0 { + return encodeJSON(replaced), n + } + return raw, 0 +} + +func collectPointers(raw json.RawMessage, out *[]jsonObject) { + if fields := object(raw); fields != nil { + if jsonString(fields[typeField]) == pointer.Marker { + *out = append(*out, fields) + return + } + for _, key := range []string{outputField, "value"} { + collectPointers(fields[key], out) + } + return + } + var blocks []json.RawMessage + if json.Unmarshal(raw, &blocks) == nil { + for _, block := range blocks { + collectPointers(block, out) + } + } +} + +func replaceTextCopies(text string, copies sourceCopies) (json.RawMessage, int) { + // Longest first makes overlapping sources deterministic. + keys := make([]string, 0, len(copies)) + for source := range copies { + keys = append(keys, source) + } + sort.Slice(keys, func(i, j int) bool { + if len(keys[i]) == len(keys[j]) { + return keys[i] < keys[j] + } + return len(keys[i]) > len(keys[j]) + }) + var blocks []json.RawMessage + n := 0 + for text != "" { + position, matched := -1, "" + for _, source := range keys { + index := sourceOffset(text, source) + if index >= 0 && (position < 0 || index < position) { + position, matched = index, source + } + } + if position < 0 { + blocks = append(blocks, encodeJSON(trace.RawSessionTextBlock{Type: textField, Text: text})) + break + } + if position > 0 { + blocks = append(blocks, encodeJSON(trace.RawSessionTextBlock{Type: textField, Text: text[:position]})) + } + blocks = append(blocks, pointer.New(copies[matched])) + text = text[position+len(matched):] + n++ + } + if n == 0 { + return nil, 0 + } + return encodeJSON(blocks), n +} + +// sourceOffset requires complete source boundaries, so a source that merely +// appears inside an unrelated line is not treated as a file read. +func sourceOffset(text, source string) int { + if source == "" { + return -1 + } + start := 0 + for start < len(text) { + rel := strings.Index(text[start:], source) + if rel < 0 { + return -1 + } + index := start + rel + end := index + len(source) + if (index == 0 || text[index-1] == '\n') && (end == len(text) || strings.HasSuffix(source, "\n") || text[end] == '\n') { + return index + } + start = index + 1 + } + return -1 +} + +func replaceWrite(fields jsonObject, name string, sources pointer.Sources, report pointer.Report) bool { + if name == "exec" { + return replaceCodePatch(fields, sources, report) + } + if name == toolApplyPatch { + patch := jsonString(fields["input"]) + // A complete file creation has exact source bytes. Updates and mixed + // patches stay inline rather than guessing their resulting content. + lines := strings.Split(strings.TrimSuffix(patch, "\n"), "\n") + if len(lines) < 4 || lines[0] != "*** Begin Patch" || !strings.HasPrefix(lines[1], "*** Add File: ") || lines[len(lines)-1] != "*** End Patch" { + return false + } + var content strings.Builder + for _, line := range lines[2 : len(lines)-1] { + if !strings.HasPrefix(line, "+") { + return false + } + content.WriteString(strings.TrimPrefix(line, "+")) + content.WriteByte('\n') + } + if material, ok := sources[pointer.Digest([]byte(content.String()))]; ok { + delete(fields, "input") + fields[pointer.Marker] = pointer.New(material) + fields["path"] = encodeJSON(strings.TrimPrefix(lines[1], "*** Add File: ")) + report.Finder(pointer.FinderFileWrite).Replaced++ + return true + } + return false + } + if name != "write_file" && name != "write" { + return false + } + args := object(json.RawMessage(jsonString(fields["arguments"]))) + if args == nil || len(args[contentField]) == 0 { + return false + } + material, ok := sources[pointer.Digest([]byte(jsonString(args[contentField])))] + if !ok { + return false + } + delete(args, contentField) + args[pointer.Marker] = pointer.New(material) + fields["arguments"] = encodeJSON(string(encodeJSON(args))) + report.Finder(pointer.FinderFileWrite).Replaced++ + return true +} + +// replaceCodePatch recognizes literal patches in Codex code mode. It never +// evaluates JavaScript or reconstructs a computed patch. Other script text +// stays available alongside the material pointer in the viewer. +func replaceCodePatch(fields jsonObject, sources pointer.Sources, report pointer.Report) bool { + code := jsonString(fields["input"]) + if !strings.Contains(code, "tools.apply_patch") { + return false + } + var out strings.Builder + position := 0 + var replacements []json.RawMessage + for _, span := range codeStringPattern.FindAllStringIndex(code, -1) { + literal := jsonString(json.RawMessage(code[span[0]:span[1]])) + if !strings.HasPrefix(literal, "*** Begin Patch\n") { + continue + } + patch := jsonObject{"input": encodeJSON(literal)} + if !replaceWrite(patch, toolApplyPatch, sources, report) { + continue + } + out.WriteString(code[position:span[0]]) + out.Write(patch[pointer.Marker]) + position = span[1] + replacements = append(replacements, patch[pointer.Marker]) + } + if len(replacements) == 0 { + return false + } + out.WriteString(code[position:]) + fields["input"] = encodeJSON(out.String()) + fields[pointer.Marker] = replacements[0] + if len(replacements) > 1 { + fields["spec_sources"] = encodeJSON(replacements) + } + return true +} + +// outputTexts reads successful text results, including code-mode envelopes, +// for skill-load recognition without interpreting arbitrary tool arguments. +func outputTexts(raw json.RawMessage) []string { + if failedOutput(raw) { + return nil + } + var text string + if json.Unmarshal(raw, &text) == nil { + if fields := object(json.RawMessage(text)); fields != nil && len(fields[outputField]) > 0 { + return outputTexts(json.RawMessage(text)) + } + stdout, ok := stdoutText(text) + if ok { + return []string{stdout} + } + return nil + } + if fields := object(raw); fields != nil { + return outputTexts(fields[outputField]) + } + var blocks []jsonObject + if json.Unmarshal(raw, &blocks) != nil { + return nil + } + var texts []string + for _, block := range blocks { + if bytes.Contains(block[typeField], []byte(textField)) { + texts = append(texts, outputTexts(block[textField])...) + } + } + return texts +} diff --git a/app/cli/internal/trace/codex/skills.go b/app/cli/internal/trace/codex/skills.go new file mode 100644 index 000000000..c95604118 --- /dev/null +++ b/app/cli/internal/trace/codex/skills.go @@ -0,0 +1,303 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "gopkg.in/yaml.v3" +) + +var _ trace.SkillTracker = (*Provider)(nil) + +var skillPathPattern = regexp.MustCompile(`[^\s"'` + "`" + `<>\\]*SKILL\.md`) + +type skillCall struct { + paths []string + cwd string +} + +// SessionSkills reads the private rollout copy, never the current agent file. +func (p *Provider) SessionSkills(opts *trace.ParseOpts) ([]trace.SkillUse, []string, error) { + if opts == nil || !state.ValidSessionID(opts.SessionID) { + return nil, nil, errors.New("read Codex skills: invalid session") + } + path, err := state.FindRawSessionFile(opts.SessionDir, opts.SessionID) + if err != nil { + return nil, nil, err + } + uses, _, err := scanCodexSkills(path, opts.SessionID, 0, false) + return uses, nil, err +} + +// NewSkillLoads recognizes newly completed loads in the live rollout. It +// validates storage and session identity before returning any skill folder. +// Pending tool calls are rescanned so a result in the next hook can pair with +// its original call. The offset counts completed uses, not pending calls. +func (p *Provider) NewSkillLoads(store *state.Store, input *trace.HookInput) []string { + if store == nil || input == nil || !state.ValidSessionID(input.SessionID) { + return nil + } + home := codexHome() + if rec, err := store.LoadSessionRecord(input.SessionID); err == nil && rec != nil && rec.CodexStorageRoot != "" { + home = rec.CodexStorageRoot + } + if home == "" { + return nil + } + roots := []string{filepath.Join(home, "sessions"), filepath.Join(home, "archived_sessions")} + path := input.TranscriptPath + if path == "" { + path, _ = findRollout(roots, input.SessionID) + } + if !filepath.IsAbs(path) || !insideStorage(path, roots) { + return nil + } + offsets, err := store.SkillScanOffsets(input.SessionID) + if err != nil { + offsets = map[string]int64{} + } + uses, next, err := scanCodexSkills(path, input.SessionID, offsets[mainTranscriptKey], true) + if err != nil { + return nil + } + offsets[mainTranscriptKey] = next + _ = store.SetSkillScanOffsets(input.SessionID, offsets) + dirs := make([]string, 0, len(uses)) + for _, use := range uses { + dirs = append(dirs, use.Dir) + } + return dirs +} + +func scanCodexSkills(path, sessionID string, after int64, live bool) ([]trace.SkillUse, int64, error) { + f, err := os.Open(path) + if err != nil { + return nil, after, err + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() { + return nil, after, errors.New("read Codex skills: not a regular rollout") + } + if info.Size() < after { + after = 0 + } + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), maxHookPayloadBytes) + if live { + scanner.Split(func(data []byte, atEOF bool) (int, []byte, error) { + if atEOF && !bytes.ContainsRune(data, '\n') { + return 0, nil, nil + } + return bufio.ScanLines(data, atEOF) + }) + } + uses := map[string]*trace.SkillUse{} + calls := map[string]skillCall{} + cwd := "" + var offset int64 + first := true + for scanner.Scan() { + line := scanner.Bytes() + offset += int64(len(line) + 1) + e, payload, ok := nativeRecord(line) + if first { + first = false + if !ok || e.Type != nativeSessionMeta || payload.ID != sessionID { + return nil, after, errors.New("read Codex skills: rollout session ID mismatch") + } + } + if !ok { + continue + } + fields := object(e.Payload) + if dir := jsonString(fields[cwdField]); dir != "" && (e.Type == nativeSessionMeta || e.Type == nativeTurnContext) { + cwd = dir + } + if e.Type != responseItem { + continue + } + consumeSkillRecord(e, payload, fields, uses, calls, cwd, offset, after) + } + if err := scanner.Err(); err != nil { + return nil, after, fmt.Errorf("read Codex skills: %w", err) + } + if first { + return nil, after, errors.New("read Codex skills: missing rollout header") + } + return trace.SortedSkillUses(uses), offset, nil +} + +func skillTag(text, tag string) string { + _, tail, ok := strings.Cut(text, "<"+tag+">") + if !ok { + return "" + } + value, _, ok := strings.Cut(tail, "") + if !ok { + return "" + } + return strings.TrimSpace(value) +} + +func resolveSkillPath(path, cwd string) string { + if strings.HasPrefix(path, "~/") { + home, err := os.UserHomeDir() + if err != nil { + return "" + } + path = filepath.Join(home, strings.TrimPrefix(path, "~/")) + } + if !filepath.IsAbs(path) { + if !filepath.IsAbs(cwd) { + return "" + } + path = filepath.Join(cwd, path) + } + return filepath.Clean(path) +} + +func skillDefinitionName(text string) string { + front, ok := strings.CutPrefix(text, "---\n") + if !ok { + return "" + } + meta, _, ok := strings.Cut(front, "\n---") + if !ok { + return "" + } + var header struct { + Name string `yaml:"name"` + } + if yaml.Unmarshal([]byte(meta), &header) != nil { + return "" + } + return header.Name +} + +// SkillRoots classifies the filesystem-backed skill scopes known to Codex. +func (p *Provider) SkillRoots(repoRoot string) skill.Roots { + home, _ := os.UserHomeDir() + return skill.Roots{ + Plugin: []string{filepath.Join(codexHome(), "plugins")}, + Organization: []string{"/etc/codex/skills"}, + User: []string{filepath.Join(home, ".agents", "skills"), filepath.Join(codexHome(), "skills")}, + Project: []string{repoRoot}, + } +} + +func consumeSkillRecord(e rolloutEntry, payload rolloutPayload, fields jsonObject, uses map[string]*trace.SkillUse, calls map[string]skillCall, cwd string, offset, after int64) { + record := func(name, path string, byModel bool) { + if offset <= after || name == "" || filepath.Base(path) != skill.DefinitionFile { + return + } + path = resolveSkillPath(path, cwd) + if path == "" { + return + } + dir := filepath.Dir(path) + if uses[dir] == nil { + uses[dir] = &trace.SkillUse{Name: name, Dir: dir} + } + uses[dir].Record(eventTimestamp(e.Timestamp), byModel, false) + } + switch payload.Type { + case messageItem: + if payload.Role != trace.RoleUser { + return + } + var blocks []struct{ Text string } + _ = json.Unmarshal(fields[contentField], &blocks) + for _, b := range blocks { + text := strings.TrimSpace(b.Text) + if !strings.HasPrefix(text, "") || !strings.HasSuffix(text, "") { + continue + } + name := skillTag(text, "name") + path := skillTag(text, "path") + record(name, path, false) + } + case functionCall, customToolCall: + if !isReadTool(payload.Name) { + return + } + input := jsonString(fields["input"]) + callCwd := cwd + if args := object(json.RawMessage(jsonString(fields["arguments"]))); args != nil { + input = jsonString(args["cmd"]) + if input == "" { + input = jsonString(args["command"]) + } + if dir := jsonString(args["workdir"]); dir != "" { + callCwd = dir + } + } + calls[payload.CallID] = skillCall{paths: skillReadPaths(input, callCwd), cwd: callCwd} + case functionCallOutput, customToolCallOutput: + call := calls[payload.CallID] + delete(calls, payload.CallID) + // A catalog or path mention cannot prove a load. Require a tool + // result containing a skill definition and one identified file. + if len(call.paths) != 1 { + return + } + for _, text := range outputTexts(fields[outputField]) { + name := skillDefinitionName(text) + path := resolveSkillPath(call.paths[0], call.cwd) + record(name, path, true) + } + } +} + +func skillReadPaths(input, cwd string) []string { + literals := codeStringPattern.FindAllString(input, -1) + inputs := make([]string, 1, len(literals)+1) + inputs[0] = input + for _, literal := range literals { + inputs = append(inputs, jsonString(json.RawMessage(literal))) + } + var paths []string + for _, command := range inputs { + for _, match := range catPathPattern.FindAllStringSubmatch(command, -1) { + for _, path := range match[1:] { + if filepath.Base(path) == skill.DefinitionFile { + paths = append(paths, resolveSkillPath(path, cwd)) + } + } + } + } + if len(paths) == 0 { + for _, path := range skillPathPattern.FindAllString(input, -1) { + if resolved := resolveSkillPath(path, cwd); resolved != "" { + paths = append(paths, resolved) + } + } + } + slices.Sort(paths) + return slices.Compact(paths) +} diff --git a/app/cli/internal/trace/codex/transcript.go b/app/cli/internal/trace/codex/transcript.go index 25ef182ac..f68ebd045 100644 --- a/app/cli/internal/trace/codex/transcript.go +++ b/app/cli/internal/trace/codex/transcript.go @@ -17,11 +17,13 @@ package codex import ( "bytes" + "encoding/base64" "encoding/json" "fmt" "strings" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" ) const ( @@ -38,16 +40,19 @@ func transcriptEntry(e rolloutEntry, payload rolloutPayload, raw json.RawMessage return raw, nil } switch payload.Type { - case messageItem, functionCall, customToolCall, "function_call_output", "custom_tool_call_output", "reasoning": + case messageItem, functionCall, customToolCall, functionCallOutput, customToolCallOutput, "reasoning": default: return raw, nil } var item struct { - Content json.RawMessage `json:"content"` - Arguments string `json:"arguments"` - Input string `json:"input"` - Output json.RawMessage `json:"output"` - Summary []struct { + Content json.RawMessage `json:"content"` + Arguments string `json:"arguments"` + Input string `json:"input"` + Output json.RawMessage `json:"output"` + Replaced json.RawMessage `json:"chainloop.replaced"` + Path string `json:"path"` + SpecSources json.RawMessage `json:"spec_sources"` + Summary []struct { Text string `json:"text"` } `json:"summary"` } @@ -80,14 +85,28 @@ func transcriptEntry(e rolloutEntry, payload rolloutPayload, raw json.RawMessage input := json.RawMessage(item.Arguments) var object map[string]json.RawMessage if payload.Type == customToolCall { - input, _ = json.Marshal(map[string]string{"input": item.Input}) + if len(item.Replaced) > 0 { + values := jsonObject{pointer.Marker: item.Replaced} + if item.Path != "" { + values["path"] = encodeJSON(item.Path) + } + if item.Input != "" { + values["input"] = encodeJSON(item.Input) + } + if len(item.SpecSources) > 0 { + values["spec_sources"] = item.SpecSources + } + input = encodeJSON(values) + } else { + input, _ = json.Marshal(map[string]string{"input": item.Input}) + } } else if json.Unmarshal(input, &object) != nil || object == nil { input, _ = json.Marshal(map[string]string{"arguments": item.Arguments}) } blocks = append(blocks, trace.RawSessionToolUseBlock{ Type: "tool_use", ID: payload.CallID, Name: payload.Name, Input: input, }) - case "function_call_output", "custom_tool_call_output": + case functionCallOutput, customToolCallOutput: entry.Type = trace.RoleUser output := item.Output if len(output) == 0 { @@ -114,7 +133,7 @@ func transcriptEntry(e rolloutEntry, payload rolloutPayload, raw json.RawMessage return raw, nil } entry.Type = trace.RoleAssistant - blocks = append(blocks, map[string]string{"type": "thinking", "thinking": strings.Join(summary, "\n")}) + blocks = append(blocks, map[string]string{typeField: "thinking", "thinking": strings.Join(summary, "\n")}) default: return raw, nil } @@ -133,14 +152,26 @@ func transcriptContent(content json.RawMessage) (json.RawMessage, error) { if json.Unmarshal(content, &text) == nil { return content, nil } - var blocks []map[string]any + var blocks []jsonObject if err := json.Unmarshal(content, &blocks); err != nil { return nil, err } for _, block := range blocks { - switch block["type"] { + switch jsonString(block[typeField]) { case "input_text", "output_text": - block["type"] = "text" + block[typeField] = encodeJSON(textField) + case blockInputImage: + if replaced := block[pointer.Marker]; len(replaced) > 0 { + block[typeField] = encodeJSON("image") + block["source"] = replaced + delete(block, pointer.Marker) + } else if data, media, ok := decodeImageURL(jsonString(block["image_url"])); ok { + block[typeField] = encodeJSON("image") + block["source"] = encodeJSON(map[string]string{typeField: "base64", "media_type": media, "data": base64.StdEncoding.EncodeToString(data)}) + } + if jsonString(block[typeField]) == "image" { + delete(block, "image_url") + } } } return json.Marshal(blocks) diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index a8a5c75a2..a579bf936 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -397,6 +397,9 @@ func checkTicketPointer(s *Session) error { if err != nil { return err } + if evidence(s).Agent.Name == codex.Name { + return checkCodexSpecPointer(s, e.Digest) + } for _, msg := range evidence(s).RawSession["main"] { if bytes.Contains(msg, []byte(pointer.Marker)) && bytes.Contains(msg, []byte(e.Digest)) { return nil @@ -426,6 +429,9 @@ func checkSkillTracked(s *Session) error { if e.Metadata == nil || e.Metadata.InvocationCount == 0 { return errors.New("the skill entry has no invocation") } + if evidence(s).Agent.Name == codex.Name { + return checkCodexSkillMaterials(s, e) + } return nil } return fmt.Errorf("the evidence has no entry for the skill %s", skillName) diff --git a/app/cli/internal/trace/e2e/checks_codex.go b/app/cli/internal/trace/e2e/checks_codex.go new file mode 100644 index 000000000..acf11a636 --- /dev/null +++ b/app/cli/internal/trace/e2e/checks_codex.go @@ -0,0 +1,170 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// codexViewerPointer deliberately ignores source_event and other native +// metadata. A matching digest there alone cannot make a viewer link work. +func codexViewerPointer(lines []json.RawMessage, digest string) bool { + for _, raw := range lines { + var entry trace.RawSessionEntry + if json.Unmarshal(raw, &entry) == nil && jsonPointer(entry.Message.Content, digest) { + return true + } + } + return false +} + +func jsonPointer(raw json.RawMessage, digest string) bool { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) == nil && fields != nil { + var kind, value string + _ = json.Unmarshal(fields["type"], &kind) + _ = json.Unmarshal(fields["digest"], &value) + if kind == pointer.Marker && value == digest { + return true + } + for _, key := range []string{"content", "source", "input", pointer.Marker} { + if jsonPointer(fields[key], digest) { + return true + } + } + return false + } + var blocks []json.RawMessage + if json.Unmarshal(raw, &blocks) == nil { + for _, block := range blocks { + if jsonPointer(block, digest) { + return true + } + } + } + return false +} + +func checkCodexSpecPointer(s *Session, digest string) error { + content, err := s.Stored.Material(digest) + if err != nil { + return err + } + root := s.Checkout + if root == "" { + root = s.Workspace.Dir + } + original, err := os.ReadFile(filepath.Join(root, ticketPath)) + if err != nil { + return err + } + for _, raw := range evidence(s).RawSession["main"] { + if inlineSource(raw, content) || inlineSource(raw, original) { + return errors.New("the Codex transcript retains an inline copy of the stored spec") + } + } + if !codexViewerPointer(evidence(s).RawSession["main"], digest) { + return errors.New("the Codex viewer message has no spec material pointer") + } + return nil +} + +func inlineSource(raw json.RawMessage, content []byte) bool { + if len(content) == 0 { + return false + } + var text string + if json.Unmarshal(raw, &text) == nil { + if bytes.Contains([]byte(text), content) { + return true + } + if json.Valid([]byte(text)) { + return inlineSource(json.RawMessage(text), content) + } + return false + } + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) == nil && fields != nil { + for _, value := range fields { + if inlineSource(value, content) { + return true + } + } + return false + } + var blocks []json.RawMessage + if json.Unmarshal(raw, &blocks) == nil { + for _, block := range blocks { + if inlineSource(block, content) { + return true + } + } + } + return false +} + +func checkCodexSkillMaterials(s *Session, entry aicodingsession.SpecEntry) error { + definition, err := s.Stored.Material(entry.Digest) + if err != nil { + return fmt.Errorf("read the Codex skill definition: %w", err) + } + if !bytes.Contains(definition, []byte(skillName)) || !bytes.Contains(definition, []byte("farewell.txt")) { + return errors.New("the Codex skill material does not contain its definition") + } + if entry.Metadata.PackageDigest == "" { + return errors.New("the Codex skill has no supporting-file package") + } + data, err := s.Stored.Material(entry.Metadata.PackageDigest) + if err != nil { + return err + } + compressed, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return err + } + defer compressed.Close() + archive := tar.NewReader(compressed) + for { + header, err := archive.Next() + if errors.Is(err, io.EOF) { + return errors.New("the Codex skill package has no matching definition") + } + if err != nil { + return err + } + if filepath.Base(header.Name) == "SKILL.md" { + content, err := io.ReadAll(io.LimitReader(archive, 1<<20)) + if err != nil { + return err + } + if !bytes.Equal(content, definition) { + return errors.New("the Codex skill package differs from its stored definition") + } + return nil + } + } +} diff --git a/app/cli/internal/trace/e2e/checks_codex_test.go b/app/cli/internal/trace/e2e/checks_codex_test.go new file mode 100644 index 000000000..cb685f5cd --- /dev/null +++ b/app/cli/internal/trace/e2e/checks_codex_test.go @@ -0,0 +1,38 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCodexViewerMaterialPointer(t *testing.T) { + for _, tc := range []struct { + name, line string + want bool + }{ + {name: "metadata alone", line: `{"message":{"content":"Hello"},"source_event":{"type":"chainloop.replaced","digest":"sha256:spec"}}`}, + {name: "tool result", line: `{"message":{"content":[{"type":"tool_result","content":[{"type":"chainloop.replaced","digest":"sha256:spec"}]}]}}`, want: true}, + {name: "image source", line: `{"message":{"content":[{"type":"image","source":{"type":"chainloop.replaced","digest":"sha256:spec"}}]}}`, want: true}, + {name: "wrong material", line: `{"message":{"content":[{"type":"chainloop.replaced","digest":"sha256:other"}]}}`}, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, codexViewerPointer([]json.RawMessage{json.RawMessage(tc.line)}, "sha256:spec")) + }) + } +} diff --git a/app/cli/internal/trace/e2e/checks_more.go b/app/cli/internal/trace/e2e/checks_more.go index 91c4d1a4e..716242af3 100644 --- a/app/cli/internal/trace/e2e/checks_more.go +++ b/app/cli/internal/trace/e2e/checks_more.go @@ -30,6 +30,7 @@ import ( "time" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" tracegit "github.com/chainloop-dev/chainloop/app/cli/internal/trace/git" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" @@ -179,6 +180,20 @@ func checkBothCommitsCredited(s *Session) error { // checkPastedImage checks that the pasted image is stored once as an image // spec entry, and that the transcript holds a pointer in place of the image. func checkPastedImage(s *Session) error { + if evidence(s).Agent.Name == codex.Name { + count := 0 + for _, e := range evidence(s).Spec { + if e.Kind == aicodingsession.SpecKindImage { + count++ + if !codexViewerPointer(evidence(s).RawSession["main"], e.Digest) { + return errors.New("the Codex viewer message has no image material pointer") + } + } + } + if count != 1 { + return fmt.Errorf("the repeated Codex paste produced %d image materials, want one", count) + } + } for _, e := range evidence(s).Spec { if e.Kind != aicodingsession.SpecKindImage || e.Title != "Pasted image 1" { continue diff --git a/app/cli/internal/trace/e2e/scenarios.go b/app/cli/internal/trace/e2e/scenarios.go index f9cd0606e..63915979e 100644 --- a/app/cli/internal/trace/e2e/scenarios.go +++ b/app/cli/internal/trace/e2e/scenarios.go @@ -240,7 +240,7 @@ Then run: git add farewell.txt && git commit -m "add farewell" && git push scenarioPastedImage: { Name: scenarioPastedImage, Seed: map[string]string{readmePath: seedReadme}, - Images: [][]byte{pastedPNG()}, + Images: [][]byte{pastedPNG(), pastedPNG()}, Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: 1. With your file-writing tool, create the file color.txt with one word: the main color of the pasted image. 2. Run this shell command: git add color.txt && git commit -m "add color" && git push`, diff --git a/app/cli/pkg/action/trace_codex_materials_test.go b/app/cli/pkg/action/trace_codex_materials_test.go new file mode 100644 index 000000000..fca065db5 --- /dev/null +++ b/app/cli/pkg/action/trace_codex_materials_test.go @@ -0,0 +1,155 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:goconst // Keep native protocol literals readable in regression fixtures. +package action + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/json" + "io" + "os" + "path/filepath" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/spec" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCodexMaterialAssembly(t *testing.T) { + const sessionID = "c0de42-session" + const task = "# Task\n\nImplement the requested change.\n" + const definition = "---\nname: example\ndescription: Test skill.\n---\n\nRead reference.txt.\n" + image := []byte("\x89PNG\r\n\x1a\nscreenshot") + encodedImage := base64.StdEncoding.EncodeToString(image) + for _, tc := range []struct { + name string + reuse bool + fail bool + }{ + {name: "materials and viewer pointers"}, + {name: "captured image wins", reuse: true}, + {name: "storage failure preserves inline image", fail: true}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + t.Setenv("CODEX_HOME", home) + root := t.TempDir() + skillDir := filepath.Join(root, ".agents", "skills", "example") + require.NoError(t, os.MkdirAll(skillDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(definition), 0600)) + reference := filepath.Join(skillDir, "reference.txt") + require.NoError(t, os.WriteFile(reference, []byte("original reference"), 0600)) + var rollout bytes.Buffer + add := func(kind string, payload any) { + t.Helper() + require.NoError(t, json.NewEncoder(&rollout).Encode(struct { + Type string `json:"type"` + Timestamp string `json:"timestamp"` + Payload any `json:"payload"` + }{Type: kind, Timestamp: "2026-10-21T10:00:00Z", Payload: payload})) + } + add("session_meta", map[string]string{"id": sessionID, "cwd": root, "cli_version": "0.163.0"}) + add("response_item", map[string]any{"type": "message", "role": "user", "content": []map[string]string{ + {"type": "input_image", "image_url": "data:image/png;base64," + encodedImage}, + {"type": "input_image", "image_url": "data:image/png;base64," + encodedImage}, + }}) + args, err := json.Marshal(map[string]string{"cmd": "cat " + filepath.Join(skillDir, "SKILL.md")}) + require.NoError(t, err) + add("response_item", map[string]string{"type": "function_call", "name": "exec_command", "call_id": "skill", "arguments": string(args)}) + add("response_item", map[string]string{"type": "function_call_output", "call_id": "skill", "output": definition}) + add("response_item", map[string]string{"type": "function_call", "name": "exec_command", "call_id": "task", "arguments": `{"cmd":"cat task.md"}`}) + add("response_item", map[string]string{"type": "function_call_output", "call_id": "task", "output": task}) + path := filepath.Join(home, "sessions", "rollout-"+sessionID+".jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, rollout.Bytes(), 0600)) + provider := codex.New() + store := state.NewOutOfTreeStore(t.TempDir()) + captureSkillLoads(provider, store, &trace.HookInput{SessionID: sessionID, TranscriptPath: path}, zerolog.Nop()) + // The first-use snapshot, including supporting files, must survive + // later edits of the live package. + require.NoError(t, os.WriteFile(reference, []byte("changed after load"), 0600)) + require.NoError(t, provider.CopySessionData(store, trace.SessionLocation{SessionID: sessionID, TranscriptPath: path})) + opts := &trace.ParseOpts{SessionID: sessionID, SessionDir: store.RawSessionDir()} + evidence, err := provider.ParseSession(t.Context(), opts) + require.NoError(t, err) + skills, warnings := readSessionSkills(provider, store, opts, root, zerolog.Nop()) + require.Empty(t, warnings) + require.Len(t, skills, 1) + captures := []spec.Capture{specCapture(t, "task.md", "---\nkind: ticket\ntitle: Task\n---\n"+task, "2026-10-21T10:00:00Z")} + captures[0].SourceDigest = pointer.Digest([]byte(task)) + if tc.reuse { + captures = append(captures, spec.Capture{FileName: "reference.png", Kind: aicodingsession.SpecKindImage, Raw: image, Verbatim: true}) + } + adder := &fakeMaterialAdder{realDigests: true} + if tc.fail { + adder.failOn = map[string]bool{"spec-c0de42-pasted-image-1": true} + } + attested, _ := attachSessionEvidence(t.Context(), adder, store, []sessionEvidence{{sessionID: sessionID, provider: provider, evidence: evidence, specs: captures, skills: skills}}, zerolog.Nop(), true) + require.Equal(t, []string{sessionID}, attested) + var result aicodingsession.Evidence + session := adder.byName("ai-coding-session-c0de42") + require.NoError(t, json.Unmarshal([]byte(session.content), &result)) + assert.Equal(t, tc.fail, bytes.Contains([]byte(session.content), []byte(encodedImage))) + imageCount := 0 + for _, entry := range result.Data.Spec { + switch entry.Kind { + case aicodingsession.SpecKindImage: + imageCount++ + assert.Contains(t, session.content, entry.Digest) + case aicodingsession.SpecKindSkill: + require.NotNil(t, entry.Metadata) + assert.Equal(t, 1, entry.Metadata.ByModel) + archive := adder.byName("spec-c0de42-skill-example-pkg") + require.Equal(t, entry.Metadata.PackageDigest, archive.digest) + compressed, err := gzip.NewReader(bytes.NewReader([]byte(archive.content))) + require.NoError(t, err) + defer compressed.Close() + reader := tar.NewReader(compressed) + found := false + for { + header, err := reader.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + if filepath.Base(header.Name) == "reference.txt" { + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "original reference", string(content)) + found = true + } + } + assert.True(t, found) + } + } + if tc.fail { + assert.Zero(t, imageCount) + } else { + assert.Equal(t, 1, imageCount) + } + assert.Contains(t, string(result.Data.RawSession["main"][5]), pointer.Marker) + }) + } +} diff --git a/docs/specs/issue-3657-codex-trace-provider.md b/docs/specs/issue-3657-codex-trace-provider.md index 75bb51022..f6252dc86 100644 --- a/docs/specs/issue-3657-codex-trace-provider.md +++ b/docs/specs/issue-3657-codex-trace-provider.md @@ -34,7 +34,9 @@ Developers who use Codex cannot attach native Trace evidence to their commits. C - Goal: Developers can record Codex sessions in local storage or the control plane. - Goal: Evidence includes conversations, models, tokens, estimated cost, tools, and code attribution. -- Non-goal: Capture child-session transcripts, track skills, or replace spec and pasted-image copies with pointers. These features need separate transcript analysis. +- ~~Non-goal: Capture child-session transcripts, track skills, or replace spec and pasted-image copies with pointers. These features need separate transcript analysis.~~ Superseded by D-004. +- Goal: Evidence includes used skills, stored spec materials, and pointers that the session viewer can resolve. +- Non-goal: Capture child-session transcripts. Milestone 2 analyzes the root session only. - Non-goal: Attribute arbitrary MCP tools or reconstruct sessions that produced no Trace hook. - Non-goal: Change the evidence schema, control-plane APIs, or default provider. @@ -64,16 +66,38 @@ Trace MUST parse its private copy of the session transcript. Copying MUST check Trace MUST send spec capture instructions at session start and reminders at each user prompt. Each hook response MUST contain at most one JSON document. Empty announcements MUST emit no output. Initialization MUST tell Codex users to review and trust the installed hooks. +### R-007: Skills used during the session + +Trace MUST record the skills that a Codex session used, with their invocation counts and first-use times when available. Trace MUST capture each skill definition and its supporting files through the existing skill evidence pipeline. Trace MUST attempt capture during the session. A copy made only at push MUST carry a warning. An available skill listing alone MUST NOT count as a skill use. + +### R-008: Pointers to spec sources + +Trace MUST replace exact copies of stored spec sources in transcript reads and writes with pointers to the corresponding materials. Replacement MUST cover both the conversation shown by the viewer and retained native copies of the same content. Partial reads, changed content, failed operations, and unsupported formats MUST remain inline. A pointer MUST reference a successfully stored material. + +### R-009: Pasted images + +Trace MUST extract each distinct pasted image from the transcript and store it once as an image spec material. Trace MUST replace repeated inline copies with pointers to that material. If an existing spec material stores the same image, Trace MUST reuse it. Images from tool results MUST NOT count as pasted images. If decoding, extraction, or storage fails, Trace MUST preserve the inline data and continue evidence assembly. + +### R-010: Verification and compatibility + +Provider regressions and live Codex tests MUST check captured skill contents, spec pointers, and pasted-image materials and pointers. Verification MUST cover image deduplication, inline fallback, and use of the captured materials and pointers by the existing session viewer. The compatibility tables MUST reflect test results and document remaining limits. Both milestones MUST pass their required verification before the parent issue closes. + ## Constraints Native hook behavior follows Codex CLI 0.163.0. Codex requires users to trust project configuration and hook definitions. Trace does not bypass that review during normal use. The test harness bypasses hook trust only in isolated test repositories. +Milestone 2 MUST use the existing evidence schema and material storage pipeline. The provider MUST declare compile-time conformance to all three optional extensions for skills, spec pointers, and pasted images. Codex MUST support these extensions. Local evidence assembly MUST retain the offline behavior in R-005. + ## Proposal The developer selects Codex during Trace initialization, reviews the installed hooks in Codex, and starts a session. Codex sends session and tool events to Trace. Trace records each patch path separately and uses working-tree snapshots for shell changes. Trace copies the Codex rollout into private storage at session boundaries and before evidence assembly. The parser retains native JSONL records and derives summaries from metadata, response items, turn context, and cumulative token counters. Git hooks use the existing evidence pipeline. +For milestone 2, Trace detects actual skill loads and attempts to copy their contents during the session. Evidence assembly recovers skill usage from the private transcript copy. The shared pipeline stores skill definitions and supporting files with their usage metadata. + +Before storing the session evidence, Trace stores spec materials and distinct pasted images. It then replaces matching transcript copies with pointers to those materials. Replacement covers viewer messages and retained native events. Unsupported or unsuccessful matches remain inline. The viewer resolves pointers through the existing material digests. + ```mermaid flowchart LR A[Initialize Trace for Codex] --> B[Review and trust hooks] @@ -109,10 +133,25 @@ The provider reads copied native rollouts and estimates cost from embedded price - Alternatives considered: Live transcript parsing depends on agent storage during assembly. Notification payloads omit conversation and usage details. - Source: drafting. +### D-004: Include skills, spec pointers, and pasted images in milestone 2 + +Supersedes the original non-goal that excluded skills and replacement of spec and pasted-image copies. Child-session transcripts remain outside scope. + +- Why: The parent issue requires these features for complete Codex support. Milestone 1 delivers the core provider first. +- Alternatives considered: Permanent exclusion leaves required evidence incomplete. A separate spec would separate requirements that the parent issue already groups into two milestones. +- Source: [Milestone 2 of issue #3657](https://github.com/chainloop-dev/chainloop/issues/3657). + ## Open Questions None for the initial provider scope. +Milestone 2 has no additional open questions. + +## Milestones + +1. **Core Codex provider.** Deliver provider selection, hooks, session evidence, attribution, local storage, control-plane uploads, and session messages. Check the behavior and document compatibility. +2. **Codex skills, spec pointers, and pasted images.** Deliver R-007 through R-010 through the shared evidence pipeline. Check the materials and pointers with live Codex sessions and the existing viewer. + ## Risks | Risk | Mitigation | @@ -120,3 +159,6 @@ None for the initial provider scope. | Codex changes its rollout format. | Preserve unknown records, warn about unreadable records, and test sanitized rollout fixtures. | | Users have not trusted the hooks. | Explain the trust step during initialization and in provider documentation. | | Estimated cost differs from billing. | Document standard API estimates and warn when the model price is unknown. | +| A skill changes or disappears before push. | Attempt capture during the session and warn when only a push-time copy is available. | +| Pointer replacement misses a retained native copy. | Check that viewer messages and native events contain no duplicate content in regression tests. | +| A pasted image cannot be decoded or stored. | Preserve its inline data and continue evidence assembly. | From a92d6d0b660d88a1d5514cc7f9e3c981a5a8f5ea Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 20:06:06 +0200 Subject: [PATCH 2/4] docs(trace): record full Codex E2E verification Refs #3657 Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: 01a15f80-7cef-7100-a8fc-389f64109471 --- app/cli/internal/trace/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 5602cef42..8c4bcc80a 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -12,9 +12,9 @@ Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --local --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Installed hooks have a `Chainloop Trace` status label in Codex. Re-running init adds labels to existing Chainloop hooks. Changing the hooks requires a new trust review. -Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 22 core feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts are not implemented. +Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: all 25 tested feature checks passed in the full Codex suite. The tables keep unverified features at **Unknown**. Subagent transcripts are not implemented. -Milestone 2 live checks passed with the same CLI and model: skills in 3/3 sessions, spec pointers in 2/3 sessions (the suite requires 2/3), and pasted images in 1/1 session. The image scenario attaches the same image twice and requires one stored material. Transcript rendering and media redaction checks also passed. Provider and assembly regressions cover malformed images, storage failures, and reuse of an existing image material. +Milestone 2 live checks passed with the same CLI and model: skills in 3/3 sessions, spec pointers in 3/3 sessions, and pasted images in 1/1 session. The image scenario attaches the same image twice and requires one stored material. Transcript rendering and media redaction checks also passed. Provider and assembly regressions cover malformed images, storage failures, and reuse of an existing image material. Codex token usage comes from cumulative rollout counters. Cost is an estimate from the embedded standard API prices, with a warning for unknown models. Subscription billing and third-party provider prices can differ. Trace reads rollouts from `$CODEX_HOME/sessions` (default `~/.codex/sessions`) or `archived_sessions`. Their JSONL format is not a stable Codex interface. @@ -123,7 +123,7 @@ Why not Yes: - **Spec capture reminder, Cursor:** Cursor has no hook at each user prompt. The agent gets the session-start instruction only. - **Spec source role, title, and description, OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). - **Skills tracking, Codex:** tracks filesystem skills loaded through explicit Codex skill instructions or successful shell reads of one identified `SKILL.md`. The definition and supporting files use the shared skill snapshot and packaging pipeline. Multiple distinct skill paths in one tool call cannot be attributed reliably and are not tracked. Skills without a filesystem folder and child sessions are not captured. A snapshot first made at push carries a warning. -- **Spec source pointers, Codex:** supports exact text results, native shell envelopes, recognized code-mode JSON envelopes, and complete file creation through native patches or literal patches in code mode. Stored source files read with `cat` can also match within combined command output and native file-change records. Mixed or computed patches, arbitrary output wrappers, transformed content, and other unsupported formats remain inline. One live session trimmed the source's trailing newline and produced no viewer pointer, so the table remains Partial. +- **Spec source pointers, Codex:** supports exact text results, native shell envelopes, recognized code-mode JSON envelopes, and complete file creation through native patches or literal patches in code mode. Stored source files read with `cat` can also match within combined command output and native file-change records. Mixed or computed patches, arbitrary output wrappers, transformed content, and other unsupported formats remain inline, so the table remains Partial. Earlier live sessions demonstrated this limit when the model trimmed the source's trailing newline. - **Pasted image pointers, Codex:** extracts base64 image data URLs from user input, deduplicates their decoded bytes, and replaces stored copies in both viewer messages and retained events. Tool-result images are not pastes. Invalid data, remote URLs, and path-only image records remain inline. - **Skills tracking, Cursor:** the provider does not track skills. The evidence has no skill entries, which means "not recorded", not "no skill used". - **Skills tracking, OpenCode:** only the skills that the model starts with the `skill` tool are counted. Skills that the user starts, and skills used in subagents, are not. A plugin installed by an earlier version does not send the skill folder for OpenCode 2: install the hooks again to update it. From 00bafbf88671c8d53e31992750f0f4de90e85d84 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 20:15:50 +0200 Subject: [PATCH 3/4] fix(trace): preserve canonical Codex material pointers Keep output metadata outside material pointers, retain canonical pointers in native timeline records, and recognize nested skill results and valid source text that resembles an error. Clarify the documented support limits and pasted-image requirement. Refs #3657 Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: 01a15f80-7cef-7100-a8fc-389f64109471 --- app/cli/internal/trace/README.md | 2 +- .../internal/trace/codex/extensions_test.go | 77 +++++++++++++ app/cli/internal/trace/codex/materials.go | 102 ++++++++++++------ app/cli/internal/trace/codex/skills.go | 4 +- app/cli/internal/trace/e2e/checks_codex.go | 2 +- .../internal/trace/e2e/checks_codex_test.go | 1 + docs/specs/issue-3657-codex-trace-provider.md | 2 +- 7 files changed, 154 insertions(+), 36 deletions(-) diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 8c4bcc80a..194b1d368 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -113,7 +113,7 @@ What the session was asked to build, and which skills it used. | Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | Unknown | | Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | Yes | | Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Yes | Unknown | Yes | No | Yes | -| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | Yes | +| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | Partial | | Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | Partial | | Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | Yes | diff --git a/app/cli/internal/trace/codex/extensions_test.go b/app/cli/internal/trace/codex/extensions_test.go index 49beabea9..7bef6f7be 100644 --- a/app/cli/internal/trace/codex/extensions_test.go +++ b/app/cli/internal/trace/codex/extensions_test.go @@ -189,6 +189,9 @@ func TestCodexCodeModeCopies(t *testing.T) { fileChange := extensionEvent(t, nativeEventMsg, map[string]any{typeField: "item_completed", "item": map[string]any{typeField: "FileChange", "changes": map[string]any{"ticket.md": map[string]string{typeField: "add", contentField: "# Capture metadata\n" + source + "\n"}}}}) raw := map[string][]json.RawMessage{mainTranscriptKey: {extensionEvent(t, nativeSessionMeta, map[string]string{"id": testSessionID, cwdField: dir}), normalizedExtension(t, call), normalizedExtension(t, result), timeline, fileChange}} New().ReplaceSpecCopies(raw, pointer.Sources{pointer.Digest([]byte(source)): digest}) + for _, line := range raw[mainTranscriptKey] { + assertCanonicalPointers(t, line) + } for _, index := range []int{2, 3, 4} { assert.NotContains(t, string(raw[mainTranscriptKey][index]), "Implement this task.") assert.Contains(t, string(raw[mainTranscriptKey][index]), digest) @@ -202,6 +205,80 @@ func TestCodexCodeModeCopies(t *testing.T) { } } +func assertCanonicalPointers(t *testing.T, raw json.RawMessage) { + t.Helper() + if fields := object(raw); fields != nil { + if jsonString(fields[typeField]) == pointer.Marker { + assert.Len(t, fields, 2, "a pointer contains only its type and digest") + } + if replaced, ok := fields[pointer.Marker]; ok { + value := object(replaced) + require.NotNil(t, value, "a text-field marker holds one pointer, not content blocks") + assert.Equal(t, pointer.Marker, jsonString(value[typeField])) + assert.Len(t, value, 2) + } + for _, value := range fields { + assertCanonicalPointers(t, value) + } + return + } + var values []json.RawMessage + if json.Unmarshal(raw, &values) == nil { + for _, value := range values { + assertCanonicalPointers(t, value) + } + } +} + +func TestCodexSkillResultEnvelopes(t *testing.T) { + const definition = "---\nname: bundled\n---\nDo the task.\n" + for _, tc := range []struct { + name string + value any + want []string + }{ + {name: "successful promise", value: map[string]any{"status": "fulfilled", "value": map[string]any{"exit_code": 0, "output": definition}}, want: []string{definition}}, + {name: "failed nested command", value: map[string]any{"status": "fulfilled", "value": map[string]any{"exit_code": 1, "output": definition}}}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := mustJSON(t, tc.value) + assert.Equal(t, tc.want, outputTexts(raw)) + assert.Equal(t, tc.want, outputTexts(mustJSON(t, string(raw)))) + }) + } +} + +func TestCodexSkillPathResolution(t *testing.T) { + assert.Equal(t, []string{"/skills/one/SKILL.md"}, skillReadPaths("cat relative/SKILL.md; cat /skills/one/SKILL.md", "")) +} + +func TestCodexSourceLooksLikeFailure(t *testing.T) { + const source = "Error: the page is missing.\n" + sources := pointer.Sources{pointer.Digest([]byte(source)): "sha256:source"} + for _, tc := range []struct { + name string + output any + want bool + }{ + {name: "plain text", output: source, want: true}, + {name: "code mode text", output: []map[string]string{{typeField: "input_text", textField: "Script completed\nWall time 0.1 seconds\nOutput:\n"}, {typeField: "input_text", textField: source}}, want: true}, + {name: "successful command", output: map[string]any{"exit_code": 0, "output": source}, want: true}, + {name: "failed command", output: map[string]any{"exit_code": 1, "output": source}}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := mustJSON(t, tc.output) + updated, n := replaceOutput(raw, sources, nil) + assert.Equal(t, tc.want, n > 0) + if tc.want { + assert.NotContains(t, string(updated), source) + assertCanonicalPointers(t, updated) + } else { + assert.Equal(t, raw, updated) + } + }) + } +} + func TestCodexLiveSkillLoads(t *testing.T) { home := t.TempDir() t.Setenv("CODEX_HOME", home) diff --git a/app/cli/internal/trace/codex/materials.go b/app/cli/internal/trace/codex/materials.go index 0ce4ff421..a3ea3db5f 100644 --- a/app/cli/internal/trace/codex/materials.go +++ b/app/cli/internal/trace/codex/materials.go @@ -46,6 +46,7 @@ const ( toolApplyPatch = "apply_patch" typeField = "type" outputField = "output" + valueField = "value" ) var catPathPattern = regexp.MustCompile(`\bcat\s+(?:--\s+)?(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"';&|()]+))`) @@ -325,8 +326,12 @@ func (p *Provider) ReplaceSpecCopies(raw map[string][]json.RawMessage, sources p case functionCallOutput, customToolCallOutput: call := calls[payload.CallID] if isReadTool(call.Name) { - if value, n := replaceOutput(fields[outputField], sources, copies); n > 0 { + replacer := outputReplacer{sources: sources, copies: copies} + if value, n := replacer.replace(fields[outputField]); n > 0 { fields[outputField] = value + if len(replacer.metadata) > 0 { + fields["codex_output"] = encodeJSON(replacer.metadata) + } res := report.Finder(pointer.FinderFileRead) res.Replaced += n changed = true @@ -361,8 +366,7 @@ func replaceTimelineItem(fields jsonObject, sources pointer.Sources, copies sour return false } if updated, n := replaceOutput(item["aggregated_output"], sources, copies); n > 0 { - delete(item, "aggregated_output") - item[pointer.Marker] = updated + storeTimelineCopy(item, "aggregated_output", updated) report.Finder(pointer.FinderFileRead).Replaced += n changed = true } @@ -377,8 +381,7 @@ func replaceTimelineItem(fields jsonObject, sources pointer.Sources, copies sour continue } if updated, n := replaceOutput(change[contentField], sources, copies); n > 0 { - delete(change, contentField) - change[pointer.Marker] = updated + storeTimelineCopy(change, contentField, updated) changes[path] = encodeJSON(change) report.Finder(pointer.FinderFileWrite).Replaced += n changed = true @@ -394,6 +397,16 @@ func replaceTimelineItem(fields jsonObject, sources pointer.Sources, copies sour return changed } +func storeTimelineCopy(fields jsonObject, original string, updated json.RawMessage) { + delete(fields, original) + var blocks []json.RawMessage + if json.Unmarshal(updated, &blocks) == nil && len(blocks) == 1 && jsonString(object(blocks[0])[typeField]) == pointer.Marker { + fields[pointer.Marker] = blocks[0] + return + } + fields[contentField] = updated +} + func isReadTool(name string) bool { switch name { case "exec", toolExecCommand, "shell", "shell_command", "Bash", "read_file", "read", "write_stdin": @@ -471,39 +484,54 @@ func failedOutput(raw json.RawMessage) bool { return json.Unmarshal(raw, &result) == nil && (result.IsError || result.ExitCode != nil && *result.ExitCode != 0) } +type outputReplacer struct { + sources pointer.Sources + copies sourceCopies + metadata []json.RawMessage +} + func replaceOutput(raw json.RawMessage, sources pointer.Sources, copies sourceCopies) (json.RawMessage, int) { - if failedOutput(raw) { + r := outputReplacer{sources: sources, copies: copies} + return r.replace(raw) +} + +func (r *outputReplacer) replace(raw json.RawMessage) (json.RawMessage, int) { + // Structured failure status wins over source matching. A text prefix is + // only a heuristic: a source document can itself start with "Error:". + if object(raw) != nil && failedOutput(raw) { return raw, 0 } var text string if json.Unmarshal(raw, &text) == nil { stdout, ok := stdoutText(text) if ok { - if material, found := sources[pointer.Digest([]byte(stdout))]; found { + if material, found := r.sources[pointer.Digest([]byte(stdout))]; found { return encodeJSON([]json.RawMessage{pointer.New(material)}), 1 } - if updated, n := replaceTextCopies(stdout, copies); n > 0 { + } + if failedOutput(raw) { + return raw, 0 + } + if ok { + if updated, n := replaceTextCopies(stdout, r.copies); n > 0 { return updated, n } } // Code mode exposes nested exec_command results as JSON text blocks. if nested := object(json.RawMessage(text)); nested != nil { - if updated, n := replaceOutput(json.RawMessage(text), sources, copies); n > 0 { - // Keep envelope metadata with the pointer while exposing a - // supported block type to the viewer. - var pointers []jsonObject - collectPointers(updated, &pointers) - if len(pointers) > 0 { - pointers[0]["codex_output"] = updated - return encodeJSON(pointers), n + if updated, n := r.replace(json.RawMessage(text)); n > 0 { + // Metadata belongs to the native event, outside canonical pointers. + if blocks := outputContent(updated); len(blocks) > 0 { + r.metadata = append(r.metadata, updated) + return encodeJSON(blocks), n } } } return raw, 0 } if fields := object(raw); fields != nil { - for _, key := range []string{outputField, "value"} { - if value, n := replaceOutput(fields[key], sources, copies); n > 0 { + for _, key := range []string{outputField, valueField} { + if value, n := r.replace(fields[key]); n > 0 { fields[key] = value return encodeJSON(fields), n } @@ -514,6 +542,12 @@ func replaceOutput(raw json.RawMessage, sources pointer.Sources, copies sourceCo if json.Unmarshal(raw, &blocks) != nil { return raw, 0 } + if len(blocks) > 0 { + frame := jsonString(blocks[0][textField]) + if strings.HasPrefix(frame, "Script failed\n") && strings.Contains(frame, "\nWall time") { + return raw, 0 + } + } n := 0 var replaced []jsonObject for _, block := range blocks { @@ -521,7 +555,7 @@ func replaceOutput(raw json.RawMessage, sources pointer.Sources, copies sourceCo replaced = append(replaced, block) continue } - if updated, count := replaceOutput(block[textField], sources, copies); count > 0 { + if updated, count := r.replace(block[textField]); count > 0 { var values []jsonObject if json.Unmarshal(updated, &values) == nil && len(values) > 0 { replaced = append(replaced, values...) @@ -537,23 +571,22 @@ func replaceOutput(raw json.RawMessage, sources pointer.Sources, copies sourceCo return raw, 0 } -func collectPointers(raw json.RawMessage, out *[]jsonObject) { +// outputContent finds the replaced result blocks inside a recognized envelope. +// It preserves surrounding text in addition to material pointers. +func outputContent(raw json.RawMessage) []json.RawMessage { if fields := object(raw); fields != nil { - if jsonString(fields[typeField]) == pointer.Marker { - *out = append(*out, fields) - return - } - for _, key := range []string{outputField, "value"} { - collectPointers(fields[key], out) + for _, key := range []string{outputField, valueField} { + if blocks := outputContent(fields[key]); len(blocks) > 0 { + return blocks + } } - return + return nil } var blocks []json.RawMessage if json.Unmarshal(raw, &blocks) == nil { - for _, block := range blocks { - collectPointers(block, out) - } + return blocks } + return nil } func replaceTextCopies(text string, copies sourceCopies) (json.RawMessage, int) { @@ -709,7 +742,7 @@ func outputTexts(raw json.RawMessage) []string { } var text string if json.Unmarshal(raw, &text) == nil { - if fields := object(json.RawMessage(text)); fields != nil && len(fields[outputField]) > 0 { + if fields := object(json.RawMessage(text)); fields != nil && (len(fields[outputField]) > 0 || len(fields[valueField]) > 0) { return outputTexts(json.RawMessage(text)) } stdout, ok := stdoutText(text) @@ -719,7 +752,12 @@ func outputTexts(raw json.RawMessage) []string { return nil } if fields := object(raw); fields != nil { - return outputTexts(fields[outputField]) + for _, key := range []string{outputField, valueField} { + if value, ok := fields[key]; ok { + return outputTexts(value) + } + } + return nil } var blocks []jsonObject if json.Unmarshal(raw, &blocks) != nil { diff --git a/app/cli/internal/trace/codex/skills.go b/app/cli/internal/trace/codex/skills.go index c95604118..653d695fe 100644 --- a/app/cli/internal/trace/codex/skills.go +++ b/app/cli/internal/trace/codex/skills.go @@ -286,7 +286,9 @@ func skillReadPaths(input, cwd string) []string { for _, match := range catPathPattern.FindAllStringSubmatch(command, -1) { for _, path := range match[1:] { if filepath.Base(path) == skill.DefinitionFile { - paths = append(paths, resolveSkillPath(path, cwd)) + if resolved := resolveSkillPath(path, cwd); resolved != "" { + paths = append(paths, resolved) + } } } } diff --git a/app/cli/internal/trace/e2e/checks_codex.go b/app/cli/internal/trace/e2e/checks_codex.go index acf11a636..1aae2434b 100644 --- a/app/cli/internal/trace/e2e/checks_codex.go +++ b/app/cli/internal/trace/e2e/checks_codex.go @@ -49,7 +49,7 @@ func jsonPointer(raw json.RawMessage, digest string) bool { _ = json.Unmarshal(fields["type"], &kind) _ = json.Unmarshal(fields["digest"], &value) if kind == pointer.Marker && value == digest { - return true + return len(fields) == 2 } for _, key := range []string{"content", "source", "input", pointer.Marker} { if jsonPointer(fields[key], digest) { diff --git a/app/cli/internal/trace/e2e/checks_codex_test.go b/app/cli/internal/trace/e2e/checks_codex_test.go index cb685f5cd..301570e2d 100644 --- a/app/cli/internal/trace/e2e/checks_codex_test.go +++ b/app/cli/internal/trace/e2e/checks_codex_test.go @@ -30,6 +30,7 @@ func TestCodexViewerMaterialPointer(t *testing.T) { {name: "tool result", line: `{"message":{"content":[{"type":"tool_result","content":[{"type":"chainloop.replaced","digest":"sha256:spec"}]}]}}`, want: true}, {name: "image source", line: `{"message":{"content":[{"type":"image","source":{"type":"chainloop.replaced","digest":"sha256:spec"}}]}}`, want: true}, {name: "wrong material", line: `{"message":{"content":[{"type":"chainloop.replaced","digest":"sha256:other"}]}}`}, + {name: "metadata inside pointer", line: `{"message":{"content":[{"type":"chainloop.replaced","digest":"sha256:spec","codex_output":{}}]}}`}, } { t.Run(tc.name, func(t *testing.T) { assert.Equal(t, tc.want, codexViewerPointer([]json.RawMessage{json.RawMessage(tc.line)}, "sha256:spec")) diff --git a/docs/specs/issue-3657-codex-trace-provider.md b/docs/specs/issue-3657-codex-trace-provider.md index f6252dc86..728e8909d 100644 --- a/docs/specs/issue-3657-codex-trace-provider.md +++ b/docs/specs/issue-3657-codex-trace-provider.md @@ -76,7 +76,7 @@ Trace MUST replace exact copies of stored spec sources in transcript reads and w ### R-009: Pasted images -Trace MUST extract each distinct pasted image from the transcript and store it once as an image spec material. Trace MUST replace repeated inline copies with pointers to that material. If an existing spec material stores the same image, Trace MUST reuse it. Images from tool results MUST NOT count as pasted images. If decoding, extraction, or storage fails, Trace MUST preserve the inline data and continue evidence assembly. +Trace MUST extract each distinct pasted image from the transcript and store it once as an image spec material. Trace MUST replace every inline occurrence of a successfully stored pasted image with a pointer to that material. If an existing spec material stores the same image, Trace MUST reuse it. Images from tool results MUST NOT count as pasted images. If decoding, extraction, or storage fails, Trace MUST preserve the inline data and continue evidence assembly. ### R-010: Verification and compatibility From ab5873dd733849ed2478c9fc90b900ed84bea7e2 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Thu, 22 Oct 2026 00:03:55 +0200 Subject: [PATCH 4/4] fix(trace): preserve unsupported and failed Codex writes Chainloop-Trace-Sessions: 01a16073-95dd-7912-9463-66bbce3e4522 Signed-off-by: Miguel Martinez Trivino --- app/cli/internal/trace/README.md | 2 +- .../internal/trace/codex/extensions_test.go | 62 +++++++++++ app/cli/internal/trace/codex/materials.go | 105 ++++++++++++------ 3 files changed, 135 insertions(+), 34 deletions(-) diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 194b1d368..f7cde0b24 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -123,7 +123,7 @@ Why not Yes: - **Spec capture reminder, Cursor:** Cursor has no hook at each user prompt. The agent gets the session-start instruction only. - **Spec source role, title, and description, OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). - **Skills tracking, Codex:** tracks filesystem skills loaded through explicit Codex skill instructions or successful shell reads of one identified `SKILL.md`. The definition and supporting files use the shared skill snapshot and packaging pipeline. Multiple distinct skill paths in one tool call cannot be attributed reliably and are not tracked. Skills without a filesystem folder and child sessions are not captured. A snapshot first made at push carries a warning. -- **Spec source pointers, Codex:** supports exact text results, native shell envelopes, recognized code-mode JSON envelopes, and complete file creation through native patches or literal patches in code mode. Stored source files read with `cat` can also match within combined command output and native file-change records. Mixed or computed patches, arbitrary output wrappers, transformed content, and other unsupported formats remain inline, so the table remains Partial. Earlier live sessions demonstrated this limit when the model trimmed the source's trailing newline. +- **Spec source pointers, Codex:** supports exact text results, native shell envelopes, recognized code-mode JSON envelopes, and complete file creation through native patches or literal patches in code mode. A code-mode write must be a single awaited `tools.apply_patch` call, optionally wrapped in `text`, with a literal argument or an immediately preceding `const` literal. Stored source files read with `cat` can also match within combined command output and native file-change records. Mixed or computed patches, other scripts, arbitrary output wrappers, transformed content, and other unsupported formats remain inline, so the table remains Partial. Earlier live sessions demonstrated this limit when the model trimmed the source's trailing newline. - **Pasted image pointers, Codex:** extracts base64 image data URLs from user input, deduplicates their decoded bytes, and replaces stored copies in both viewer messages and retained events. Tool-result images are not pastes. Invalid data, remote URLs, and path-only image records remain inline. - **Skills tracking, Cursor:** the provider does not track skills. The evidence has no skill entries, which means "not recorded", not "no skill used". - **Skills tracking, OpenCode:** only the skills that the model starts with the `skill` tool are counted. Skills that the user starts, and skills used in subagents, are not. A plugin installed by an earlier version does not send the skill folder for OpenCode 2: install the hooks again to update it. diff --git a/app/cli/internal/trace/codex/extensions_test.go b/app/cli/internal/trace/codex/extensions_test.go index 7bef6f7be..a5d2ce1da 100644 --- a/app/cli/internal/trace/codex/extensions_test.go +++ b/app/cli/internal/trace/codex/extensions_test.go @@ -145,6 +145,68 @@ func TestCodexPastedImages(t *testing.T) { assert.JSONEq(t, string(pointer.New("sha256:stored-image")), string(blocks[0].Source)) } +func TestCodexCodePatchBoundaries(t *testing.T) { + const source = "original\n" + patch := string(mustJSON(t, "*** Begin Patch\n*** Add File: task.md\n+original\n*** End Patch")) + for _, tc := range []struct { + name, code string + want bool + }{ + {name: "direct literal", code: "text(await tools.apply_patch(" + patch + "));", want: true}, + {name: "constant literal", code: "const patch = " + patch + "; text(await tools.apply_patch(patch));", want: true}, + {name: "bare awaited literal", code: "await tools.apply_patch(" + patch + ");", want: true}, + {name: "changed constant", code: "const patch = " + patch + "; text(await tools.apply_patch(patch.replace('original', 'changed')));"}, + {name: "changed literal", code: "text(await tools.apply_patch(" + patch + ".replace('original', 'changed')));"}, + {name: "concatenated literal", code: "text(await tools.apply_patch(" + patch + " + suffix));"}, + {name: "unused literal", code: "const patch = " + patch + "; text(await tools.apply_patch(other));"}, + {name: "unrelated call", code: "text(" + patch + "); text(await tools.apply_patch(other));"}, + {name: "commented call", code: "// text(await tools.apply_patch(" + patch + "));"}, + } { + t.Run(tc.name, func(t *testing.T) { + call := extensionEvent(t, responseItem, map[string]string{typeField: customToolCall, "name": "exec", callIDField: testExtensionCallID, "input": tc.code}) + result := extensionEvent(t, responseItem, map[string]string{typeField: customToolCallOutput, callIDField: testExtensionCallID, outputField: testSuccess}) + raw := map[string][]json.RawMessage{mainTranscriptKey: {normalizedExtension(t, call), normalizedExtension(t, result)}} + before := string(mustJSON(t, raw)) + report := New().ReplaceSpecCopies(raw, pointer.Sources{pointer.Digest([]byte(source)): "sha256:original"}) + after := string(mustJSON(t, raw)) + if !tc.want { + assert.Equal(t, before, after) + assert.Empty(t, report) + return + } + assert.Contains(t, after, pointer.Marker) + assert.NotContains(t, after, "*** Begin Patch") + assertCanonicalPointers(t, raw[mainTranscriptKey][0]) + }) + } +} + +func TestCodexFailedWriteEnvelopes(t *testing.T) { + const source = "original\n" + patch := string(mustJSON(t, "*** Begin Patch\n*** Add File: task.md\n+original\n*** End Patch")) + for _, tc := range []struct { + name string + output any + }{ + {name: "JSON text", output: `{"exit_code":1,"output":"permission denied"}`}, + {name: "text block", output: []map[string]string{{typeField: "input_text", textField: `{"exit_code":1,"output":"permission denied"}`}}}, + {name: "promise", output: map[string]any{"status": "fulfilled", valueField: map[string]any{"exit_code": 1, outputField: "permission denied"}}}, + {name: "JSON promise", output: `{"status":"fulfilled","value":{"exit_code":1,"output":"permission denied"}}`}, + {name: "nested output", output: map[string]any{outputField: map[string]any{"is_error": true}}}, + {name: "tool error", output: `{"isError":true,"content":[{"type":"text","text":"permission denied"}]}`}, + {name: "rejected promise", output: map[string]any{"status": "rejected", "reason": "permission denied"}}, + } { + t.Run(tc.name, func(t *testing.T) { + call := extensionEvent(t, responseItem, map[string]string{typeField: customToolCall, "name": "exec", callIDField: testExtensionCallID, "input": "text(await tools.apply_patch(" + patch + "));"}) + result := extensionEvent(t, responseItem, map[string]any{typeField: customToolCallOutput, callIDField: testExtensionCallID, outputField: tc.output}) + raw := map[string][]json.RawMessage{mainTranscriptKey: {normalizedExtension(t, call), normalizedExtension(t, result)}} + before := string(mustJSON(t, raw)) + New().ReplaceSpecCopies(raw, pointer.Sources{pointer.Digest([]byte(source)): "sha256:original"}) + assert.Equal(t, before, string(mustJSON(t, raw))) + }) + } +} + func TestCodexSessionSkills(t *testing.T) { dir := t.TempDir() skillDir := filepath.Join(dir, ".agents", "skills", "test-skill") diff --git a/app/cli/internal/trace/codex/materials.go b/app/cli/internal/trace/codex/materials.go index a3ea3db5f..15c939cea 100644 --- a/app/cli/internal/trace/codex/materials.go +++ b/app/cli/internal/trace/codex/materials.go @@ -53,6 +53,10 @@ var catPathPattern = regexp.MustCompile(`\bcat\s+(?:--\s+)?(?:"([^"\n]+)"|'([^'\ var codeStringPattern = regexp.MustCompile(`"(?:[^"\\]|\\.)*"`) +var codePatchAssignmentPattern = regexp.MustCompile(`^\s*const\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*("(?:[^"\\]|\\.)*")\s*;\s*`) + +var codePatchCallPattern = regexp.MustCompile(`^\s*(text\s*\(\s*)?await\s+tools\.apply_patch\s*\(\s*("(?:[^"\\]|\\.)*"|[A-Za-z_$][A-Za-z0-9_$]*)\s*\)\s*(\))?\s*;?\s*$`) + type sourceCopies map[string]string // commandSources loads only explicitly read files, with the same text-size @@ -459,6 +463,9 @@ func stdoutText(text string) (string, bool) { } func failedOutput(raw json.RawMessage) bool { + if structuredOutputFailed(raw) { + return true + } text := jsonString(raw) if text != "" { for _, prefix := range []string{"Error:", "error:", "Failed to ", "apply_patch verification failed:", "Script failed"} { @@ -477,11 +484,36 @@ func failedOutput(raw json.RawMessage) bool { } } } - var result struct { - ExitCode *int `json:"exit_code"` - IsError bool `json:"is_error"` + return false +} + +// structuredOutputFailed follows recognized code-mode envelopes without +// treating document text in a successful result as an error message. +func structuredOutputFailed(raw json.RawMessage) bool { + if text := jsonString(raw); text != "" { + return structuredOutputFailed(json.RawMessage(text)) + } + if fields := object(raw); fields != nil { + var result struct { + ExitCode *int `json:"exit_code"` + IsError bool `json:"is_error"` + ToolError bool `json:"isError"` + Status string `json:"status"` + } + if json.Unmarshal(raw, &result) == nil && (result.IsError || result.ToolError || result.Status == "rejected" || result.ExitCode != nil && *result.ExitCode != 0) { + return true + } + return structuredOutputFailed(fields[outputField]) || structuredOutputFailed(fields[valueField]) + } + var blocks []struct{ Text string } + if json.Unmarshal(raw, &blocks) == nil { + for _, block := range blocks { + if structuredOutputFailed(json.RawMessage(block.Text)) { + return true + } + } } - return json.Unmarshal(raw, &result) == nil && (result.IsError || result.ExitCode != nil && *result.ExitCode != 0) + return false } type outputReplacer struct { @@ -498,7 +530,7 @@ func replaceOutput(raw json.RawMessage, sources pointer.Sources, copies sourceCo func (r *outputReplacer) replace(raw json.RawMessage) (json.RawMessage, int) { // Structured failure status wins over source matching. A text prefix is // only a heuristic: a source document can itself start with "Error:". - if object(raw) != nil && failedOutput(raw) { + if structuredOutputFailed(raw) { return raw, 0 } var text string @@ -697,43 +729,50 @@ func replaceWrite(fields jsonObject, name string, sources pointer.Sources, repor return true } -// replaceCodePatch recognizes literal patches in Codex code mode. It never -// evaluates JavaScript or reconstructs a computed patch. Other script text -// stays available alongside the material pointer in the viewer. +// replaceCodePatch accepts one awaited patch call, optionally wrapped in text, +// with a literal argument or an immediately preceding const literal. Other +// scripts stay inline because their patch arguments may have been computed. func replaceCodePatch(fields jsonObject, sources pointer.Sources, report pointer.Report) bool { code := jsonString(fields["input"]) - if !strings.Contains(code, "tools.apply_patch") { + span := codePatchLiteralSpan(code) + if span == nil { return false } - var out strings.Builder - position := 0 - var replacements []json.RawMessage - for _, span := range codeStringPattern.FindAllStringIndex(code, -1) { - literal := jsonString(json.RawMessage(code[span[0]:span[1]])) - if !strings.HasPrefix(literal, "*** Begin Patch\n") { - continue - } - patch := jsonObject{"input": encodeJSON(literal)} - if !replaceWrite(patch, toolApplyPatch, sources, report) { - continue - } - out.WriteString(code[position:span[0]]) - out.Write(patch[pointer.Marker]) - position = span[1] - replacements = append(replacements, patch[pointer.Marker]) - } - if len(replacements) == 0 { + literal := jsonString(json.RawMessage(code[span[0]:span[1]])) + patch := jsonObject{"input": encodeJSON(literal)} + if !replaceWrite(patch, toolApplyPatch, sources, report) { return false } - out.WriteString(code[position:]) - fields["input"] = encodeJSON(out.String()) - fields[pointer.Marker] = replacements[0] - if len(replacements) > 1 { - fields["spec_sources"] = encodeJSON(replacements) - } + fields["input"] = encodeJSON(code[:span[0]] + string(patch[pointer.Marker]) + code[span[1]:]) + fields[pointer.Marker] = patch[pointer.Marker] + fields["path"] = patch["path"] return true } +func codePatchLiteralSpan(code string) []int { + assignment := codePatchAssignmentPattern.FindStringSubmatchIndex(code) + offset := 0 + if assignment != nil { + offset = assignment[1] + } + call := codePatchCallPattern.FindStringSubmatchIndex(code[offset:]) + // The optional text wrapper must have its own closing parenthesis. + if call == nil || (call[2] >= 0) != (call[6] >= 0) { + return nil + } + argument := code[offset+call[4] : offset+call[5]] + if assignment != nil { + if argument != code[assignment[2]:assignment[3]] { + return nil + } + return assignment[4:6] + } + if !strings.HasPrefix(argument, `"`) { + return nil + } + return []int{offset + call[4], offset + call[5]} +} + // outputTexts reads successful text results, including code-mode envelopes, // for skill-load recognition without interpreting arbitrary tool arguments. func outputTexts(raw json.RawMessage) []string {