From be4ea7c4be8359f8bb6c553bd4386b9239eddfbd Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Tue, 10 Nov 2026 15:44:07 +0530 Subject: [PATCH 1/2] feat(sleep): add AgenticReplayBackend with three-valued outcome and isolated worktree (#155) --- skillopt_sleep/__init__.py | 4 +- skillopt_sleep/agentic_replay.py | 585 +++++++++++++++++++++++++++++++ skillopt_sleep/backend.py | 5 +- skillopt_sleep/config.py | 10 +- skillopt_sleep/replay.py | 15 +- skillopt_sleep/types.py | 10 + tests/test_agentic_replay.py | 286 +++++++++++++++ 7 files changed, 911 insertions(+), 4 deletions(-) create mode 100644 skillopt_sleep/agentic_replay.py create mode 100644 tests/test_agentic_replay.py diff --git a/skillopt_sleep/__init__.py b/skillopt_sleep/__init__.py index 9c7581f9e..05c197345 100644 --- a/skillopt_sleep/__init__.py +++ b/skillopt_sleep/__init__.py @@ -16,5 +16,7 @@ """ from __future__ import annotations -__all__ = ["__version__"] +__all__ = ["__version__", "AgenticReplayBackend"] __version__ = "0.2.0" + +from skillopt_sleep.agentic_replay import AgenticReplayBackend diff --git a/skillopt_sleep/agentic_replay.py b/skillopt_sleep/agentic_replay.py new file mode 100644 index 000000000..68b97b989 --- /dev/null +++ b/skillopt_sleep/agentic_replay.py @@ -0,0 +1,585 @@ +"""SkillOpt-Sleep — AgenticReplayBackend (Issue #155 safe first slice). + +Replay mined tasks with an interactive agent in an isolated, throwaway workspace, +and gate primarily on deterministic task verification (builds, tests, and checks) +rather than purely single-turn text completion. + +Key properties: + 1. Safe, isolated workspace with environment scrubbing (no host credential leakage) + and network isolation by default. + 2. Pinned starting commit and disposable throwaway git worktree / scratch dir. + 3. Task verification as the primary gate (deterministic test exit code/assertions); + diff similarity remains secondary audit context. + 4. Three-valued outcome: pass | fail | inconclusive. + Infrastructure failures (timeouts, budget exhaustion, missing fixtures, or + insufficient decidable actions) emit `inconclusive` with typed reason codes + so they are never counted as evidence that a candidate skill failed. + 5. Strict budgeting: explicit turn, timeout, token, and replay caps. + 6. Staged diff and report for human review without automatic adoption. +""" +from __future__ import annotations + +import os +import re +import shutil +import stat +import subprocess +import tempfile +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Tuple + +from skillopt_sleep.backend import Backend, MockBackend +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + +# ── Three-valued outcome constants ─────────────────────────────────────────── +OUTCOME_PASS = "pass" +OUTCOME_FAIL = "fail" +OUTCOME_INCONCLUSIVE = "inconclusive" + +# Typed reason codes for inconclusive outcomes +REASON_TIMEOUT = "timeout" +REASON_INFRA = "infra" +REASON_FIXTURE_FAILURE = "fixture_failure" +REASON_BUDGET_EXHAUSTED = "budget_exhausted" +REASON_INSUFFICIENT_EVIDENCE = "insufficient_evidence" +REASON_NETWORK_UNAVAILABLE = "network_unavailable" + +# Windows console window suppression +_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0) if os.name == "nt" else 0 + + +def _remove_readonly(func: Callable, path: str, _excinfo: Any) -> None: + """Clear the read-only bit and retry removal (handles Windows .git permissions).""" + try: + os.chmod(path, stat.S_IWRITE) + func(path) + except OSError: + pass + + +@dataclass +class AgenticReplayConfig: + """Budgets, sandbox boundaries, and execution settings for agentic replay.""" + + enabled: bool = False + agent_cmd: str = "" # Path or command to drive agent CLI + max_turns: int = 10 # Hard turn cap + timeout_sec: float = 60.0 # Wall-clock timeout in seconds + max_tokens: int = 20000 # Token spend cap + max_replays_per_run: int = 5 # Strict budget cap on agentic replays + sandbox_type: str = "workspace" # "workspace" (throwaway worktree) | "container" + isolate_network: bool = True # Disable external network by default + isolate_env: bool = True # Scrub host credentials and keys + min_decidable_actions: int = 1 # Floor below which outcome is insufficient_evidence + pinned_commit: str = "" # Reproducible starting commit SHA + + +class AgenticWorkspace: + """Disposable, isolated scratch workspace for running an agentic replay safely.""" + + _SENSITIVE_ENV_PREFIXES = ( + "AWS_", "AZURE_", "OPENAI_", "ANTHROPIC_", "GITHUB_", "GH_", + "GITLAB_", "SLACK_", "STRIPE_", "SSH_", "API_KEY", "SECRET", + ) + + def __init__( + self, + base_dir: str = "", + pinned_commit: str = "", + isolate_env: bool = True, + isolate_network: bool = True, + ) -> None: + self.base_dir = base_dir + self.pinned_commit = pinned_commit + self.isolate_env = isolate_env + self.isolate_network = isolate_network + self._temp_dir: Optional[str] = None + self.workspace_path: str = "" + + def __enter__(self) -> "AgenticWorkspace": + self.setup() + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.cleanup() + + def setup(self) -> str: + """Create and initialize disposable workspace directory.""" + self._temp_dir = tempfile.mkdtemp(prefix="skillopt_agentic_") + self.workspace_path = self._temp_dir + + # If a base directory was provided, copy its content into the workspace + if self.base_dir and os.path.isdir(self.base_dir): + shutil.copytree(self.base_dir, self.workspace_path, dirs_exist_ok=True) + + # Initialize git repository if not already present + git_dir = os.path.join(self.workspace_path, ".git") + if not os.path.exists(git_dir): + try: + subprocess.run( + ["git", "init"], + cwd=self.workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + creationflags=_NO_WINDOW, + ) + subprocess.run( + ["git", "config", "user.name", "SkillOpt Agentic Sandbox"], + cwd=self.workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + creationflags=_NO_WINDOW, + ) + subprocess.run( + ["git", "config", "user.email", "sandbox@skillopt.local"], + cwd=self.workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + creationflags=_NO_WINDOW, + ) + subprocess.run( + ["git", "add", "."], + cwd=self.workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=_NO_WINDOW, + ) + subprocess.run( + ["git", "commit", "-m", "baseline", "--allow-empty"], + cwd=self.workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + creationflags=_NO_WINDOW, + ) + except (subprocess.SubprocessError, OSError): + pass + elif self.pinned_commit: + try: + subprocess.run( + ["git", "checkout", self.pinned_commit], + cwd=self.workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + creationflags=_NO_WINDOW, + ) + except (subprocess.SubprocessError, OSError) as exc: + raise RuntimeError( + f"failed to checkout pinned commit {self.pinned_commit}: {exc}" + ) from exc + + return self.workspace_path + + def build_isolated_env(self) -> Dict[str, str]: + """Construct sanitized environment variables.""" + env = dict(os.environ) + if self.isolate_env: + # Strip sensitive credential keys + keys_to_remove = [ + k for k in env + if any(k.upper().startswith(prefix) for prefix in self._SENSITIVE_ENV_PREFIXES) + or "KEY" in k.upper() + or "TOKEN" in k.upper() + or "PASSWORD" in k.upper() + ] + for k in keys_to_remove: + env.pop(k, None) + + # Isolate home/user profile to prevent tampering with host configs + sandbox_home = os.path.join(self.workspace_path, ".sandbox_home") + os.makedirs(sandbox_home, exist_ok=True) + env["HOME"] = sandbox_home + env["USERPROFILE"] = sandbox_home + env["XDG_CONFIG_HOME"] = os.path.join(sandbox_home, ".config") + + if self.isolate_network: + # Direct proxy traffic to non-routable loopback sink + env["HTTP_PROXY"] = "http://127.0.0.1:9" + env["HTTPS_PROXY"] = "http://127.0.0.1:9" + env["ALL_PROXY"] = "http://127.0.0.1:9" + env["NO_PROXY"] = "" + + return env + + def get_diff(self) -> str: + """Extract unified diff against HEAD, including untracked files.""" + if not self.workspace_path or not os.path.exists(self.workspace_path): + return "" + try: + # Mark untracked files with intent-to-add so git diff includes newly added files + subprocess.run( + ["git", "add", "-N", "."], + cwd=self.workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=_NO_WINDOW, + ) + res = subprocess.run( + ["git", "diff", "HEAD"], + cwd=self.workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + creationflags=_NO_WINDOW, + ) + if res.returncode == 0: + return res.stdout + except (subprocess.SubprocessError, OSError): + pass + return "" + + def cleanup(self) -> None: + """Safely delete disposable workspace directory.""" + if self._temp_dir and os.path.exists(self._temp_dir): + try: + shutil.rmtree(self._temp_dir, onerror=_remove_readonly) + except OSError: + pass + self._temp_dir = None + self.workspace_path = "" + + +class TaskVerifier: + """Deterministic acceptance verifier executing tests or checks in the workspace.""" + + def __init__( + self, + test_cmd: Optional[List[str]] = None, + verifier_fn: Optional[Callable[[str], Any]] = None, + expected_files: Optional[Dict[str, str]] = None, + timeout: float = 30.0, + ) -> None: + self.test_cmd = test_cmd or [] + self.verifier_fn = verifier_fn + self.expected_files = expected_files or {} + self.timeout = timeout + + def verify( + self, + workspace_path: str, + env: Optional[Dict[str, str]] = None, + ) -> Tuple[str, float, float, str, str]: + """Run verification checks. + + Returns: + (outcome, hard, soft, rationale, inconclusive_reason) + """ + if not os.path.isdir(workspace_path): + return ( + OUTCOME_INCONCLUSIVE, 0.0, 0.0, + "Workspace directory does not exist", REASON_FIXTURE_FAILURE, + ) + + # 1. Custom verifier callable if provided + if self.verifier_fn is not None: + try: + res = self.verifier_fn(workspace_path) + if isinstance(res, tuple) and len(res) >= 2: + passed, rationale = bool(res[0]), str(res[1]) + else: + passed, rationale = bool(res), "Verifier returned boolean result" + if passed: + return (OUTCOME_PASS, 1.0, 1.0, rationale, "") + return (OUTCOME_FAIL, 0.0, 0.0, rationale, "") + except Exception as exc: + return ( + OUTCOME_INCONCLUSIVE, 0.0, 0.0, + f"Verifier function crashed: {exc}", REASON_INFRA, + ) + + # 2. Check required files and content matches if specified + if self.expected_files: + for rel_path, expected_text in self.expected_files.items(): + full_path = os.path.join(workspace_path, rel_path) + if not os.path.exists(full_path): + return ( + OUTCOME_FAIL, 0.0, 0.0, + f"Expected file missing: {rel_path}", "", + ) + if expected_text: + try: + with open(full_path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + if expected_text not in content: + return ( + OUTCOME_FAIL, 0.0, 0.0, + f"Expected content not found in {rel_path}", "", + ) + except OSError as exc: + return ( + OUTCOME_INCONCLUSIVE, 0.0, 0.0, + f"Failed to read file {rel_path}: {exc}", REASON_INFRA, + ) + + # 3. Execute test command if defined + if self.test_cmd: + try: + proc = subprocess.run( + self.test_cmd, + cwd=workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=self.timeout, + env=env, + creationflags=_NO_WINDOW, + ) + if proc.returncode == 0: + return ( + OUTCOME_PASS, 1.0, 1.0, + f"Test command succeeded (code 0):\n{proc.stdout[:300]}", "", + ) + # Test failed deterministically + fail_summary = (proc.stdout + "\n" + proc.stderr).strip()[-500:] + return ( + OUTCOME_FAIL, 0.0, 0.0, + f"Test command failed (code {proc.returncode}):\n{fail_summary}", "", + ) + except subprocess.TimeoutExpired: + return ( + OUTCOME_INCONCLUSIVE, 0.0, 0.0, + f"Test command timed out after {self.timeout}s", REASON_TIMEOUT, + ) + except FileNotFoundError as exc: + return ( + OUTCOME_INCONCLUSIVE, 0.0, 0.0, + f"Test command binary not found: {exc}", REASON_INFRA, + ) + except OSError as exc: + return ( + OUTCOME_INCONCLUSIVE, 0.0, 0.0, + f"Test command failed to execute: {exc}", REASON_INFRA, + ) + + # If no explicit check is configured, accept clean workspace + return (OUTCOME_PASS, 1.0, 1.0, "No failing verifier rules triggered", "") + + +class AgenticReplayBackend(Backend): + """Tiered agentic replay backend running tasks in disposable worktrees.""" + + name = "agentic" + + def __init__( + self, + config: Optional[AgenticReplayConfig] = None, + base_backend: Optional[Backend] = None, + verifier: Optional[TaskVerifier] = None, + ) -> None: + self.config = config or AgenticReplayConfig() + self.base_backend = base_backend or MockBackend() + self.verifier = verifier or TaskVerifier() + self.replays_run: int = 0 + self._tokens_spent: int = 0 + + def tokens_used(self) -> int: + return self._tokens_spent + self.base_backend.tokens_used() + + def attempt( + self, + task: TaskRecord, + skill: str, + memory: str, + sample_id: int = 0, + ) -> str: + """Run the task attempt via the base backend or agent CLI.""" + return self.base_backend.attempt(task, skill, memory, sample_id=sample_id) + + def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]: + """Default judge delegating to base backend.""" + return self.base_backend.judge(task, response) + + def reflect( + self, + failures: List[Tuple[TaskRecord, ReplayResult]], + successes: List[Tuple[TaskRecord, ReplayResult]], + skill: str, + memory: str, + *, + edit_budget: int, + evolve_skill: bool, + evolve_memory: bool, + ) -> List[EditRecord]: + """Filter out inconclusive runs before reflection so infra failures do not skew edits.""" + clean_failures = [ + (t, r) for t, r in failures + if getattr(r, "outcome", "") != OUTCOME_INCONCLUSIVE + ] + return self.base_backend.reflect( + clean_failures, + successes, + skill, + memory, + edit_budget=edit_budget, + evolve_skill=evolve_skill, + evolve_memory=evolve_memory, + ) + + def replay_agentic_task( + self, + task: TaskRecord, + skill: str, + memory: str, + sample_id: int = 0, + fixture_dir: str = "", + test_cmd: Optional[List[str]] = None, + expected_files: Optional[Dict[str, str]] = None, + agent_runner: Optional[Callable[[str, Dict[str, str]], List[str]]] = None, + ) -> ReplayResult: + """Run an end-to-end agentic replay within a disposable workspace.""" + # 1. Budget gate check + if self.replays_run >= self.config.max_replays_per_run: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_BUDGET_EXHAUSTED, + fail_reason="Max agentic replays budget reached for this run", + judge_rationale=f"budget limit ({self.config.max_replays_per_run}) reached", + ) + + t0 = time.time() + self.replays_run += 1 + tools_called: List[str] = [] + diff: str = "" + + # 2. Spin up disposable workspace + base_dir = fixture_dir + if not base_dir and task.context_excerpt and os.path.isdir(task.context_excerpt): + base_dir = task.context_excerpt + workspace = AgenticWorkspace( + base_dir=base_dir, + pinned_commit=self.config.pinned_commit, + isolate_env=self.config.isolate_env, + isolate_network=self.config.isolate_network, + ) + + try: + workspace_path = workspace.setup() + env = workspace.build_isolated_env() + + # 3. Execute agentic interaction + timed_out = False + infra_error = False + error_msg = "" + actions_count = 0 + + if agent_runner is not None: + try: + tools_called = agent_runner(workspace_path, env) or [] + actions_count = len(tools_called) + except subprocess.TimeoutExpired: + timed_out = True + except Exception as exc: + infra_error = True + error_msg = str(exc) + elif self.config.agent_cmd: + try: + cmd = [self.config.agent_cmd, task.intent] + proc = subprocess.run( + cmd, + cwd=workspace_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=self.config.timeout_sec, + env=env, + creationflags=_NO_WINDOW, + ) + actions_count = 1 if proc.returncode == 0 else 0 + except subprocess.TimeoutExpired: + timed_out = True + except Exception as exc: + infra_error = True + error_msg = str(exc) + else: + # Mock / simulated agentic execution from task properties + simulated_resp = self.attempt(task, skill, memory, sample_id=sample_id) + # Extract simulated tool calls or edit markers + if "TOOL_CALL:" in simulated_resp: + for line in simulated_resp.splitlines(): + if "TOOL_CALL:" in line: + tools_called.append(line.split("TOOL_CALL:", 1)[1].strip()) + actions_count = len(tools_called) if tools_called else (1 if simulated_resp else 0) + + # Check timeouts / infra issues + if timed_out: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_TIMEOUT, + fail_reason=f"Agent run timed out after {self.config.timeout_sec}s", + judge_rationale="turn/time budget exceeded", + latency_ms=round((time.time() - t0) * 1000.0, 1), + ) + if infra_error: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_INFRA, + fail_reason=f"Infrastructure error during agent run: {error_msg}", + judge_rationale=f"infra failure: {error_msg}", + latency_ms=round((time.time() - t0) * 1000.0, 1), + ) + + # 4. Check decidable action floor (issue #155 constraint) + if actions_count < self.config.min_decidable_actions: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_INSUFFICIENT_EVIDENCE, + fail_reason=( + f"Decidable actions ({actions_count}) below threshold " + f"({self.config.min_decidable_actions})" + ), + judge_rationale="insufficient decidable actions", + tools_called=tools_called, + latency_ms=round((time.time() - t0) * 1000.0, 1), + ) + + # 5. Extract unified diff + diff = workspace.get_diff() + + # 6. Task verification (primary acceptance gate) + active_verifier = self.verifier + if test_cmd or expected_files: + active_verifier = TaskVerifier( + test_cmd=test_cmd, + expected_files=expected_files, + timeout=min(self.config.timeout_sec, 30.0), + ) + + outcome, hard, soft, rationale, reason = active_verifier.verify(workspace_path, env) + + tokens = 500 # Baseline token estimate for agentic interaction + self._tokens_spent += tokens + latency_ms = (time.time() - t0) * 1000.0 + + return ReplayResult( + id=task.id, + hard=hard, + soft=soft, + outcome=outcome, + inconclusive_reason=reason, + diff=diff, + response=f"Agentic replay in {workspace_path} (diff: {len(diff)} chars)", + fail_reason="" if outcome == OUTCOME_PASS else rationale, + judge_rationale=rationale, + tools_called=tools_called, + tokens=tokens, + latency_ms=round(latency_ms, 1), + ) + finally: + workspace.cleanup() diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py index 605a57169..e84a613d8 100644 --- a/skillopt_sleep/backend.py +++ b/skillopt_sleep/backend.py @@ -2781,10 +2781,13 @@ def get_backend( project_dir or os.getcwd(), ".skillopt-sleep-handoff" ) return HandoffBackend(model=model, handoff_dir=hdir) + if n in {"agentic", "agentic_replay", "agentic-replay"}: + from skillopt_sleep.agentic_replay import AgenticReplayBackend + return AgenticReplayBackend() if n in {"mock", ""}: return MockBackend() raise ValueError( - f"Unknown backend: {name!r}. Supported backends are: 'azure', 'azure-responses', " + f"Unknown backend: {name!r}. Supported backends are: 'agentic', 'azure', 'azure-responses', " "'claude', 'codex', 'copilot', 'cursor', 'handoff', 'mock', 'opencode', 'pi'." ) diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index 64814979d..16f983818 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -68,7 +68,15 @@ "gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts) "gate_mixed_weight": 0.5, "gate_no_regression": False, # reject any candidate that lowers a val-task score - "replay_mode": "mock", # report label; fresh-worktree replay is not implemented + "replay_mode": "mock", # report label; "mock" | "text" | "agentic" (issue #155) + # ── agentic replay tier (opt-in; issue #155) ─────────────────────────── + "agentic_replay": False, # opt-in tiered agentic replay in isolated workspace + "agentic_max_turns": 10, # turn cap per agentic replay + "agentic_timeout_sec": 60.0, # wall-clock timeout in seconds + "agentic_max_replays": 5, # nightly budget cap on agentic replays + "agentic_isolate_network": True, # disable external network by default + "agentic_isolate_env": True, # scrub host credentials and user keys + "agentic_min_actions": 1, # floor below which outcome is inconclusive (insufficient_evidence) # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─ "dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task "dream_factor": 0, # >0 => add N synthetic variants of each task to the dream diff --git a/skillopt_sleep/replay.py b/skillopt_sleep/replay.py index 6eba0b861..97fb78103 100644 --- a/skillopt_sleep/replay.py +++ b/skillopt_sleep/replay.py @@ -32,6 +32,8 @@ def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str, """``sample_id`` distinguishes repeated dream rollouts of the same (task, skill, memory) in the attempt cache — without it all K rollouts collapse to one cached response and the contrastive signal is always 0.""" + if hasattr(backend, "replay_agentic_task"): + return backend.replay_agentic_task(task, skill, memory, sample_id=sample_id) import time tools = _required_tools(task) tools_called: List[str] = [] @@ -166,9 +168,20 @@ def replay_batch( return results -def aggregate_scores(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]: +def aggregate_scores( + pairs: List[Tuple[TaskRecord, ReplayResult]], + *, + exclude_inconclusive: bool = False, +) -> Tuple[float, float]: if not pairs: return 0.0, 0.0 + if exclude_inconclusive: + valid = [r for _t, r in pairs if getattr(r, "outcome", "") != "inconclusive"] + if not valid: + return 0.0, 0.0 + hard = sum(r.hard for r in valid) / len(valid) + soft = sum(r.soft for r in valid) / len(valid) + return hard, soft hard = sum(r.hard for _t, r in pairs) / len(pairs) soft = sum(r.soft for _t, r in pairs) / len(pairs) return hard, soft diff --git a/skillopt_sleep/types.py b/skillopt_sleep/types.py index d5c053d5a..7c4fe61e3 100644 --- a/skillopt_sleep/types.py +++ b/skillopt_sleep/types.py @@ -115,6 +115,16 @@ class ReplayResult: # evidence for audit/debugging; they may contain regexes or other grader # implementation details and must not be used as learning context. optimizer_feedback: str = "" + # Three-valued outcome: pass | fail | inconclusive (issue #155). + # Timeouts, budget exhaustion, or environment failures must not be counted + # as evidence that a candidate skill failed. + outcome: str = "" + inconclusive_reason: str = "" # infra | timeout | budget_exhausted | fixture_failure | insufficient_evidence + diff: str = "" # unified diff produced in agentic replay + + def __post_init__(self) -> None: + if not self.outcome: + self.outcome = "pass" if self.hard >= 1.0 else "fail" def to_dict(self) -> Dict[str, Any]: return asdict(self) diff --git a/tests/test_agentic_replay.py b/tests/test_agentic_replay.py new file mode 100644 index 000000000..617ebc9d7 --- /dev/null +++ b/tests/test_agentic_replay.py @@ -0,0 +1,286 @@ +"""Tests for AgenticReplayBackend and agentic replay slice (Issue #155).""" +import os +import subprocess +import sys +import tempfile +import unittest + +from skillopt_sleep.agentic_replay import ( + OUTCOME_FAIL, + OUTCOME_INCONCLUSIVE, + OUTCOME_PASS, + REASON_BUDGET_EXHAUSTED, + REASON_INFRA, + REASON_INSUFFICIENT_EVIDENCE, + REASON_TIMEOUT, + AgenticReplayBackend, + AgenticReplayConfig, + AgenticWorkspace, + TaskVerifier, +) +from skillopt_sleep.backend import get_backend +from skillopt_sleep.replay import aggregate_scores, replay_one +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + + +class TestReplayResultOutcomes(unittest.TestCase): + def test_default_pass_fail_inference(self): + passed = ReplayResult(id="t1", hard=1.0) + self.assertEqual(passed.outcome, OUTCOME_PASS) + + failed = ReplayResult(id="t2", hard=0.0) + self.assertEqual(failed.outcome, OUTCOME_FAIL) + + mixed = ReplayResult(id="t3", hard=0.5) + self.assertEqual(mixed.outcome, OUTCOME_FAIL) + + def test_explicit_inconclusive_outcome(self): + res = ReplayResult( + id="t1", + hard=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_TIMEOUT, + diff="--- a/file\n+++ b/file\n", + ) + self.assertEqual(res.outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(res.inconclusive_reason, REASON_TIMEOUT) + self.assertIn("--- a/file", res.diff) + + d = res.to_dict() + self.assertEqual(d["outcome"], OUTCOME_INCONCLUSIVE) + self.assertEqual(d["inconclusive_reason"], REASON_TIMEOUT) + self.assertIn("--- a/file", d["diff"]) + + +class TestAgenticWorkspace(unittest.TestCase): + def test_workspace_isolation_and_git_baseline(self): + with AgenticWorkspace(isolate_env=True, isolate_network=True) as ws: + self.assertTrue(os.path.isdir(ws.workspace_path)) + # Verify git repo was initialized with baseline commit + git_res = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ws.workspace_path, + capture_output=True, + text=True, + ) + self.assertEqual(git_res.returncode, 0) + self.assertTrue(len(git_res.stdout.strip()) >= 7) + + # Check environment scrubbing + os.environ["TEST_SECRET_KEY"] = "supersecret" + os.environ["AWS_ACCESS_KEY_ID"] = "AKIAEXAMPLE" + os.environ["GITHUB_TOKEN"] = "ghp_example" + env = ws.build_isolated_env() + + self.assertNotIn("TEST_SECRET_KEY", env) + self.assertNotIn("AWS_ACCESS_KEY_ID", env) + self.assertNotIn("GITHUB_TOKEN", env) + self.assertEqual(env.get("HTTP_PROXY"), "http://127.0.0.1:9") + self.assertTrue(os.path.isdir(env.get("HOME", ""))) + self.assertIn(".sandbox_home", env.get("HOME", "")) + + # Test git diff tracking + file_path = os.path.join(ws.workspace_path, "sample.txt") + with open(file_path, "w", encoding="utf-8") as f: + f.write("hello world\n") + + diff = ws.get_diff() + self.assertIn("sample.txt", diff) + self.assertIn("+hello world", diff) + + # Verify workspace is cleaned up on exit + self.assertFalse(os.path.exists(ws.workspace_path)) + + +class TestTaskVerifier(unittest.TestCase): + def test_verifier_fn(self): + verifier = TaskVerifier(verifier_fn=lambda path: (True, "all checks passed")) + with tempfile.TemporaryDirectory() as tmp: + outcome, hard, soft, rationale, reason = verifier.verify(tmp) + self.assertEqual(outcome, OUTCOME_PASS) + self.assertEqual(hard, 1.0) + self.assertEqual(soft, 1.0) + self.assertEqual(reason, "") + + failing_verifier = TaskVerifier(verifier_fn=lambda path: (False, "assertion failed")) + with tempfile.TemporaryDirectory() as tmp: + outcome, hard, soft, rationale, reason = failing_verifier.verify(tmp) + self.assertEqual(outcome, OUTCOME_FAIL) + self.assertEqual(hard, 0.0) + self.assertEqual(soft, 0.0) + + def test_expected_files(self): + with tempfile.TemporaryDirectory() as tmp: + verifier = TaskVerifier(expected_files={"output.json": '"status": "ok"'}) + outcome, hard, _, _, _ = verifier.verify(tmp) + self.assertEqual(outcome, OUTCOME_FAIL) + + with open(os.path.join(tmp, "output.json"), "w", encoding="utf-8") as f: + f.write('{"status": "ok", "value": 42}') + + outcome, hard, _, _, _ = verifier.verify(tmp) + self.assertEqual(outcome, OUTCOME_PASS) + self.assertEqual(hard, 1.0) + + def test_test_cmd_execution(self): + with tempfile.TemporaryDirectory() as tmp: + # Pass command + verifier = TaskVerifier(test_cmd=[sys.executable, "-c", "exit(0)"]) + outcome, hard, _, _, _ = verifier.verify(tmp) + self.assertEqual(outcome, OUTCOME_PASS) + self.assertEqual(hard, 1.0) + + # Deterministic fail command + failing_verifier = TaskVerifier(test_cmd=[sys.executable, "-c", "exit(1)"]) + outcome, hard, _, _, _ = failing_verifier.verify(tmp) + self.assertEqual(outcome, OUTCOME_FAIL) + self.assertEqual(hard, 0.0) + + # Timeout command + timeout_verifier = TaskVerifier( + test_cmd=[sys.executable, "-c", "import time; time.sleep(2)"], + timeout=0.2, + ) + outcome, hard, _, _, reason = timeout_verifier.verify(tmp) + self.assertEqual(outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(reason, REASON_TIMEOUT) + + +class TestAgenticReplayBackend(unittest.TestCase): + def test_successful_agentic_replay(self): + backend = AgenticReplayBackend() + task = TaskRecord(id="task_build", project="test", intent="fix build config") + + def agent_runner(workspace_path, env): + target_file = os.path.join(workspace_path, "build.txt") + with open(target_file, "w", encoding="utf-8") as f: + f.write("build=success\n") + return ["write_file", "run_test"] + + result = backend.replay_agentic_task( + task, + skill="use correct build flags", + memory="", + expected_files={"build.txt": "build=success"}, + agent_runner=agent_runner, + ) + + self.assertEqual(result.outcome, OUTCOME_PASS) + self.assertEqual(result.hard, 1.0) + self.assertEqual(result.soft, 1.0) + self.assertIn("build.txt", result.diff) + self.assertEqual(result.tools_called, ["write_file", "run_test"]) + + def test_failing_agentic_replay(self): + backend = AgenticReplayBackend() + task = TaskRecord(id="task_fail", project="test", intent="break test") + + def agent_runner(workspace_path, env): + return ["run_tests"] + + result = backend.replay_agentic_task( + task, + skill="", + memory="", + test_cmd=[sys.executable, "-c", "sys.exit(1)"], + agent_runner=agent_runner, + ) + + self.assertEqual(result.outcome, OUTCOME_FAIL) + self.assertEqual(result.hard, 0.0) + self.assertIn("failed", result.fail_reason.lower()) + + def test_insufficient_decidable_actions(self): + config = AgenticReplayConfig(min_decidable_actions=2) + backend = AgenticReplayBackend(config=config) + task = TaskRecord(id="task_thin", project="test", intent="simple query") + + def zero_action_runner(workspace_path, env): + return [] # 0 actions, below threshold of 2 + + result = backend.replay_agentic_task( + task, + skill="", + memory="", + agent_runner=zero_action_runner, + ) + + self.assertEqual(result.outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(result.inconclusive_reason, REASON_INSUFFICIENT_EVIDENCE) + + def test_budget_exhaustion(self): + config = AgenticReplayConfig(max_replays_per_run=1) + backend = AgenticReplayBackend(config=config) + task1 = TaskRecord(id="task_1", project="test", intent="first task") + task2 = TaskRecord(id="task_2", project="test", intent="second task") + + res1 = backend.replay_agentic_task(task1, skill="", memory="") + self.assertNotEqual(res1.inconclusive_reason, REASON_BUDGET_EXHAUSTED) + + # Second replay exceeds budget cap of 1 + res2 = backend.replay_agentic_task(task2, skill="", memory="") + self.assertEqual(res2.outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(res2.inconclusive_reason, REASON_BUDGET_EXHAUSTED) + + def test_tiered_reflection_excludes_inconclusive(self): + backend = AgenticReplayBackend() + task_real_fail = TaskRecord(id="tf", project="p", intent="real fail") + task_infra_fail = TaskRecord(id="ti", project="p", intent="infra fail") + + r_real_fail = ReplayResult(id="tf", hard=0.0, outcome=OUTCOME_FAIL) + r_infra_fail = ReplayResult( + id="ti", + hard=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_TIMEOUT, + ) + + failures = [(task_real_fail, r_real_fail), (task_infra_fail, r_infra_fail)] + # Reflect should only process real failures, discarding the infra failure + edits = backend.reflect( + failures, + [], + skill="baseline", + memory="", + edit_budget=2, + evolve_skill=True, + evolve_memory=False, + ) + self.assertIsInstance(edits, list) + + def test_aggregate_scores_exclude_inconclusive(self): + task1 = TaskRecord(id="t1", project="p", intent="i1") + task2 = TaskRecord(id="t2", project="p", intent="i2") + task3 = TaskRecord(id="t3", project="p", intent="i3") + + r1 = ReplayResult(id="t1", hard=1.0, soft=1.0, outcome=OUTCOME_PASS) + r2 = ReplayResult(id="t2", hard=0.0, soft=0.0, outcome=OUTCOME_FAIL) + r3 = ReplayResult( + id="t3", + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_TIMEOUT, + ) + + pairs = [(task1, r1), (task2, r2), (task3, r3)] + + # Standard aggregation: 1 / 3 = 0.3333 + hard_all, soft_all = aggregate_scores(pairs) + self.assertAlmostEqual(hard_all, 1.0 / 3.0) + + # Excluding inconclusive: 1 / 2 = 0.5 + hard_filtered, soft_filtered = aggregate_scores(pairs, exclude_inconclusive=True) + self.assertAlmostEqual(hard_filtered, 0.5) + + def test_get_backend_agentic(self): + be = get_backend("agentic") + self.assertIsInstance(be, AgenticReplayBackend) + self.assertEqual(be.name, "agentic") + + be2 = get_backend("agentic_replay") + self.assertIsInstance(be2, AgenticReplayBackend) + + +if __name__ == "__main__": + unittest.main() From c3afe2542fbca6478dee823033204a7e63010c4d Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Wed, 11 Nov 2026 19:48:17 +0530 Subject: [PATCH 2/2] fix(agentic-replay): narrow to safe workspace lifecycle, harden git/env isolation, and add pairwise gate exclusion --- skillopt_sleep/__init__.py | 9 +- skillopt_sleep/agentic_replay.py | 1535 ++++++++++++++++++++++-------- skillopt_sleep/backend.py | 50 +- skillopt_sleep/config.py | 12 +- skillopt_sleep/consolidate.py | 218 +++-- skillopt_sleep/cycle.py | 42 +- skillopt_sleep/dream.py | 2 + skillopt_sleep/replay.py | 36 + tests/test_agentic_replay.py | 917 ++++++++++++++---- 9 files changed, 2134 insertions(+), 687 deletions(-) diff --git a/skillopt_sleep/__init__.py b/skillopt_sleep/__init__.py index 05c197345..787a75455 100644 --- a/skillopt_sleep/__init__.py +++ b/skillopt_sleep/__init__.py @@ -16,7 +16,14 @@ """ from __future__ import annotations +from typing import Any + __all__ = ["__version__", "AgenticReplayBackend"] __version__ = "0.2.0" -from skillopt_sleep.agentic_replay import AgenticReplayBackend + +def __getattr__(name: str) -> Any: + if name == "AgenticReplayBackend": + from skillopt_sleep.agentic_replay import AgenticReplayBackend + return AgenticReplayBackend + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/skillopt_sleep/agentic_replay.py b/skillopt_sleep/agentic_replay.py index 68b97b989..055f5e65f 100644 --- a/skillopt_sleep/agentic_replay.py +++ b/skillopt_sleep/agentic_replay.py @@ -1,101 +1,311 @@ -"""SkillOpt-Sleep — AgenticReplayBackend (Issue #155 safe first slice). - -Replay mined tasks with an interactive agent in an isolated, throwaway workspace, -and gate primarily on deterministic task verification (builds, tests, and checks) -rather than purely single-turn text completion. - -Key properties: - 1. Safe, isolated workspace with environment scrubbing (no host credential leakage) - and network isolation by default. - 2. Pinned starting commit and disposable throwaway git worktree / scratch dir. - 3. Task verification as the primary gate (deterministic test exit code/assertions); - diff similarity remains secondary audit context. - 4. Three-valued outcome: pass | fail | inconclusive. - Infrastructure failures (timeouts, budget exhaustion, missing fixtures, or - insufficient decidable actions) emit `inconclusive` with typed reason codes - so they are never counted as evidence that a candidate skill failed. - 5. Strict budgeting: explicit turn, timeout, token, and replay caps. - 6. Staged diff and report for human review without automatic adoption. +"""Tiered Agentic Replay helper and disposable scratch workspace (Issue #155). + +This module implements the first workspace-lifecycle slice for agentic replay: +- A disposable scratch copy (NOT a linked git worktree and NOT an OS/container + sandbox) with an independent bare harness git directory outside the diffed tree. +- Strict allowlist environment scrubbing for child subprocesses, redirecting + ``HOME`` / ``USERPROFILE`` / XDG / CLI config directories to empty locations + outside the diffed working directory. + Note: there is NO OS, user, or network isolation boundary in this slice. A + child running under the same UID can still read files by absolute path + (such as ``~/.ssh`` or ``~/.claude``) or open network sockets directly. +- Deterministic task verification against workspace state/tests, with no + verifier configured treated as ``inconclusive`` (``insufficient_evidence``). +- Explicit opt-in execution (``enabled=False`` by default) for injectable + test/runner callables; live CLI binary execution is not enabled here. """ from __future__ import annotations +import inspect +import json import os import re import shutil +import signal import stat import subprocess +import sys +import tarfile import tempfile +import threading import time -from dataclasses import dataclass, field -from typing import Any, Callable, Dict, List, Optional, Tuple +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple -from skillopt_sleep.backend import Backend, MockBackend from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord -# ── Three-valued outcome constants ─────────────────────────────────────────── +# Outcomes OUTCOME_PASS = "pass" OUTCOME_FAIL = "fail" OUTCOME_INCONCLUSIVE = "inconclusive" -# Typed reason codes for inconclusive outcomes +# Typed inconclusive reasons REASON_TIMEOUT = "timeout" REASON_INFRA = "infra" REASON_FIXTURE_FAILURE = "fixture_failure" REASON_BUDGET_EXHAUSTED = "budget_exhausted" REASON_INSUFFICIENT_EVIDENCE = "insufficient_evidence" -REASON_NETWORK_UNAVAILABLE = "network_unavailable" -# Windows console window suppression -_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0) if os.name == "nt" else 0 +# Strict OS-only allowlist for child subprocesses (matches Backend isolation ethos) +_ENV_ALLOWLIST_COMMON = ( + "PATH", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TZ", + "TERM", + "COLORTERM", +) +_ENV_ALLOWLIST_WINDOWS = ( + "SYSTEMROOT", + "WINDIR", + "COMSPEC", + "PATHEXT", + "OS", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", + "SYSTEMDRIVE", +) + + +class FixtureSetupError(RuntimeError): + """Raised when fixture validation, git materialization, or baseline setup fails.""" + + +@dataclass +class AgenticReplayConfig: + """Configuration, opt-in gate, and budgets for agentic replay.""" + + enabled: bool = False # Strict boolean opt-in; disabled by default + fixture_root: str = "" # Explicit root required for directory fixtures + max_turns: int = 10 # Turn/action cap per agentic replay + max_tokens: int = 0 # Cumulative token cap across run (0 = unlimited) + timeout_sec: float = 60.0 # Wall-clock timeout in seconds + max_replays_per_run: int = 5 # Per-night/run cap on agentic replays + isolate_env: bool = True # Use allowlisted env + isolated empty config dirs + min_decidable_actions: int = 1 # Minimum actions required to score vs inconclusive + min_coverage: float = 0.5 # Minimum pairwise decidable val fraction for gate + pinned_commit: str = "" # Optional commit materialized via git archive + max_output_bytes: int = 64 * 1024 # Cap buffered stdout/stderr per subprocess + max_diff_bytes: int = 256 * 1024 # Cap buffered unified diff per replay + def __post_init__(self) -> None: + self.enabled = self.enabled is True + self.isolate_env = bool(self.isolate_env) -def _remove_readonly(func: Callable, path: str, _excinfo: Any) -> None: - """Clear the read-only bit and retry removal (handles Windows .git permissions).""" + +def agentic_config_from_sleep_config(cfg: Mapping[str, Any]) -> AgenticReplayConfig: + """Build an :class:`AgenticReplayConfig` from a SleepConfig/dict mapping.""" + return AgenticReplayConfig( + enabled=cfg.get("agentic_replay", False) is True, + fixture_root=str(cfg.get("agentic_fixture_root", "") or ""), + max_turns=int(cfg.get("agentic_max_turns", 10) or 0), + max_tokens=int(cfg.get("agentic_max_tokens", 0) or 0), + timeout_sec=float(cfg.get("agentic_timeout_sec", 60.0) or 60.0), + max_replays_per_run=int(cfg.get("agentic_max_replays", 5) or 0), + isolate_env=bool(cfg.get("agentic_isolate_env", True)), + min_decidable_actions=int(cfg.get("agentic_min_actions", 1) or 0), + min_coverage=float(cfg.get("agentic_min_coverage", 0.5) or 0.5), + ) + + +def _ensure_within_root(target: str, root: str) -> str: + """Resolve ``target`` and verify it stays strictly within ``root``.""" + if not root or not str(root).strip(): + raise FixtureSetupError("fixture_root must be configured when loading fixture directories") + real_root = os.path.realpath(root) + if not os.path.isdir(real_root): + raise FixtureSetupError(f"fixture_root is not an existing directory: {root!r}") + candidate = target if os.path.isabs(target) else os.path.join(real_root, target) + real_target = os.path.realpath(candidate) try: - os.chmod(path, stat.S_IWRITE) - func(path) + common = os.path.commonpath([real_root, real_target]) + except ValueError as exc: + raise FixtureSetupError(f"Fixture path {target!r} is outside fixture_root {root!r}") from exc + if common != real_root: + raise FixtureSetupError(f"Fixture path {target!r} escapes fixture_root {root!r}") + if not os.path.isdir(real_target): + raise FixtureSetupError(f"Fixture path is not a directory: {target!r}") + return real_target + + +def _read_stream_capped(stream: Any, cap_bytes: int, dest: List[Tuple[bytes, bool]]) -> None: + """Drain ``stream`` in chunks, retaining at most ``cap_bytes`` bytes in memory.""" + buf = bytearray() + truncated = False + limit = max(0, int(cap_bytes)) + try: + while True: + chunk = stream.read(65536) + if not chunk: + break + if len(buf) < limit: + take = limit - len(buf) + buf.extend(chunk[:take]) + if len(chunk) > take: + truncated = True + else: + truncated = True except OSError: pass + finally: + try: + stream.close() + except OSError: + pass + dest.append((bytes(buf), truncated)) -@dataclass -class AgenticReplayConfig: - """Budgets, sandbox boundaries, and execution settings for agentic replay.""" +def _kill_process_tree(proc: subprocess.Popen[bytes]) -> None: + """Terminate ``proc`` and any descendants started in its process group.""" + if os.name == "nt": + try: + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + capture_output=True, + timeout=5.0, + ) + except (subprocess.SubprocessError, OSError): + pass + try: + proc.kill() + except OSError: + pass + else: + try: + os.killpg(proc.pid, signal.SIGKILL) + except OSError: + try: + proc.kill() + except OSError: + pass - enabled: bool = False - agent_cmd: str = "" # Path or command to drive agent CLI - max_turns: int = 10 # Hard turn cap - timeout_sec: float = 60.0 # Wall-clock timeout in seconds - max_tokens: int = 20000 # Token spend cap - max_replays_per_run: int = 5 # Strict budget cap on agentic replays - sandbox_type: str = "workspace" # "workspace" (throwaway worktree) | "container" - isolate_network: bool = True # Disable external network by default - isolate_env: bool = True # Scrub host credentials and keys - min_decidable_actions: int = 1 # Floor below which outcome is insufficient_evidence - pinned_commit: str = "" # Reproducible starting commit SHA +def _run_bounded_subprocess( + cmd: Sequence[str], + *, + cwd: str, + env: Optional[Dict[str, str]] = None, + stdin_bytes: Optional[bytes] = None, + timeout_sec: float = 60.0, + max_output_bytes: int = 64 * 1024, +) -> Tuple[bool, int, bytes, bytes, bool, bool]: + """Run ``cmd`` in its own process group with bounded output capture. -class AgenticWorkspace: - """Disposable, isolated scratch workspace for running an agentic replay safely.""" + Returns ``(timed_out, returncode, stdout_bytes, stderr_bytes, stdout_trunc, stderr_trunc)``. + """ + if isinstance(cmd, (str, bytes)) or not cmd: + raise ValueError("cmd must be a non-empty sequence of string arguments") + cmd_list = [str(part) for part in cmd] - _SENSITIVE_ENV_PREFIXES = ( - "AWS_", "AZURE_", "OPENAI_", "ANTHROPIC_", "GITHUB_", "GH_", - "GITLAB_", "SLACK_", "STRIPE_", "SSH_", "API_KEY", "SECRET", + popen_kwargs: Dict[str, Any] = { + "cwd": cwd, + "env": env, + "stdin": subprocess.PIPE if stdin_bytes is not None else subprocess.DEVNULL, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + } + if os.name == "nt": + popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + popen_kwargs["start_new_session"] = True + + proc = subprocess.Popen(cmd_list, **popen_kwargs) + + out_holder: List[Tuple[bytes, bool]] = [] + err_holder: List[Tuple[bytes, bool]] = [] + out_thread = threading.Thread( + target=_read_stream_capped, + args=(proc.stdout, max_output_bytes, out_holder), + daemon=True, ) + err_thread = threading.Thread( + target=_read_stream_capped, + args=(proc.stderr, max_output_bytes, err_holder), + daemon=True, + ) + out_thread.start() + err_thread.start() + + in_thread: Optional[threading.Thread] = None + if stdin_bytes is not None and proc.stdin is not None: + def _feed_stdin() -> None: + try: + proc.stdin.write(stdin_bytes) # type: ignore[union-attr] + proc.stdin.flush() # type: ignore[union-attr] + except OSError: + pass + finally: + try: + proc.stdin.close() # type: ignore[union-attr] + except OSError: + pass + + in_thread = threading.Thread(target=_feed_stdin, daemon=True) + in_thread.start() + + timed_out = False + try: + proc.wait(timeout=max(0.05, float(timeout_sec))) + except subprocess.TimeoutExpired: + timed_out = True + _kill_process_tree(proc) + try: + proc.wait(timeout=2.0) + except subprocess.TimeoutExpired: + pass + + if in_thread is not None: + in_thread.join(timeout=1.0) + out_thread.join(timeout=1.5) + err_thread.join(timeout=1.5) + + out_bytes, out_trunc = out_holder[0] if out_holder else (b"", False) + err_bytes, err_trunc = err_holder[0] if err_holder else (b"", False) + rc = proc.returncode if proc.returncode is not None else -1 + return timed_out, rc, out_bytes, err_bytes, out_trunc, err_trunc + + +class AgenticWorkspace: + """Disposable scratch copy for running one agentic replay. + + Directory layout places ``home``, ``tmp``, XDG/CLI config directories, + prompt payload files, and ``harness.git`` alongside—NOT inside—``work``, + so agent tool writes to ``$HOME`` or git metadata cannot pollute ``get_diff()``. + """ def __init__( self, - base_dir: str = "", - pinned_commit: str = "", + base_dir: Optional[str] = None, + pinned_commit: Optional[str] = None, isolate_env: bool = True, - isolate_network: bool = True, + *, + fixture_root: str = "", + max_diff_bytes: int = 256 * 1024, + max_output_bytes: int = 64 * 1024, ) -> None: self.base_dir = base_dir - self.pinned_commit = pinned_commit - self.isolate_env = isolate_env - self.isolate_network = isolate_network - self._temp_dir: Optional[str] = None - self.workspace_path: str = "" + self.pinned_commit = pinned_commit or "" + self.isolate_env = bool(isolate_env) + self.fixture_root = fixture_root + self.max_diff_bytes = max(1024, int(max_diff_bytes)) + self.max_output_bytes = max(1024, int(max_output_bytes)) + + self._root_dir = tempfile.mkdtemp(prefix="skillopt_agentic_") + self.workspace_path = os.path.join(self._root_dir, "work") + self.home_dir = os.path.join(self._root_dir, "home") + self._tmp_dir = os.path.join(self._root_dir, "tmp") + self._xdg_config_home = os.path.join(self._root_dir, "xdg_config") + self._xdg_data_home = os.path.join(self._root_dir, "xdg_data") + self._xdg_cache_home = os.path.join(self._root_dir, "xdg_cache") + self._xdg_state_home = os.path.join(self._root_dir, "xdg_state") + self._claude_config_dir = os.path.join(self._root_dir, "claude_config") + self._codex_home = os.path.join(self._root_dir, "codex_home") + self._prompt_dir = os.path.join(self._root_dir, "prompt") + self._harness_git_dir = os.path.join(self._root_dir, "harness.git") + self._empty_hooks_dir = os.path.join(self._root_dir, "empty_hooks") + self._empty_gitconfig = os.path.join(self._root_dir, "empty_gitconfig") + self._is_setup = False def __enter__(self) -> "AgenticWorkspace": self.setup() @@ -104,295 +314,668 @@ def __enter__(self) -> "AgenticWorkspace": def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: self.cleanup() - def setup(self) -> str: - """Create and initialize disposable workspace directory.""" - self._temp_dir = tempfile.mkdtemp(prefix="skillopt_agentic_") - self.workspace_path = self._temp_dir - - # If a base directory was provided, copy its content into the workspace - if self.base_dir and os.path.isdir(self.base_dir): - shutil.copytree(self.base_dir, self.workspace_path, dirs_exist_ok=True) + def _ensure_layout_dirs(self) -> None: + for d in ( + self.workspace_path, + self.home_dir, + self._tmp_dir, + self._xdg_config_home, + self._xdg_data_home, + self._xdg_cache_home, + self._xdg_state_home, + self._claude_config_dir, + self._codex_home, + self._prompt_dir, + self._harness_git_dir, + self._empty_hooks_dir, + ): + os.makedirs(d, exist_ok=True) + with open(self._empty_gitconfig, "wb"): + pass - # Initialize git repository if not already present - git_dir = os.path.join(self.workspace_path, ".git") - if not os.path.exists(git_dir): - try: - subprocess.run( - ["git", "init"], - cwd=self.workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - creationflags=_NO_WINDOW, - ) - subprocess.run( - ["git", "config", "user.name", "SkillOpt Agentic Sandbox"], - cwd=self.workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - creationflags=_NO_WINDOW, - ) - subprocess.run( - ["git", "config", "user.email", "sandbox@skillopt.local"], - cwd=self.workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - creationflags=_NO_WINDOW, - ) - subprocess.run( - ["git", "add", "."], - cwd=self.workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - creationflags=_NO_WINDOW, - ) - subprocess.run( - ["git", "commit", "-m", "baseline", "--allow-empty"], - cwd=self.workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - creationflags=_NO_WINDOW, - ) - except (subprocess.SubprocessError, OSError): - pass - elif self.pinned_commit: - try: - subprocess.run( - ["git", "checkout", self.pinned_commit], - cwd=self.workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - creationflags=_NO_WINDOW, - ) - except (subprocess.SubprocessError, OSError) as exc: - raise RuntimeError( - f"failed to checkout pinned commit {self.pinned_commit}: {exc}" - ) from exc + def _allowlisted_env(self) -> Dict[str, str]: + """Minimal allowlisted child environment with isolated empty home/config dirs.""" + self._ensure_layout_dirs() + allowlist = list(_ENV_ALLOWLIST_COMMON) + if os.name == "nt": + allowlist.extend(_ENV_ALLOWLIST_WINDOWS) + env: Dict[str, str] = {} + for key in allowlist: + val = os.environ.get(key) + if val is not None: + env[key] = val - return self.workspace_path + env.update({ + "HOME": self.home_dir, + "USERPROFILE": self.home_dir, + "TMPDIR": self._tmp_dir, + "TMP": self._tmp_dir, + "TEMP": self._tmp_dir, + "XDG_CONFIG_HOME": self._xdg_config_home, + "XDG_DATA_HOME": self._xdg_data_home, + "XDG_CACHE_HOME": self._xdg_cache_home, + "XDG_STATE_HOME": self._xdg_state_home, + "CLAUDE_CONFIG_DIR": self._claude_config_dir, + "CODEX_HOME": self._codex_home, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": self._empty_gitconfig, + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "", + "SSH_ASKPASS": "", + "NO_COLOR": "1", + }) + if os.name == "nt": + drive, tail = os.path.splitdrive(self.home_dir) + if drive: + env["HOMEDRIVE"] = drive + env["HOMEPATH"] = tail or "\\" + return env def build_isolated_env(self) -> Dict[str, str]: - """Construct sanitized environment variables.""" - env = dict(os.environ) + """Return the child subprocess environment according to ``isolate_env``.""" + self._ensure_layout_dirs() if self.isolate_env: - # Strip sensitive credential keys - keys_to_remove = [ - k for k in env - if any(k.upper().startswith(prefix) for prefix in self._SENSITIVE_ENV_PREFIXES) - or "KEY" in k.upper() - or "TOKEN" in k.upper() - or "PASSWORD" in k.upper() + return self._allowlisted_env() + return dict(os.environ) + + def _safe_git_prefix(self, git_dir: str, work_tree: Optional[str] = None) -> List[str]: + cmd = [ + "git", + f"--git-dir={git_dir}", + ] + if work_tree is not None: + cmd.append(f"--work-tree={work_tree}") + cmd.extend([ + "-c", f"core.hooksPath={self._empty_hooks_dir}", + "-c", "core.fsmonitor=", + "-c", "diff.external=", + "-c", "commit.gpgsign=false", + "-c", "tag.gpgsign=false", + ]) + return cmd + + def _run_harness_git( + self, + args: Sequence[str], + *, + git_dir: Optional[str] = None, + work_tree: Optional[str] = None, + cwd: Optional[str] = None, + timeout_sec: float = 30.0, + max_output_bytes: Optional[int] = None, + ) -> Tuple[bytes, bool]: + """Run harness-side git strictly under allowlisted env without repo config hooks.""" + env = self._allowlisted_env() + cmd = ( + self._safe_git_prefix(git_dir, work_tree) + if git_dir is not None + else [ + "git", + "-c", f"core.hooksPath={self._empty_hooks_dir}", + "-c", "core.fsmonitor=", + "-c", "diff.external=", + "-c", "commit.gpgsign=false", + "-c", "tag.gpgsign=false", ] - for k in keys_to_remove: - env.pop(k, None) - - # Isolate home/user profile to prevent tampering with host configs - sandbox_home = os.path.join(self.workspace_path, ".sandbox_home") - os.makedirs(sandbox_home, exist_ok=True) - env["HOME"] = sandbox_home - env["USERPROFILE"] = sandbox_home - env["XDG_CONFIG_HOME"] = os.path.join(sandbox_home, ".config") - - if self.isolate_network: - # Direct proxy traffic to non-routable loopback sink - env["HTTP_PROXY"] = "http://127.0.0.1:9" - env["HTTPS_PROXY"] = "http://127.0.0.1:9" - env["ALL_PROXY"] = "http://127.0.0.1:9" - env["NO_PROXY"] = "" + ) + cmd.extend(args) + timed_out, rc, out_b, err_b, out_trunc, _ = _run_bounded_subprocess( + cmd, + cwd=cwd or self._root_dir, + env=env, + timeout_sec=timeout_sec, + max_output_bytes=max_output_bytes or self.max_output_bytes, + ) + if timed_out: + raise FixtureSetupError(f"Harness git command timed out: {' '.join(args[:3])}") + if rc != 0: + err_msg = err_b.decode("utf-8", errors="replace").strip() + raise FixtureSetupError( + f"Harness git command failed (exit {rc}): {err_msg or ' '.join(args[:3])}" + ) + return out_b, out_trunc - return env + def _check_no_dotgit_pointers_or_alternates( + self, + directory: str, + *, + allow_top_git_dir: bool, + ) -> None: + """Reject linked ``.git`` files/symlinks and ``objects/info/alternates``.""" + top_dotgit = os.path.join(directory, ".git") + if os.path.islink(top_dotgit): + raise FixtureSetupError(f"Symlinked .git is not allowed in {directory!r}") + if os.path.isfile(top_dotgit): + raise FixtureSetupError( + f"Linked .git file is not allowed without archive materialization: {directory!r}" + ) + if os.path.isdir(top_dotgit): + if not allow_top_git_dir: + raise FixtureSetupError(f"Unexpected .git directory in {directory!r}") + alt = os.path.join(top_dotgit, "objects", "info", "alternates") + if os.path.exists(alt) or os.path.islink(alt): + raise FixtureSetupError(f"Git alternates file is not allowed: {alt!r}") - def get_diff(self) -> str: - """Extract unified diff against HEAD, including untracked files.""" - if not self.workspace_path or not os.path.exists(self.workspace_path): - return "" + for dirpath, dirnames, filenames in os.walk(directory, followlinks=False): + # Top-level .git dir was checked above; skip descending into it for nested checks + if dirpath == directory and ".git" in dirnames: + dirnames.remove(".git") + for name in filenames: + if name == ".git": + raise FixtureSetupError( + f"Nested .git file/submodule pointer is not allowed: {os.path.join(dirpath, name)!r}" + ) + for name in list(dirnames): + full = os.path.join(dirpath, name) + if name == ".git": + if os.path.islink(full) or not allow_top_git_dir: + raise FixtureSetupError(f"Nested .git entry is not allowed: {full!r}") + if os.path.islink(full): + # Check symlink does not point outside directory + real_dir = os.path.realpath(directory) + real_target = os.path.realpath(full) + try: + common = os.path.commonpath([real_dir, real_target]) + except ValueError as exc: + raise FixtureSetupError(f"Symlink escapes directory: {full!r}") from exc + if common != real_dir: + raise FixtureSetupError(f"Symlink escapes directory: {full!r}") + + def _check_source_repo_alternates(self, source_dir: str) -> None: + """Verify git object store for ``source_dir`` has no ``objects/info/alternates``.""" + for flag in ("--git-dir", "--git-common-dir"): + out_b, _ = self._run_harness_git(["-C", source_dir, "rev-parse", flag]) + raw_dir = out_b.decode("utf-8", errors="replace").strip() + if not raw_dir: + continue + resolved = raw_dir if os.path.isabs(raw_dir) else os.path.join(source_dir, raw_dir) + alt = os.path.join(os.path.realpath(resolved), "objects", "info", "alternates") + if os.path.exists(alt) or os.path.islink(alt): + raise FixtureSetupError(f"Git alternates file is not allowed in fixture: {alt!r}") + + def _materialize_via_git_archive(self, source_dir: str, pinned_commit: str) -> None: + """Materialize ``pinned_commit`` from ``source_dir`` without copying any ``.git``.""" + if not re.fullmatch(r"[A-Za-z0-9._/@^{}~-]+", pinned_commit) or pinned_commit.startswith("-"): + raise FixtureSetupError(f"Invalid pinned_commit reference: {pinned_commit!r}") + self._check_source_repo_alternates(source_dir) + + sha_bytes, _ = self._run_harness_git([ + "-C", source_dir, "rev-parse", "--verify", "--end-of-options", f"{pinned_commit}^{{commit}}", + ]) + resolved_sha = sha_bytes.decode("utf-8", errors="replace").strip() + if not re.fullmatch(r"[0-9a-fA-F]{7,64}", resolved_sha): + raise FixtureSetupError(f"Unexpected commit resolution for {pinned_commit!r}") + + archive_path = os.path.join(self._root_dir, "fixture_archive.tar") + env = self._allowlisted_env() + cmd = [ + "git", + "-C", source_dir, + "-c", f"core.hooksPath={self._empty_hooks_dir}", + "-c", "core.fsmonitor=", + "-c", "commit.gpgsign=false", + "archive", + "--format=tar", + resolved_sha, + ] try: - # Mark untracked files with intent-to-add so git diff includes newly added files - subprocess.run( - ["git", "add", "-N", "."], + with open(archive_path, "wb") as out_f: + popen_kw: Dict[str, Any] = { + "cwd": self._root_dir, + "env": env, + "stdin": subprocess.DEVNULL, + "stdout": out_f, + "stderr": subprocess.PIPE, + } + if os.name == "nt": + popen_kw["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + popen_kw["start_new_session"] = True + proc = subprocess.Popen(cmd, **popen_kw) + err_holder: List[Tuple[bytes, bool]] = [] + err_t = threading.Thread( + target=_read_stream_capped, + args=(proc.stderr, self.max_output_bytes, err_holder), + daemon=True, + ) + err_t.start() + try: + proc.wait(timeout=30.0) + except subprocess.TimeoutExpired as exc: + _kill_process_tree(proc) + err_t.join(timeout=1.0) + raise FixtureSetupError("git archive timed out") from exc + err_t.join(timeout=1.0) + if proc.returncode != 0: + err_b = err_holder[0][0] if err_holder else b"" + raise FixtureSetupError( + f"git archive failed for {pinned_commit!r}: " + f"{err_b.decode('utf-8', errors='replace').strip()}" + ) + + real_work = os.path.realpath(self.workspace_path) + with tarfile.open(archive_path, mode="r:") as tf: + safe_members: List[tarfile.TarInfo] = [] + for member in tf.getmembers(): + if member.name in ("", ".") or member.name.startswith("./"): + rel = member.name[2:] if member.name.startswith("./") else "" + else: + rel = member.name + norm_parts = [p for p in rel.replace("\\", "/").split("/") if p] + if not norm_parts: + if member.isdir(): + continue + raise FixtureSetupError("Unsafe empty archive member") + if os.path.isabs(rel) or os.path.splitdrive(rel)[0] or ".." in norm_parts: + raise FixtureSetupError(f"Archive path escape rejected: {member.name!r}") + if ".git" in norm_parts: + raise FixtureSetupError(f"Archive member inside .git rejected: {member.name!r}") + if member.isdev() or member.isfifo() or member.ischr() or member.isblk(): + raise FixtureSetupError(f"Special device member rejected: {member.name!r}") + dest_path = os.path.realpath(os.path.join(real_work, *norm_parts)) + if os.path.commonpath([real_work, dest_path]) != real_work: + raise FixtureSetupError(f"Archive target escapes workspace: {member.name!r}") + if member.issym() or member.islnk(): + link_base = real_work if member.islnk() else os.path.dirname(dest_path) + link_target = os.path.realpath(os.path.join(link_base, member.linkname)) + if os.path.commonpath([real_work, link_target]) != real_work: + raise FixtureSetupError( + f"Archive link target escapes workspace: {member.linkname!r}" + ) + safe_members.append(member) + tf.extractall(path=self.workspace_path, members=safe_members) + except (tarfile.TarError, OSError) as exc: + raise FixtureSetupError(f"Failed extracting archived fixture: {exc}") from exc + finally: + if os.path.exists(archive_path): + try: + os.remove(archive_path) + except OSError: + pass + + def _copy_plain_fixture(self, source_dir: str) -> None: + """Copy plain fixture files into ``workspace_path`` without copying ``.git``.""" + self._check_no_dotgit_pointers_or_alternates(source_dir, allow_top_git_dir=True) + real_source = os.path.realpath(source_dir) + for dirpath, dirnames, filenames in os.walk(real_source, followlinks=False): + dirnames[:] = [d for d in dirnames if d != ".git"] + rel_dir = os.path.relpath(dirpath, real_source) + dst_dir = ( + self.workspace_path + if rel_dir == "." + else os.path.join(self.workspace_path, rel_dir) + ) + os.makedirs(dst_dir, exist_ok=True) + for fname in filenames: + if fname == ".git": + continue + src_file = os.path.join(dirpath, fname) + if os.path.islink(src_file): + real_target = os.path.realpath(src_file) + try: + common = os.path.commonpath([real_source, real_target]) + except ValueError as exc: + raise FixtureSetupError(f"Fixture file symlink escapes root: {src_file!r}") from exc + if common != real_source: + raise FixtureSetupError(f"Fixture file symlink escapes root: {src_file!r}") + continue + shutil.copy2(src_file, os.path.join(dst_dir, fname)) + + def setup(self) -> None: + """Materialize workspace copy and initialize separate harness + local git baselines.""" + if self._is_setup: + return + self._ensure_layout_dirs() + + try: + self._run_harness_git(["init", "--bare", self._harness_git_dir]) + + if self.base_dir: + source_dir = ( + _ensure_within_root(self.base_dir, self.fixture_root) + if self.fixture_root + else os.path.realpath(self.base_dir) + ) + if not os.path.isdir(source_dir): + raise FixtureSetupError(f"Fixture directory does not exist: {self.base_dir!r}") + + if self.pinned_commit: + self._materialize_via_git_archive(source_dir, self.pinned_commit) + else: + self._copy_plain_fixture(source_dir) + elif self.pinned_commit: + raise FixtureSetupError("pinned_commit requires a base_dir git fixture") + + # Seed harness bare git dir with baseline commit (never touches workspace .git) + self._run_harness_git( + ["add", "-A"], + git_dir=self._harness_git_dir, + work_tree=self.workspace_path, cwd=self.workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - creationflags=_NO_WINDOW, ) - res = subprocess.run( - ["git", "diff", "HEAD"], + self._run_harness_git( + [ + "-c", "user.name=SkillOpt Agentic Scratch", + "-c", "user.email=agentic@skillopt.local", + "commit", "--allow-empty", "-m", "baseline", + ], + git_dir=self._harness_git_dir, + work_tree=self.workspace_path, cwd=self.workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - creationflags=_NO_WINDOW, ) - if res.returncode == 0: - return res.stdout - except (subprocess.SubprocessError, OSError): - pass - return "" + + # Give the scratch copy its own self-contained .git so agent commands like + # `git rev-parse HEAD` work without ever seeing the host repository. + self._run_harness_git(["-C", self.workspace_path, "init"]) + self._run_harness_git(["-C", self.workspace_path, "add", "-A"]) + self._run_harness_git([ + "-C", self.workspace_path, + "-c", "user.name=SkillOpt Agentic Scratch", + "-c", "user.email=agentic@skillopt.local", + "commit", "--allow-empty", "-m", "baseline", + ]) + self._is_setup = True + except FixtureSetupError: + raise + except (subprocess.SubprocessError, OSError) as exc: + raise FixtureSetupError(f"Workspace setup failed: {exc}") from exc + + def build_prompt_text(self, task: TaskRecord, skill: str = "", memory: str = "") -> str: + """Render a structured text prompt for the task, skill, and memory.""" + parts = [f"Task: {task.intent}"] + if task.context_excerpt: + parts.append(f"Context:\n{task.context_excerpt}") + if skill: + parts.append(f"Skill:\n{skill}") + if memory: + parts.append(f"Memory:\n{memory}") + return "\n\n".join(parts) + + def write_prompt_payload(self, task: TaskRecord, skill: str = "", memory: str = "") -> str: + """Write prompt JSON outside ``workspace_path`` so it stays out of ``get_diff()``.""" + self._ensure_layout_dirs() + prompt_file = os.path.join(self._prompt_dir, "task_payload.json") + payload = { + "task_id": task.id, + "intent": task.intent, + "context_excerpt": task.context_excerpt, + "skill": skill, + "memory": memory, + "prompt": self.build_prompt_text(task, skill=skill, memory=memory), + } + with open(prompt_file, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + return prompt_file + + def run_subprocess( + self, + cmd: Sequence[str], + *, + stdin_text: Optional[str] = None, + timeout_sec: float = 60.0, + env: Optional[Dict[str, str]] = None, + max_output_bytes: Optional[int] = None, + ) -> Tuple[bool, int, str, str]: + """Execute a list-argv command inside ``workspace_path`` with bounded pipes. + + Returns ``(timed_out, returncode, stdout_text, stderr_text)``. + """ + if not self._is_setup: + self.setup() + effective_env = env if env is not None else self.build_isolated_env() + cap = self.max_output_bytes if max_output_bytes is None else max_output_bytes + stdin_b = stdin_text.encode("utf-8", errors="replace") if stdin_text is not None else None + timed_out, rc, out_b, err_b, out_trunc, err_trunc = _run_bounded_subprocess( + cmd, + cwd=self.workspace_path, + env=effective_env, + stdin_bytes=stdin_b, + timeout_sec=timeout_sec, + max_output_bytes=cap, + ) + out_txt = out_b.decode("utf-8", errors="replace") + err_txt = err_b.decode("utf-8", errors="replace") + if out_trunc: + out_txt += f"\n... [stdout truncated at {cap} bytes]" + if err_trunc: + err_txt += f"\n... [stderr truncated at {cap} bytes]" + return timed_out, rc, out_txt, err_txt + + def get_diff(self) -> str: + """Compute unified diff via harness ``harness.git`` without reading ``work/.git/config``.""" + if not self._is_setup or not os.path.isdir(self.workspace_path): + return "" + self._check_no_dotgit_pointers_or_alternates(self.workspace_path, allow_top_git_dir=True) + self._run_harness_git( + ["add", "-N", "."], + git_dir=self._harness_git_dir, + work_tree=self.workspace_path, + cwd=self.workspace_path, + ) + diff_b, truncated = self._run_harness_git( + ["diff", "--no-ext-diff", "--no-textconv", "HEAD"], + git_dir=self._harness_git_dir, + work_tree=self.workspace_path, + cwd=self.workspace_path, + max_output_bytes=self.max_diff_bytes, + ) + text = diff_b.decode("utf-8", errors="replace") + if truncated: + text += f"\n... [diff truncated at {self.max_diff_bytes} bytes]\n" + return text def cleanup(self) -> None: - """Safely delete disposable workspace directory.""" - if self._temp_dir and os.path.exists(self._temp_dir): + """Remove temporary scratch directories (clearing read-only git object attrs).""" + if not os.path.exists(self._root_dir): + self._is_setup = False + return + + def _handle_readonly(func: Any, path: str, _exc: Any) -> None: try: - shutil.rmtree(self._temp_dir, onerror=_remove_readonly) + os.chmod(path, stat.S_IWRITE | stat.S_IREAD) + func(path) except OSError: pass - self._temp_dir = None - self.workspace_path = "" + + if sys.version_info >= (3, 12): + shutil.rmtree(self._root_dir, onexc=_handle_readonly) + else: + shutil.rmtree(self._root_dir, onerror=_handle_readonly) + # Ensure clean removal even if a transient handle delayed deletion + if os.path.exists(self._root_dir): + shutil.rmtree(self._root_dir, ignore_errors=True) + self._is_setup = False class TaskVerifier: - """Deterministic acceptance verifier executing tests or checks in the workspace.""" + """Deterministic task verifier acting as the primary acceptance gate.""" def __init__( self, - test_cmd: Optional[List[str]] = None, - verifier_fn: Optional[Callable[[str], Any]] = None, - expected_files: Optional[Dict[str, str]] = None, - timeout: float = 30.0, + test_cmd: Optional[Sequence[str]] = None, + expected_files: Optional[Dict[str, Optional[str]]] = None, + verifier_fn: Optional[Callable[[str], Tuple[bool, str]]] = None, + timeout_sec: float = 30.0, + max_output_bytes: int = 64 * 1024, ) -> None: - self.test_cmd = test_cmd or [] + if test_cmd is not None and isinstance(test_cmd, (str, bytes)): + raise ValueError("test_cmd must be a list/sequence of string arguments") + self.test_cmd = [str(part) for part in test_cmd] if test_cmd else None + self.expected_files = dict(expected_files) if expected_files else {} self.verifier_fn = verifier_fn - self.expected_files = expected_files or {} - self.timeout = timeout + self.timeout_sec = float(timeout_sec) + self.max_output_bytes = max(1024, int(max_output_bytes)) def verify( self, workspace_path: str, + *, env: Optional[Dict[str, str]] = None, ) -> Tuple[str, float, float, str, str]: - """Run verification checks. + """Run checks against ``workspace_path``. - Returns: - (outcome, hard, soft, rationale, inconclusive_reason) + Returns ``(outcome, hard, soft, rationale, inconclusive_reason)``. """ - if not os.path.isdir(workspace_path): + if not self.verifier_fn and not self.expected_files and not self.test_cmd: return ( - OUTCOME_INCONCLUSIVE, 0.0, 0.0, - "Workspace directory does not exist", REASON_FIXTURE_FAILURE, + OUTCOME_INCONCLUSIVE, + 0.0, + 0.0, + "No task verifier configured", + REASON_INSUFFICIENT_EVIDENCE, ) - # 1. Custom verifier callable if provided if self.verifier_fn is not None: try: - res = self.verifier_fn(workspace_path) - if isinstance(res, tuple) and len(res) >= 2: - passed, rationale = bool(res[0]), str(res[1]) - else: - passed, rationale = bool(res), "Verifier returned boolean result" + passed, msg = self.verifier_fn(workspace_path) if passed: - return (OUTCOME_PASS, 1.0, 1.0, rationale, "") - return (OUTCOME_FAIL, 0.0, 0.0, rationale, "") + return (OUTCOME_PASS, 1.0, 1.0, msg or "Verifier passed", "") + return (OUTCOME_FAIL, 0.0, 0.0, msg or "Verifier failed", "") except Exception as exc: return ( - OUTCOME_INCONCLUSIVE, 0.0, 0.0, - f"Verifier function crashed: {exc}", REASON_INFRA, + OUTCOME_INCONCLUSIVE, + 0.0, + 0.0, + f"Verifier error: {exc}", + REASON_INFRA, ) - # 2. Check required files and content matches if specified - if self.expected_files: - for rel_path, expected_text in self.expected_files.items(): - full_path = os.path.join(workspace_path, rel_path) - if not os.path.exists(full_path): + real_ws = os.path.realpath(workspace_path) + for rel_path, expected_substr in self.expected_files.items(): + full_path = os.path.realpath(os.path.join(real_ws, rel_path)) + try: + common = os.path.commonpath([real_ws, full_path]) + except ValueError: + common = "" + if common != real_ws: + return ( + OUTCOME_INCONCLUSIVE, + 0.0, + 0.0, + f"Expected file path escapes workspace: {rel_path}", + REASON_FIXTURE_FAILURE, + ) + if not os.path.exists(full_path): + return (OUTCOME_FAIL, 0.0, 0.0, f"Expected file missing: {rel_path}", "") + if expected_substr is not None: + try: + with open(full_path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + except OSError as exc: + return (OUTCOME_FAIL, 0.0, 0.0, f"Cannot read {rel_path}: {exc}", "") + if expected_substr not in content: return ( - OUTCOME_FAIL, 0.0, 0.0, - f"Expected file missing: {rel_path}", "", + OUTCOME_FAIL, + 0.0, + 0.0, + f"Expected content missing in {rel_path}", + "", ) - if expected_text: - try: - with open(full_path, "r", encoding="utf-8", errors="replace") as f: - content = f.read() - if expected_text not in content: - return ( - OUTCOME_FAIL, 0.0, 0.0, - f"Expected content not found in {rel_path}", "", - ) - except OSError as exc: - return ( - OUTCOME_INCONCLUSIVE, 0.0, 0.0, - f"Failed to read file {rel_path}: {exc}", REASON_INFRA, - ) - # 3. Execute test command if defined if self.test_cmd: try: - proc = subprocess.run( + timed_out, rc, out_b, err_b, out_trunc, err_trunc = _run_bounded_subprocess( self.test_cmd, cwd=workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=self.timeout, env=env, - creationflags=_NO_WINDOW, + timeout_sec=self.timeout_sec, + max_output_bytes=self.max_output_bytes, ) - if proc.returncode == 0: + if timed_out: return ( - OUTCOME_PASS, 1.0, 1.0, - f"Test command succeeded (code 0):\n{proc.stdout[:300]}", "", + OUTCOME_INCONCLUSIVE, + 0.0, + 0.0, + f"Verifier command timed out after {self.timeout_sec}s", + REASON_TIMEOUT, ) - # Test failed deterministically - fail_summary = (proc.stdout + "\n" + proc.stderr).strip()[-500:] - return ( - OUTCOME_FAIL, 0.0, 0.0, - f"Test command failed (code {proc.returncode}):\n{fail_summary}", "", - ) - except subprocess.TimeoutExpired: + if rc == 0: + return (OUTCOME_PASS, 1.0, 1.0, "Verifier test command passed", "") + out_s = (out_b + err_b).decode("utf-8", errors="replace").strip() + suffix = " [truncated]" if (out_trunc or err_trunc) else "" return ( - OUTCOME_INCONCLUSIVE, 0.0, 0.0, - f"Test command timed out after {self.timeout}s", REASON_TIMEOUT, + OUTCOME_FAIL, + 0.0, + 0.0, + f"Verifier exited with {rc}: {out_s[:300]}{suffix}", + "", ) - except FileNotFoundError as exc: + except (subprocess.SubprocessError, OSError, ValueError) as exc: return ( - OUTCOME_INCONCLUSIVE, 0.0, 0.0, - f"Test command binary not found: {exc}", REASON_INFRA, - ) - except OSError as exc: - return ( - OUTCOME_INCONCLUSIVE, 0.0, 0.0, - f"Test command failed to execute: {exc}", REASON_INFRA, + OUTCOME_INCONCLUSIVE, + 0.0, + 0.0, + f"Verifier infrastructure error: {exc}", + REASON_INFRA, ) - # If no explicit check is configured, accept clean workspace - return (OUTCOME_PASS, 1.0, 1.0, "No failing verifier rules triggered", "") + return (OUTCOME_PASS, 1.0, 1.0, "All configured verification checks passed", "") + + +def _invoke_runner( + runner: Callable[..., Any], + workspace_path: str, + env: Dict[str, str], + context: Dict[str, Any], +) -> Any: + """Call ``runner`` with ``(workspace_path, env, context)`` or ``(workspace_path, env)``.""" + try: + sig = inspect.signature(runner) + except (ValueError, TypeError): + return runner(workspace_path, env, context) + params = list(sig.parameters.values()) + accepts_varargs = any(p.kind == inspect.Parameter.VAR_POSITIONAL for p in params) + positional = [ + p for p in params + if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + if accepts_varargs or len(positional) >= 3: + return runner(workspace_path, env, context) + if any(p.kind == inspect.Parameter.KEYWORD_ONLY and p.name == "context" for p in params): + return runner(workspace_path, env, context=context) + return runner(workspace_path, env) -class AgenticReplayBackend(Backend): - """Tiered agentic replay backend running tasks in disposable worktrees.""" +class AgenticReplayBackend: + """Tiered opt-in agentic replay backend running in disposable scratch copies.""" name = "agentic" def __init__( self, config: Optional[AgenticReplayConfig] = None, - base_backend: Optional[Backend] = None, - verifier: Optional[TaskVerifier] = None, + *, + base_backend: Optional[Any] = None, + agent_runner: Optional[Callable[..., Any]] = None, + default_verifier: Optional[TaskVerifier] = None, ) -> None: self.config = config or AgenticReplayConfig() - self.base_backend = base_backend or MockBackend() - self.verifier = verifier or TaskVerifier() - self.replays_run: int = 0 - self._tokens_spent: int = 0 + self.base_backend = base_backend + self.agent_runner = agent_runner + self.default_verifier = default_verifier + self._lock = threading.Lock() + self.replays_run = 0 + self.tokens_spent = 0 + self.preferences = "" + + def reset_run_budget(self) -> None: + """Reset per-night/run replay and token counters.""" + with self._lock: + self.replays_run = 0 + self.tokens_spent = 0 def tokens_used(self) -> int: - return self._tokens_spent + self.base_backend.tokens_used() + with self._lock: + local = self.tokens_spent + base = self.base_backend.tokens_used() if self.base_backend is not None else 0 + return local + base def attempt( - self, - task: TaskRecord, - skill: str, - memory: str, - sample_id: int = 0, + self, task: TaskRecord, skill: str, memory: str, sample_id: int = 0 ) -> str: - """Run the task attempt via the base backend or agent CLI.""" - return self.base_backend.attempt(task, skill, memory, sample_id=sample_id) - - def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]: - """Default judge delegating to base backend.""" - return self.base_backend.judge(task, response) + if self.base_backend is not None: + return self.base_backend.attempt(task, skill, memory, sample_id=sample_id) + return "" def reflect( self, @@ -401,185 +984,349 @@ def reflect( skill: str, memory: str, *, - edit_budget: int, - evolve_skill: bool, - evolve_memory: bool, + edit_budget: int = 4, + evolve_skill: bool = True, + evolve_memory: bool = True, + **kwargs: Any, ) -> List[EditRecord]: - """Filter out inconclusive runs before reflection so infra failures do not skew edits.""" - clean_failures = [ - (t, r) for t, r in failures + """Delegate reflection to ``base_backend``, excluding inconclusive runs.""" + filtered_failures = [ + (t, r) for (t, r) in failures if getattr(r, "outcome", "") != OUTCOME_INCONCLUSIVE ] - return self.base_backend.reflect( - clean_failures, - successes, - skill, - memory, - edit_budget=edit_budget, - evolve_skill=evolve_skill, - evolve_memory=evolve_memory, - ) + filtered_successes = [ + (t, r) for (t, r) in successes + if getattr(r, "outcome", "") != OUTCOME_INCONCLUSIVE + ] + if self.base_backend is not None: + return self.base_backend.reflect( + filtered_failures, + filtered_successes, + skill, + memory, + edit_budget=edit_budget, + evolve_skill=evolve_skill, + evolve_memory=evolve_memory, + **kwargs, + ) + return [] + + def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]: + if self.base_backend is not None: + return self.base_backend.judge(task, response) + return (0.0, 0.0, "No base backend judge available") + + def _reserve_budget(self) -> Optional[str]: + """Thread-safely reserve one replay slot, returning fail reason if exhausted.""" + with self._lock: + if ( + self.config.max_replays_per_run > 0 + and self.replays_run >= self.config.max_replays_per_run + ): + return "Max agentic replays budget reached for this run" + if ( + self.config.max_tokens > 0 + and self.tokens_spent >= self.config.max_tokens + ): + return "Max agentic token budget reached for this run" + self.replays_run += 1 + return None + + def _resolve_fixture_dir( + self, task: TaskRecord, fixture_dir: Optional[str] + ) -> Optional[str]: + """Resolve an optional fixture directory exclusively from ``fixture_root``.""" + requested = fixture_dir + if not requested and isinstance(task.judge, Mapping): + raw = task.judge.get("fixture_dir") + if isinstance(raw, str) and raw.strip(): + requested = raw.strip() + if requested: + return _ensure_within_root(requested, self.config.fixture_root) + if self.config.fixture_root: + return _ensure_within_root(self.config.fixture_root, self.config.fixture_root) + return None def replay_agentic_task( self, task: TaskRecord, - skill: str, - memory: str, + skill: str = "", + memory: str = "", + *, + verifier: Optional[TaskVerifier] = None, + fixture_dir: Optional[str] = None, sample_id: int = 0, - fixture_dir: str = "", - test_cmd: Optional[List[str]] = None, - expected_files: Optional[Dict[str, str]] = None, - agent_runner: Optional[Callable[[str, Dict[str, str]], List[str]]] = None, ) -> ReplayResult: - """Run an end-to-end agentic replay within a disposable workspace.""" - # 1. Budget gate check - if self.replays_run >= self.config.max_replays_per_run: + """Run one opt-in agentic replay inside a disposable scratch copy.""" + del sample_id + t0 = time.monotonic() + + if self.config.enabled is not True: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_INFRA, + fail_reason="Agentic replay is disabled (set agentic_replay=True to opt in)", + task_type="agentic", + ) + + if self.agent_runner is None: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_INFRA, + fail_reason="No agent_runner configured (live CLI execution is disabled in this slice)", + task_type="agentic", + ) + + budget_err = self._reserve_budget() + if budget_err is not None: return ReplayResult( id=task.id, hard=0.0, soft=0.0, outcome=OUTCOME_INCONCLUSIVE, inconclusive_reason=REASON_BUDGET_EXHAUSTED, - fail_reason="Max agentic replays budget reached for this run", - judge_rationale=f"budget limit ({self.config.max_replays_per_run}) reached", + fail_reason=budget_err, + task_type="agentic", + ) + + try: + resolved_fixture = self._resolve_fixture_dir(task, fixture_dir) + except FixtureSetupError as exc: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_FIXTURE_FAILURE, + fail_reason=str(exc), + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", ) - t0 = time.time() - self.replays_run += 1 - tools_called: List[str] = [] - diff: str = "" - - # 2. Spin up disposable workspace - base_dir = fixture_dir - if not base_dir and task.context_excerpt and os.path.isdir(task.context_excerpt): - base_dir = task.context_excerpt - workspace = AgenticWorkspace( - base_dir=base_dir, - pinned_commit=self.config.pinned_commit, + ws = AgenticWorkspace( + base_dir=resolved_fixture, + pinned_commit=self.config.pinned_commit or None, isolate_env=self.config.isolate_env, - isolate_network=self.config.isolate_network, + fixture_root=self.config.fixture_root, + max_diff_bytes=self.config.max_diff_bytes, + max_output_bytes=self.config.max_output_bytes, ) - try: - workspace_path = workspace.setup() - env = workspace.build_isolated_env() + try: + ws.setup() + except FixtureSetupError as exc: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_FIXTURE_FAILURE, + fail_reason=str(exc), + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", + ) - # 3. Execute agentic interaction - timed_out = False - infra_error = False - error_msg = "" - actions_count = 0 + env = ws.build_isolated_env() + prompt_file = ws.write_prompt_payload(task, skill=skill, memory=memory) + prompt_text = ws.build_prompt_text(task, skill=skill, memory=memory) + prompt_ctx = { + "task": task, + "task_id": task.id, + "intent": task.intent, + "context_excerpt": task.context_excerpt, + "skill": skill, + "memory": memory, + "prompt": prompt_text, + "prompt_file": prompt_file, + "workspace": ws, + "timeout_sec": self.config.timeout_sec, + "max_turns": self.config.max_turns, + "max_output_bytes": self.config.max_output_bytes, + } - if agent_runner is not None: - try: - tools_called = agent_runner(workspace_path, env) or [] - actions_count = len(tools_called) - except subprocess.TimeoutExpired: - timed_out = True - except Exception as exc: - infra_error = True - error_msg = str(exc) - elif self.config.agent_cmd: - try: - cmd = [self.config.agent_cmd, task.intent] - proc = subprocess.run( - cmd, - cwd=workspace_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=self.config.timeout_sec, - env=env, - creationflags=_NO_WINDOW, - ) - actions_count = 1 if proc.returncode == 0 else 0 - except subprocess.TimeoutExpired: - timed_out = True - except Exception as exc: - infra_error = True - error_msg = str(exc) - else: - # Mock / simulated agentic execution from task properties - simulated_resp = self.attempt(task, skill, memory, sample_id=sample_id) - # Extract simulated tool calls or edit markers - if "TOOL_CALL:" in simulated_resp: - for line in simulated_resp.splitlines(): - if "TOOL_CALL:" in line: - tools_called.append(line.split("TOOL_CALL:", 1)[1].strip()) - actions_count = len(tools_called) if tools_called else (1 if simulated_resp else 0) - - # Check timeouts / infra issues - if timed_out: + response_text = "" + actions_taken = 0 + turns_taken = 0 + tokens_used_now = 0 + runner_rc = 0 + runner_outcome = "" + runner_reason = "" + runner_fail_msg = "" + + try: + res = _invoke_runner(self.agent_runner, ws.workspace_path, env, prompt_ctx) + if isinstance(res, dict): + response_text = str(res.get("response", "")) + actions_taken = int(res.get("actions", 1)) + turns_taken = int(res.get("turns", actions_taken)) + tokens_used_now = max(0, int(res.get("tokens", 0) or 0)) + raw_rc = res.get("returncode", res.get("exit_code", 0)) + runner_rc = 0 if raw_rc is None else int(raw_rc) + runner_outcome = str(res.get("outcome", "") or "") + runner_reason = str(res.get("inconclusive_reason", "") or "") + runner_fail_msg = str(res.get("fail_reason", "") or "") + elif isinstance(res, tuple) and len(res) >= 2: + response_text = str(res[0]) + actions_taken = int(res[1]) + turns_taken = actions_taken + else: + response_text = str(res) + actions_taken = 1 + turns_taken = 1 + except subprocess.TimeoutExpired: return ReplayResult( id=task.id, hard=0.0, soft=0.0, outcome=OUTCOME_INCONCLUSIVE, inconclusive_reason=REASON_TIMEOUT, - fail_reason=f"Agent run timed out after {self.config.timeout_sec}s", - judge_rationale="turn/time budget exceeded", - latency_ms=round((time.time() - t0) * 1000.0, 1), + fail_reason=f"Agent execution timed out after {self.config.timeout_sec}s", + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", + ) + except FixtureSetupError as exc: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_FIXTURE_FAILURE, + fail_reason=str(exc), + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", + ) + except Exception as exc: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_INFRA, + fail_reason=f"Agent runner infrastructure error: {exc}", + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", + ) + + with self._lock: + self.tokens_spent += tokens_used_now + over_token_budget = ( + self.config.max_tokens > 0 and self.tokens_spent > self.config.max_tokens + ) + + if runner_outcome == OUTCOME_INCONCLUSIVE: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + response=response_text[:4000], + tokens=tokens_used_now, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=runner_reason or REASON_INFRA, + fail_reason=runner_fail_msg or "Runner reported inconclusive outcome", + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", ) - if infra_error: + + if runner_rc != 0: return ReplayResult( id=task.id, hard=0.0, soft=0.0, + response=response_text[:4000], + tokens=tokens_used_now, outcome=OUTCOME_INCONCLUSIVE, inconclusive_reason=REASON_INFRA, - fail_reason=f"Infrastructure error during agent run: {error_msg}", - judge_rationale=f"infra failure: {error_msg}", - latency_ms=round((time.time() - t0) * 1000.0, 1), + fail_reason=runner_fail_msg or f"Agent process exited with code {runner_rc}", + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", ) - # 4. Check decidable action floor (issue #155 constraint) - if actions_count < self.config.min_decidable_actions: + if over_token_budget or ( + self.config.max_turns > 0 and turns_taken > self.config.max_turns + ): return ReplayResult( id=task.id, hard=0.0, soft=0.0, + response=response_text[:4000], + tokens=tokens_used_now, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_BUDGET_EXHAUSTED, + fail_reason=( + "Max agentic token budget exceeded" + if over_token_budget + else f"Turn budget exceeded ({turns_taken} > {self.config.max_turns})" + ), + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", + ) + + if actions_taken < self.config.min_decidable_actions: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + response=response_text[:4000], + tokens=tokens_used_now, outcome=OUTCOME_INCONCLUSIVE, inconclusive_reason=REASON_INSUFFICIENT_EVIDENCE, fail_reason=( - f"Decidable actions ({actions_count}) below threshold " + f"Actions taken ({actions_taken}) below decidable floor " f"({self.config.min_decidable_actions})" ), - judge_rationale="insufficient decidable actions", - tools_called=tools_called, - latency_ms=round((time.time() - t0) * 1000.0, 1), + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", ) - # 5. Extract unified diff - diff = workspace.get_diff() - - # 6. Task verification (primary acceptance gate) - active_verifier = self.verifier - if test_cmd or expected_files: - active_verifier = TaskVerifier( - test_cmd=test_cmd, - expected_files=expected_files, - timeout=min(self.config.timeout_sec, 30.0), + try: + diff = ws.get_diff() + except FixtureSetupError as exc: + return ReplayResult( + id=task.id, + hard=0.0, + soft=0.0, + response=response_text[:4000], + tokens=tokens_used_now, + outcome=OUTCOME_INCONCLUSIVE, + inconclusive_reason=REASON_FIXTURE_FAILURE, + fail_reason=str(exc), + latency_ms=(time.monotonic() - t0) * 1000.0, + task_type="agentic", ) - outcome, hard, soft, rationale, reason = active_verifier.verify(workspace_path, env) - - tokens = 500 # Baseline token estimate for agentic interaction - self._tokens_spent += tokens - latency_ms = (time.time() - t0) * 1000.0 + active_verifier = ( + verifier + or self.default_verifier + or TaskVerifier( + timeout_sec=self.config.timeout_sec, + max_output_bytes=self.config.max_output_bytes, + ) + ) + outcome, hard, soft, rationale, inc_reason = active_verifier.verify( + ws.workspace_path, env=env + ) + latency_ms = (time.monotonic() - t0) * 1000.0 + fail_reason = "" if outcome == OUTCOME_PASS else rationale return ReplayResult( id=task.id, hard=hard, soft=soft, + response=response_text[:4000], + fail_reason=fail_reason, + task_type="agentic", + judge_rationale=rationale, + tokens=tokens_used_now, + latency_ms=latency_ms, outcome=outcome, - inconclusive_reason=reason, + inconclusive_reason=inc_reason, diff=diff, - response=f"Agentic replay in {workspace_path} (diff: {len(diff)} chars)", - fail_reason="" if outcome == OUTCOME_PASS else rationale, - judge_rationale=rationale, - tools_called=tools_called, - tokens=tokens, - latency_ms=round(latency_ms, 1), ) finally: - workspace.cleanup() + ws.cleanup() diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py index e84a613d8..8b2a4df2f 100644 --- a/skillopt_sleep/backend.py +++ b/skillopt_sleep/backend.py @@ -29,7 +29,7 @@ import tempfile import threading from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Mapping, Optional, Tuple from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord @@ -2751,6 +2751,7 @@ def get_backend( opencode_tool_replay: bool = False, azure_endpoint: str = "", project_dir: str = "", + agentic_config: Optional[Mapping[str, Any]] = None, ) -> Backend: n = (name or "mock").strip().lower() if n in {"pi", "pi_cli", "pi_coding_agent", "pi-coding-agent"}: @@ -2782,8 +2783,18 @@ def get_backend( ) return HandoffBackend(model=model, handoff_dir=hdir) if n in {"agentic", "agentic_replay", "agentic-replay"}: - from skillopt_sleep.agentic_replay import AgenticReplayBackend - return AgenticReplayBackend() + from skillopt_sleep.agentic_replay import ( + AgenticReplayBackend, + AgenticReplayConfig, + agentic_config_from_sleep_config, + ) + if isinstance(agentic_config, AgenticReplayConfig): + cfg_obj = agentic_config + elif agentic_config is not None: + cfg_obj = agentic_config_from_sleep_config(agentic_config) + else: + cfg_obj = AgenticReplayConfig() + return AgenticReplayBackend(cfg_obj) if n in {"mock", ""}: return MockBackend() raise ValueError( @@ -2808,6 +2819,7 @@ def build_backend( azure_endpoint: str = "", preferences: str = "", project_dir: str = "", + agentic_config: Optional[Mapping[str, Any]] = None, ) -> Backend: """Build a single or dual backend. @@ -2816,11 +2828,16 @@ def build_backend( from (backend, model). ``preferences`` (free text) is attached so reflect uses it as a prior (set on the optimizer for dual backends). """ - has_split = any([optimizer_backend, optimizer_model, target_backend, target_model]) - if not has_split: - be = get_backend( - backend, - model=model, + def _call_get(resolved_name: str, resolved_model: str) -> Backend: + extra: Dict[str, Any] = {} + if ( + agentic_config is not None + and (resolved_name or "").strip().lower() in {"agentic", "agentic_replay", "agentic-replay"} + ): + extra["agentic_config"] = agentic_config + return get_backend( + resolved_name, + model=resolved_model, codex_path=codex_path, pi_path=pi_path, cursor_path=cursor_path, @@ -2828,19 +2845,16 @@ def build_backend( opencode_tool_replay=opencode_tool_replay, azure_endpoint=azure_endpoint, project_dir=project_dir, + **extra, ) + + has_split = any([optimizer_backend, optimizer_model, target_backend, target_model]) + if not has_split: + be = _call_get(backend, model) be.preferences = preferences return be - tgt = get_backend(target_backend or backend, model=target_model or model, - codex_path=codex_path, pi_path=pi_path, cursor_path=cursor_path, - opencode_path=opencode_path, opencode_tool_replay=opencode_tool_replay, - azure_endpoint=azure_endpoint, - project_dir=project_dir) - opt = get_backend(optimizer_backend or backend, model=optimizer_model or model, - codex_path=codex_path, pi_path=pi_path, cursor_path=cursor_path, - opencode_path=opencode_path, opencode_tool_replay=opencode_tool_replay, - azure_endpoint=azure_endpoint, - project_dir=project_dir) + tgt = _call_get(target_backend or backend, target_model or model) + opt = _call_get(optimizer_backend or backend, optimizer_model or model) opt.preferences = preferences # reflect runs on the optimizer dual = DualBackend(target=tgt, optimizer=opt) dual.preferences = preferences diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index 16f983818..a4a316cd5 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -70,13 +70,15 @@ "gate_no_regression": False, # reject any candidate that lowers a val-task score "replay_mode": "mock", # report label; "mock" | "text" | "agentic" (issue #155) # ── agentic replay tier (opt-in; issue #155) ─────────────────────────── - "agentic_replay": False, # opt-in tiered agentic replay in isolated workspace - "agentic_max_turns": 10, # turn cap per agentic replay + "agentic_replay": False, # strict boolean opt-in for agentic replay runner execution + "agentic_max_turns": 10, # turn/action cap per agentic replay + "agentic_max_tokens": 0, # cumulative token cap across replays (0 = unlimited) "agentic_timeout_sec": 60.0, # wall-clock timeout in seconds "agentic_max_replays": 5, # nightly budget cap on agentic replays - "agentic_isolate_network": True, # disable external network by default - "agentic_isolate_env": True, # scrub host credentials and user keys - "agentic_min_actions": 1, # floor below which outcome is inconclusive (insufficient_evidence) + "agentic_isolate_env": True, # allowlist child environment + empty config dirs + "agentic_min_actions": 1, # decidable floor below which outcome is inconclusive + "agentic_fixture_root": "", # explicit root required to resolve directory fixtures + "agentic_min_coverage": 0.5, # minimum pairwise decidable val fraction for gate # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─ "dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task "dream_factor": 0, # >0 => add N synthetic variants of each task to the dream diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py index cda388fba..eb5457a19 100644 --- a/skillopt_sleep/consolidate.py +++ b/skillopt_sleep/consolidate.py @@ -18,7 +18,7 @@ # research package, so this open-source tool stays decoupled from the paper code). from skillopt_sleep.gate import evaluate_gate, select_gate_score from skillopt_sleep.memory import apply_edits_detailed -from skillopt_sleep.replay import aggregate_scores, replay_batch +from skillopt_sleep.replay import aggregate_scores, pairwise_aggregate_scores, replay_batch from skillopt_sleep.types import ( EditRecord, ReplayResult, @@ -129,6 +129,35 @@ def _holdout_detail(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> List[dict]: return out +def _has_inconclusive(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> bool: + return any(getattr(r, "outcome", "") == "inconclusive" for _t, r in pairs) + + +def _score_val_comparison( + val_tasks: List[TaskRecord], + ref_pairs: List[Tuple[TaskRecord, ReplayResult]], + cand_pairs: List[Tuple[TaskRecord, ReplayResult]], + min_decidable_coverage: float, +) -> Tuple[float, float, float, float, int, int, float, bool]: + """Compute baseline/candidate scores, excluding inconclusive pairs identically on both sides.""" + if _has_inconclusive(ref_pairs) or _has_inconclusive(cand_pairs): + (b_h, b_s), (c_h, c_s), n_decidable, _n_tot, coverage = pairwise_aggregate_scores( + ref_pairs, cand_pairs + ) + n_total = len(val_tasks) if val_tasks else _n_tot + if val_tasks and _n_tot: + coverage = n_decidable / len(val_tasks) + abstained = bool(val_tasks) and ( + n_decidable == 0 or coverage < float(min_decidable_coverage) + ) + return b_h, b_s, c_h, c_s, n_decidable, n_total, coverage, abstained + + b_h, b_s = aggregate_scores(ref_pairs) + c_h, c_s = aggregate_scores(cand_pairs) + n_total = len(val_tasks) + return b_h, b_s, c_h, c_s, n_total, n_total, 1.0, False + + def _task_deltas( tasks: List[TaskRecord], baseline_pairs: List[Tuple[TaskRecord, ReplayResult]], @@ -143,17 +172,17 @@ def _task_deltas( for task in tasks: baseline = baseline_by_id.get(task.id) candidate = candidate_by_id.get(task.id) - baseline_score = ( - select_gate_score(baseline.hard, baseline.soft, metric, mixed_weight) - if baseline is not None - else None - ) - candidate_score = ( - select_gate_score(candidate.hard, candidate.soft, metric, mixed_weight) - if candidate is not None - else None - ) - if baseline_score is None or candidate_score is None: + if baseline is None or candidate is None: + baseline_score = ( + select_gate_score(baseline.hard, baseline.soft, metric, mixed_weight) + if baseline is not None + else None + ) + candidate_score = ( + select_gate_score(candidate.hard, candidate.soft, metric, mixed_weight) + if candidate is not None + else None + ) out.append({ "task_id": task.id, "tags": list(task.tags or []), @@ -163,6 +192,28 @@ def _task_deltas( "scores_are_finite": False, }) continue + + base_inc = getattr(baseline, "outcome", "") == "inconclusive" + cand_inc = getattr(candidate, "outcome", "") == "inconclusive" + if base_inc or cand_inc: + b_val = None if base_inc else _finite_score( + select_gate_score(baseline.hard, baseline.soft, metric, mixed_weight) + ) + c_val = None if cand_inc else _finite_score( + select_gate_score(candidate.hard, candidate.soft, metric, mixed_weight) + ) + out.append({ + "task_id": task.id, + "tags": list(task.tags or []), + "baseline_score": b_val, + "candidate_score": c_val, + "status": "inconclusive", + "scores_are_finite": True, + }) + continue + + baseline_score = select_gate_score(baseline.hard, baseline.soft, metric, mixed_weight) + candidate_score = select_gate_score(candidate.hard, candidate.soft, metric, mixed_weight) scores_are_finite = math.isfinite(baseline_score) and math.isfinite( candidate_score ) @@ -204,6 +255,7 @@ def consolidate( evolve_memory: bool = True, night: int = 1, budget_exhausted: Optional[Callable[[], bool]] = None, + min_decidable_coverage: float = 0.5, ) -> ConsolidationResult: """Run one consolidation epoch: reflect -> bounded edit -> gate. @@ -263,7 +315,7 @@ def _budget_spent(step: str) -> bool: else: evlog.set_phase(backend, "baseline_val") base_pairs = replay_batch(backend, val_tasks, skill, memory) - base_hard, base_soft = aggregate_scores(base_pairs) + base_hard, base_soft = aggregate_scores(base_pairs, exclude_inconclusive=True) holdout_detail = _holdout_detail(base_pairs) base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight) if ev is not None: @@ -275,8 +327,14 @@ def _budget_spent(step: str) -> bool: # ── reflect over TRAIN-split failures/successes ─────────────────────── evlog.set_phase(backend, "train") train_pairs = replay_batch(backend, train_tasks, skill, memory) - failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0] - successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0] + failures = [ + (t, r) for (t, r) in train_pairs + if r.hard < 1.0 and getattr(r, "outcome", "") != "inconclusive" + ] + successes = [ + (t, r) for (t, r) in train_pairs + if r.hard >= 1.0 and getattr(r, "outcome", "") != "inconclusive" + ] cand_skill, cand_memory = skill, memory all_applied: List[EditRecord] = [] @@ -317,7 +375,14 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: trial_memory = new_doc if which == "memory" else cand_memory evlog.set_phase(backend, f"gate_trial:{which}") pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory) - h, s = aggregate_scores(pairs) + b_h, b_s, h, s, n_decidable, _n_tot, coverage, below_floor = _score_val_comparison( + val_tasks, current_pairs, pairs, min_decidable_coverage + ) + trial_base_score = ( + select_gate_score(b_h, b_s, gate_metric, gate_mixed_weight) + if (_has_inconclusive(current_pairs) or _has_inconclusive(pairs)) + else base_score + ) cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight) task_deltas = _task_deltas( val_tasks, current_pairs, pairs, gate_metric, gate_mixed_weight @@ -326,21 +391,27 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: gate_no_regression and any(row["status"] == "regressed" for row in task_deltas) ) - trial_base_score = base_score - improved = cand_score > base_score and not blocked_by_regression - gate_trials.append({ + improved = (not below_floor) and cand_score > trial_base_score and not blocked_by_regression + trial_record: dict = { "target": which, "baseline_score": _finite_score(trial_base_score), "candidate_score": _finite_score(cand_score), "accepted": improved, "blocked_by_regression": blocked_by_regression, "task_deltas": task_deltas, - }) + } + if below_floor or _has_inconclusive(current_pairs) or _has_inconclusive(pairs): + trial_record["abstained_inconclusive"] = below_floor + trial_record["decidable_coverage"] = coverage + trial_record["n_decidable"] = n_decidable + gate_trials.append(trial_record) if ev is not None: ev.log("gate", "trial", target=which, mode="gated", baseline_score=trial_base_score, cand_hard=h, cand_soft=s, cand_score=cand_score, accepted=improved, blocked_by_regression=blocked_by_regression, + abstained_inconclusive=below_floor, + decidable_coverage=coverage, task_deltas=task_deltas, n_edits=len(applied)) if improved: @@ -441,7 +512,11 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: else: evlog.set_phase(backend, "final_val") final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory) - final_hard, final_soft = aggregate_scores(final_pairs) + base_hard_cmp, base_soft_cmp, final_hard, final_soft, n_decidable, _n_tot, coverage, abstain_inconclusive = _score_val_comparison( + val_tasks, base_pairs, final_pairs, min_decidable_coverage + ) + base_hard = base_hard_cmp + base_soft = base_soft_cmp final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight) base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight) final_task_deltas = _task_deltas( @@ -451,65 +526,74 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: gate_no_regression and any(row["status"] == "regressed" for row in final_task_deltas) ) - if _HAVE_REPO_GATE: - gate = evaluate_gate( - candidate_skill=cand_skill, - cand_hard=final_hard, - current_skill=skill, - current_score=base_gate_score, - best_skill=skill, - best_score=base_gate_score, - best_step=night - 1, - global_step=night, - cand_soft=final_soft, - metric=gate_metric, - mixed_weight=gate_mixed_weight, - ) - action = gate.action - accepted = ( - bool(all_applied) - and final_score > base_gate_score - and not final_blocked_by_regression - ) - else: - action = "accept" if final_score > base_gate_score else "reject" - accepted = ( - bool(all_applied) - and final_score > base_gate_score - and not final_blocked_by_regression - ) - # The gate scores documents, not edit bookkeeping: when every proposed - # edit was dropped during the per-target trials, `all_applied` is empty - # and nothing changed, yet the score comparison can still yield an - # accept-flavoured action. Reporting that as "accept_new_best" while - # `accepted` is False makes the headline contradict the outcome. - if not accepted and action in {"accept", "accept_new_best"}: - action = "reject" - # The gate cannot certify an improvement it measured on the same tasks - # the edits were derived from -- that comparison cannot detect - # overfitting, which is exactly how a reward hack scores 1.0. Abstain - # rather than report a verdict the evidence does not support. Edits are - # still staged, so a human can review them. - if accepted and holdout_leaked: - action = "reject_unverified" + if abstain_inconclusive: + action = "abstain_inconclusive" accepted = False + else: + if _HAVE_REPO_GATE: + gate = evaluate_gate( + candidate_skill=cand_skill, + cand_hard=final_hard, + current_skill=skill, + current_score=base_gate_score, + best_skill=skill, + best_score=base_gate_score, + best_step=night - 1, + global_step=night, + cand_soft=final_soft, + metric=gate_metric, + mixed_weight=gate_mixed_weight, + ) + action = gate.action + accepted = ( + bool(all_applied) + and final_score > base_gate_score + and not final_blocked_by_regression + ) + else: + action = "accept" if final_score > base_gate_score else "reject" + accepted = ( + bool(all_applied) + and final_score > base_gate_score + and not final_blocked_by_regression + ) + # The gate scores documents, not edit bookkeeping: when every proposed + # edit was dropped during the per-target trials, `all_applied` is empty + # and nothing changed, yet the score comparison can still yield an + # accept-flavoured action. Reporting that as "accept_new_best" while + # `accepted` is False makes the headline contradict the outcome. + if not accepted and action in {"accept", "accept_new_best"}: + action = "reject" + # The gate cannot certify an improvement it measured on the same tasks + # the edits were derived from -- that comparison cannot detect + # overfitting, which is exactly how a reward hack scores 1.0. Abstain + # rather than report a verdict the evidence does not support. Edits are + # still staged, so a human can review them. + if accepted and holdout_leaked: + action = "reject_unverified" + accepted = False # A per-target trial can improve and tentatively apply an edit, while a - # later fresh final replay regresses. The returned documents already - # roll back in that case; keep the edit bookkeeping/report consistent - # by moving those tentative edits into the rejected set as well. + # later fresh final replay regresses or becomes inconclusive. The returned + # documents already roll back in that case; keep the edit bookkeeping/report + # consistent by moving those tentative edits into the rejected set as well. if not accepted and all_applied: for edit in all_applied: if edit not in all_rejected: all_rejected.append(edit) all_applied = [] - gate_trials.append({ + final_trial_record: dict = { "target": "final", "baseline_score": _finite_score(base_gate_score), "candidate_score": _finite_score(final_score), "accepted": accepted, "blocked_by_regression": final_blocked_by_regression, "task_deltas": final_task_deltas, - }) + } + if abstain_inconclusive or _has_inconclusive(base_pairs) or _has_inconclusive(final_pairs): + final_trial_record["abstained_inconclusive"] = abstain_inconclusive + final_trial_record["decidable_coverage"] = coverage + final_trial_record["n_decidable"] = n_decidable + gate_trials.append(final_trial_record) if ev is not None: w = max(0.0, min(1.0, float(gate_mixed_weight))) diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index b3711f7d8..7229d61e5 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -18,7 +18,7 @@ import time import unicodedata from dataclasses import dataclass -from typing import Dict, List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence from skillopt_sleep import evidence from skillopt_sleep.backend import Backend, CursorBackendError, build_backend @@ -118,6 +118,22 @@ def _night_token_budget(cfg: SleepConfig) -> Budget: return Budget(max_tokens=limit or None) +def _agentic_config_payload(cfg: SleepConfig) -> dict[str, Any]: + """Extract agentic replay configuration keys from SleepConfig.""" + keys = ( + "agentic_replay", + "agentic_max_turns", + "agentic_max_tokens", + "agentic_timeout_sec", + "agentic_max_replays", + "agentic_isolate_env", + "agentic_min_actions", + "agentic_fixture_root", + "agentic_min_coverage", + ) + return {k: cfg.get(k) for k in keys if k in cfg.data} + + # ── Model-swap detection (F16) ─────────────────────────────── def _make_model_key(cfg: SleepConfig) -> str: """Stable string identifying the backend object(s) actually used. @@ -142,6 +158,7 @@ def _make_model_key(cfg: SleepConfig) -> str: opencode_tool_replay=cfg.get("opencode_tool_replay", False), azure_endpoint=cfg.get("azure_endpoint", ""), project_dir=cfg.get("invoked_project", "") or os.getcwd(), + agentic_config=_agentic_config_payload(cfg), ) except Exception: backend = str(cfg.get("backend", "mock") or "mock") @@ -179,6 +196,18 @@ def _make_backend_key(backend: Backend) -> str: return f"{name}::{model}" +def _reset_backend_run_budgets(backend: Backend) -> None: + """Reset per-night replay/token budgets on direct or split-role backends.""" + for resolved in ( + backend, + getattr(backend, "target", None), + getattr(backend, "optimizer", None), + ): + reset_fn = getattr(resolved, "reset_run_budget", None) + if callable(reset_fn): + reset_fn() + + def _claude_isolation_modes(backend: Backend, *, reset: bool = False) -> dict[str, list[str]]: """Keep per-night mode evidence for direct and split-role backends.""" roles = { @@ -741,9 +770,11 @@ def run_sleep_cycle( azure_endpoint=cfg.get("azure_endpoint", ""), preferences=cfg.get("preferences", ""), project_dir=project, + agentic_config=_agentic_config_payload(cfg), ) _check_model_change(cfg, state, backend) # F16: warn if model changed between nights _claude_isolation_modes(backend, reset=True) + _reset_backend_run_budgets(backend) night = state.begin_night(clock) started = _now_iso(clock) backend.preferences = cfg.get("preferences", "") @@ -805,6 +836,9 @@ def budget_exhausted() -> bool: cycle_config["opencode_tool_replay"] = ( cfg.get("opencode_tool_replay", False) is True ) + cycle_config["agentic_replay"] = ( + cfg.get("agentic_replay", False) is True + ) ev.log("cycle", "start", night=night, project=project, backend=backend.name, model=cfg.get("model", ""), config=cycle_config) @@ -981,6 +1015,7 @@ def budget_exhausted() -> bool: evolve_memory=cfg.get("evolve_memory", True), night=night, budget_exhausted=budget_exhausted, + min_decidable_coverage=float(cfg.get("agentic_min_coverage", 0.5) or 0.5), ) except CursorBackendError: _discard_unstaged_evidence(staging_dir_pre) @@ -1027,7 +1062,7 @@ def budget_exhausted() -> bool: final_skill = result.new_skill if result.accepted else skill final_memory = result.new_memory if result.accepted else memory test_pairs = replay_batch(backend, test_tasks, final_skill, final_memory) - test_hard, test_soft = aggregate_scores(test_pairs) + test_hard, test_soft = aggregate_scores(test_pairs, exclude_inconclusive=True) ev.log("test", "held_out_score", night=night, n_test=len(test_tasks), hard=round(test_hard, 4), soft=round(test_soft, 4), @@ -1086,6 +1121,9 @@ def budget_exhausted() -> bool: evolve_skill=cfg.get("evolve_skill", True), night=night, budget_exhausted=budget_exhausted, + min_decidable_coverage=float( + cfg.get("agentic_min_coverage", 0.5) or 0.5 + ), ) except CursorBackendError: _discard_unstaged_evidence(staging_dir_pre) diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py index 4dfdbc850..8afe27832 100644 --- a/skillopt_sleep/dream.py +++ b/skillopt_sleep/dream.py @@ -137,6 +137,7 @@ def dream_consolidate( evolve_memory: bool = True, night: int = 1, budget_exhausted: Optional[Callable[[], bool]] = None, + min_decidable_coverage: float = 0.5, ) -> ConsolidationResult: """Recall similar past experience + dream synthetic variants, then run one gated consolidation epoch over the enlarged training pool. @@ -170,4 +171,5 @@ def dream_consolidate( rollouts_k=dream_rollouts, evolve_skill=evolve_skill, evolve_memory=evolve_memory, night=night, budget_exhausted=budget_exhausted, + min_decidable_coverage=min_decidable_coverage, ) diff --git a/skillopt_sleep/replay.py b/skillopt_sleep/replay.py index 97fb78103..130878754 100644 --- a/skillopt_sleep/replay.py +++ b/skillopt_sleep/replay.py @@ -187,6 +187,42 @@ def aggregate_scores( return hard, soft +def pairwise_aggregate_scores( + base_pairs: List[Tuple[TaskRecord, ReplayResult]], + cand_pairs: List[Tuple[TaskRecord, ReplayResult]], +) -> Tuple[Tuple[float, float], Tuple[float, float], int, int, float]: + """Compare baseline and candidate over the exact intersection of decidable task IDs. + + Tasks marked ``outcome == "inconclusive"`` (or missing) on either side are + excluded pairwise so budget exhaustion or infra failures never bias one side. + Returns ``((base_hard, base_soft), (cand_hard, cand_soft), n_decidable, n_total, coverage)``. + """ + all_ids = list(dict.fromkeys([t.id for t, _r in base_pairs] + [t.id for t, _r in cand_pairs])) + n_total = len(all_ids) + if n_total == 0: + return (0.0, 0.0), (0.0, 0.0), 0, 0, 1.0 + + base_by_id = {t.id: r for t, r in base_pairs} + cand_by_id = {t.id: r for t, r in cand_pairs} + shared_ids = [ + tid for tid in all_ids + if base_by_id.get(tid) is not None + and cand_by_id.get(tid) is not None + and getattr(base_by_id[tid], "outcome", "") != "inconclusive" + and getattr(cand_by_id[tid], "outcome", "") != "inconclusive" + ] + n_decidable = len(shared_ids) + coverage = n_decidable / n_total + if n_decidable == 0: + return (0.0, 0.0), (0.0, 0.0), 0, n_total, coverage + + base_hard = sum(base_by_id[tid].hard for tid in shared_ids) / n_decidable + base_soft = sum(base_by_id[tid].soft for tid in shared_ids) / n_decidable + cand_hard = sum(cand_by_id[tid].hard for tid in shared_ids) / n_decidable + cand_soft = sum(cand_by_id[tid].soft for tid in shared_ids) / n_decidable + return (base_hard, base_soft), (cand_hard, cand_soft), n_decidable, n_total, coverage + + def aggregate_cost(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]: """Mean (tokens, latency_ms) per task — the cost objectives.""" if not pairs: diff --git a/tests/test_agentic_replay.py b/tests/test_agentic_replay.py index 617ebc9d7..1ea365788 100644 --- a/tests/test_agentic_replay.py +++ b/tests/test_agentic_replay.py @@ -1,28 +1,72 @@ -"""Tests for AgenticReplayBackend and agentic replay slice (Issue #155).""" +"""Tests for AgenticReplayBackend and workspace lifecycle slice (Issue #155).""" +from __future__ import annotations + import os import subprocess import sys import tempfile import unittest +from concurrent.futures import ThreadPoolExecutor +from unittest import mock from skillopt_sleep.agentic_replay import ( OUTCOME_FAIL, OUTCOME_INCONCLUSIVE, OUTCOME_PASS, REASON_BUDGET_EXHAUSTED, + REASON_FIXTURE_FAILURE, REASON_INFRA, REASON_INSUFFICIENT_EVIDENCE, REASON_TIMEOUT, AgenticReplayBackend, AgenticReplayConfig, AgenticWorkspace, + FixtureSetupError, TaskVerifier, ) -from skillopt_sleep.backend import get_backend -from skillopt_sleep.replay import aggregate_scores, replay_one +from skillopt_sleep.backend import MockBackend, build_backend, get_backend +from skillopt_sleep.config import load_config +from skillopt_sleep.consolidate import consolidate +from skillopt_sleep.cycle import run_sleep_cycle +from skillopt_sleep.replay import aggregate_scores, pairwise_aggregate_scores, replay_batch, replay_one from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord +def _init_git_repo(repo_dir: str, files: dict[str, str] | None = None) -> str: + """Create a small standalone git repository and return its HEAD commit SHA.""" + os.makedirs(repo_dir, exist_ok=True) + for rel, content in (files or {"app.txt": "baseline\n"}).items(): + full = os.path.join(repo_dir, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "w", encoding="utf-8") as f: + f.write(content) + git_cfg = ["-c", "user.name=Test", "-c", "user.email=t@example.com", "-c", "commit.gpgsign=false"] + subprocess.run(["git", "init", "-b", "main", repo_dir], check=True, capture_output=True) + subprocess.run(["git", "-C", repo_dir, "add", "."], check=True, capture_output=True) + subprocess.run(["git", "-C", repo_dir, *git_cfg, "commit", "-m", "initial"], check=True, capture_output=True) + res = subprocess.run(["git", "-C", repo_dir, "rev-parse", "HEAD"], check=True, capture_output=True, text=True) + return res.stdout.strip() + + +def _write_fake_agent_script(path: str, body: str) -> str: + with open(path, "w", encoding="utf-8") as f: + f.write(body) + return path + + +def _pid_alive(pid: int) -> bool: + if pid <= 0: + return False + if os.name == "nt": + res = subprocess.run(["tasklist", "/FI", f"PID eq {pid}", "/NH"], capture_output=True, text=True) + return str(pid) in res.stdout + try: + os.kill(pid, 0) + return True + except OSError: + return False + + class TestReplayResultOutcomes(unittest.TestCase): def test_default_pass_fail_inference(self): passed = ReplayResult(id="t1", hard=1.0) @@ -52,234 +96,707 @@ def test_explicit_inconclusive_outcome(self): self.assertIn("--- a/file", d["diff"]) -class TestAgenticWorkspace(unittest.TestCase): - def test_workspace_isolation_and_git_baseline(self): - with AgenticWorkspace(isolate_env=True, isolate_network=True) as ws: - self.assertTrue(os.path.isdir(ws.workspace_path)) - # Verify git repo was initialized with baseline commit - git_res = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=ws.workspace_path, - capture_output=True, - text=True, - ) - self.assertEqual(git_res.returncode, 0) - self.assertTrue(len(git_res.stdout.strip()) >= 7) - - # Check environment scrubbing - os.environ["TEST_SECRET_KEY"] = "supersecret" - os.environ["AWS_ACCESS_KEY_ID"] = "AKIAEXAMPLE" - os.environ["GITHUB_TOKEN"] = "ghp_example" - env = ws.build_isolated_env() - - self.assertNotIn("TEST_SECRET_KEY", env) - self.assertNotIn("AWS_ACCESS_KEY_ID", env) - self.assertNotIn("GITHUB_TOKEN", env) - self.assertEqual(env.get("HTTP_PROXY"), "http://127.0.0.1:9") - self.assertTrue(os.path.isdir(env.get("HOME", ""))) - self.assertIn(".sandbox_home", env.get("HOME", "")) - - # Test git diff tracking - file_path = os.path.join(ws.workspace_path, "sample.txt") - with open(file_path, "w", encoding="utf-8") as f: - f.write("hello world\n") - - diff = ws.get_diff() - self.assertIn("sample.txt", diff) - self.assertIn("+hello world", diff) - - # Verify workspace is cleaned up on exit - self.assertFalse(os.path.exists(ws.workspace_path)) - +class TestAgenticOptInAndConfigWiring(unittest.TestCase): + def test_disabled_by_default_blocks_execution(self): + called = False -class TestTaskVerifier(unittest.TestCase): - def test_verifier_fn(self): - verifier = TaskVerifier(verifier_fn=lambda path: (True, "all checks passed")) - with tempfile.TemporaryDirectory() as tmp: - outcome, hard, soft, rationale, reason = verifier.verify(tmp) - self.assertEqual(outcome, OUTCOME_PASS) - self.assertEqual(hard, 1.0) - self.assertEqual(soft, 1.0) - self.assertEqual(reason, "") + def _runner(ws_path, env, ctx): + nonlocal called + called = True + return {"response": "ran", "actions": 1} - failing_verifier = TaskVerifier(verifier_fn=lambda path: (False, "assertion failed")) - with tempfile.TemporaryDirectory() as tmp: - outcome, hard, soft, rationale, reason = failing_verifier.verify(tmp) - self.assertEqual(outcome, OUTCOME_FAIL) - self.assertEqual(hard, 0.0) - self.assertEqual(soft, 0.0) + for flag in (False, "true", 1): + cfg = AgenticReplayConfig(enabled=flag) # type: ignore[arg-type] + self.assertFalse(cfg.enabled) + backend = AgenticReplayBackend( + cfg, + agent_runner=_runner, + default_verifier=TaskVerifier(verifier_fn=lambda _p: (True, "ok")), + ) + res = backend.replay_agentic_task(TaskRecord(id="t1", project="/p", intent="do task")) + self.assertFalse(called) + self.assertEqual(res.outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(res.inconclusive_reason, REASON_INFRA) - def test_expected_files(self): + def test_config_keys_wired_through_load_config_build_backend_and_cycle(self): with tempfile.TemporaryDirectory() as tmp: - verifier = TaskVerifier(expected_files={"output.json": '"status": "ok"'}) - outcome, hard, _, _, _ = verifier.verify(tmp) - self.assertEqual(outcome, OUTCOME_FAIL) - - with open(os.path.join(tmp, "output.json"), "w", encoding="utf-8") as f: - f.write('{"status": "ok", "value": 42}') + fixture_root = os.path.join(tmp, "fixtures") + os.makedirs(fixture_root) + cfg = load_config( + backend="agentic", + agentic_replay=True, + agentic_max_turns=3, + agentic_max_tokens=1500, + agentic_timeout_sec=12.5, + agentic_max_replays=7, + agentic_isolate_env=False, + agentic_min_actions=2, + agentic_fixture_root=fixture_root, + agentic_min_coverage=0.75, + ) + payload = {k: cfg.get(k) for k in cfg.data if k.startswith("agentic_")} + be_alias = get_backend("agentic_replay", agentic_config=payload) + self.assertEqual(be_alias.name, "agentic") + self.assertTrue(be_alias.config.enabled) + + be = build_backend(backend="agentic", agentic_config=payload) + self.assertIsInstance(be, AgenticReplayBackend) + # Disabled runner invocation via replay_one fails closed cleanly + disabled_be = get_backend("agentic") + res_dis = replay_one(disabled_be, TaskRecord(id="r1", project="/p", intent="hi"), "", "") + self.assertEqual(res_dis.outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(res_dis.inconclusive_reason, REASON_INFRA) + self.assertTrue(be.config.enabled) + self.assertEqual(be.config.max_turns, 3) + self.assertEqual(be.config.max_tokens, 1500) + self.assertEqual(be.config.timeout_sec, 12.5) + self.assertEqual(be.config.max_replays_per_run, 7) + self.assertFalse(be.config.isolate_env) + self.assertEqual(be.config.min_decidable_actions, 2) + self.assertEqual(be.config.fixture_root, fixture_root) + self.assertEqual(be.config.min_coverage, 0.75) + + # Per-night reset clears replays_run at start of run_sleep_cycle + be.replays_run = 6 + be.tokens_spent = 999 + proj = os.path.join(tmp, "proj") + os.makedirs(proj) + run_cfg = load_config( + projects="invoked", + invoked_project=proj, + state_dir=os.path.join(tmp, "state"), + claude_home=os.path.join(tmp, "claude"), + evidence_log=False, + evolve_skill=False, + evolve_memory=False, + ) + run_sleep_cycle(run_cfg, backend=be, seed_tasks=[], dry_run=True) + self.assertEqual(be.replays_run, 0) + self.assertEqual(be.tokens_spent, 0) + + # Verify run_sleep_cycle itself constructs AgenticReplayBackend from SleepConfig + built_backends = [] + real_builder = build_backend + + def _capture_builder(**kwargs): + res_be = real_builder(**kwargs) + built_backends.append(res_be) + return res_be + + agentic_cfg = load_config( + backend="agentic", + agentic_replay=True, + agentic_max_turns=4, + agentic_max_replays=9, + projects="invoked", + invoked_project=proj, + state_dir=os.path.join(tmp, "state2"), + claude_home=os.path.join(tmp, "claude"), + evidence_log=False, + evolve_skill=False, + evolve_memory=False, + ) + with mock.patch("skillopt_sleep.cycle.build_backend", side_effect=_capture_builder): + run_sleep_cycle(agentic_cfg, seed_tasks=[], dry_run=True) + self.assertGreaterEqual(len(built_backends), 1) + self.assertIsInstance(built_backends[-1], AgenticReplayBackend) + self.assertTrue(built_backends[-1].config.enabled) + self.assertEqual(built_backends[-1].config.max_turns, 4) + self.assertEqual(built_backends[-1].config.max_replays_per_run, 9) + + def test_lazy_import_of_agentic_backend_in_package_init(self): + script = ( + "import sys, skillopt_sleep; " + "assert 'skillopt_sleep.agentic_replay' not in sys.modules; " + "from skillopt_sleep import AgenticReplayBackend; " + "assert AgenticReplayBackend.name == 'agentic'" + ) + res = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + self.assertEqual(res.returncode, 0, res.stderr) + + +class TestAgenticWorkspaceAndFakeAgentSubprocess(unittest.TestCase): + def test_workspace_scratch_copy_env_allowlist_and_home_outside_diff(self): + sensitive = { + "TEST_SECRET_KEY": "supersecret", + "AWS_ACCESS_KEY_ID": "AKIAEXAMPLE", + "GITHUB_TOKEN": "ghp_example", + "GOOGLE_APPLICATION_CREDENTIALS": "/tmp/creds.json", + "KUBECONFIG": "/tmp/kubeconfig", + "DATABASE_URL": "postgres://user:pass@host/db", + "GIT_ASKPASS": "/tmp/askpass.sh", + "DOCKER_HOST": "tcp://127.0.0.1:2375", + "DBUS_SESSION_BUS_ADDRESS": "unix:path=/run/user/1000/bus", + "IDENTITY_HEADER": "secret-hdr", + "IDENTITY_ENDPOINT": "http://169.254.169.254", + "CLAUDE_CONFIG_DIR": "/real/claude", + "CODEX_HOME": "/real/codex", + "HTTP_PROXY": "http://corp:8080", + "https_proxy": "https://corp:8080", + "no_proxy": "localhost", + } + with mock.patch.dict(os.environ, sensitive): + with AgenticWorkspace(isolate_env=True) as ws: + self.assertTrue(os.path.isdir(ws.workspace_path)) + self.assertFalse( + os.path.commonpath([ + os.path.realpath(ws.workspace_path), + os.path.realpath(ws.home_dir), + ]) == os.path.realpath(ws.workspace_path), + "HOME must sit outside diffed workspace_path", + ) + git_res = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ws.workspace_path, + capture_output=True, + text=True, + ) + self.assertEqual(git_res.returncode, 0) + self.assertGreaterEqual(len(git_res.stdout.strip()), 7) + + env = ws.build_isolated_env() + for forbidden in ( + "TEST_SECRET_KEY", "AWS_ACCESS_KEY_ID", "GITHUB_TOKEN", + "GOOGLE_APPLICATION_CREDENTIALS", "KUBECONFIG", "DATABASE_URL", + "DOCKER_HOST", "DBUS_SESSION_BUS_ADDRESS", + "IDENTITY_HEADER", "IDENTITY_ENDPOINT", + "HTTP_PROXY", "https_proxy", "no_proxy", + ): + self.assertNotIn(forbidden, env) + + self.assertEqual(env["GIT_ASKPASS"], "") + self.assertEqual(env["GIT_CONFIG_NOSYSTEM"], "1") + self.assertTrue(os.path.isfile(env["GIT_CONFIG_GLOBAL"])) + self.assertEqual(os.path.getsize(env["GIT_CONFIG_GLOBAL"]), 0) + self.assertEqual(env["HOME"], ws.home_dir) + self.assertEqual(os.listdir(env["CLAUDE_CONFIG_DIR"]), []) + self.assertEqual(os.listdir(env["CODEX_HOME"]), []) + self.assertNotIn(".sandbox_home", env["HOME"]) + + # Verify fake agent binary sees allowlisted env via subprocess and writes to HOME without dirtying diff + script_dir = tempfile.mkdtemp(prefix="skillopt_fake_") + try: + script_path = _write_fake_agent_script( + os.path.join(script_dir, "agent.py"), + ( + "import json, os\n" + "assert 'KUBECONFIG' not in os.environ\n" + "assert 'DATABASE_URL' not in os.environ\n" + "assert 'https_proxy' not in os.environ\n" + "with open(os.path.join(os.environ['HOME'], 'cache.txt'), 'w') as hf:\n" + " hf.write('temp-home-artifact\\n')\n" + "with open('sample.txt', 'w') as wf:\n" + " wf.write('hello world\\n')\n" + "print(json.dumps({'home': os.environ['HOME']}))\n" + ), + ) + timed_out, rc, out, err = ws.run_subprocess([sys.executable, script_path], env=env) + self.assertFalse(timed_out, err) + self.assertEqual(rc, 0, err) + finally: + for name in os.listdir(script_dir): + os.remove(os.path.join(script_dir, name)) + os.rmdir(script_dir) + + diff = ws.get_diff() + self.assertIn("sample.txt", diff) + self.assertIn("+hello world", diff) + self.assertNotIn("cache.txt", diff) + self.assertNotIn("temp-home-artifact", diff) - outcome, hard, _, _, _ = verifier.verify(tmp) - self.assertEqual(outcome, OUTCOME_PASS) - self.assertEqual(hard, 1.0) + self.assertFalse(os.path.exists(ws.workspace_path)) - def test_test_cmd_execution(self): + def test_fake_agent_worktree_fixture_cannot_touch_real_repo(self): with tempfile.TemporaryDirectory() as tmp: - # Pass command - verifier = TaskVerifier(test_cmd=[sys.executable, "-c", "exit(0)"]) - outcome, hard, _, _, _ = verifier.verify(tmp) - self.assertEqual(outcome, OUTCOME_PASS) - self.assertEqual(hard, 1.0) - - # Deterministic fail command - failing_verifier = TaskVerifier(test_cmd=[sys.executable, "-c", "exit(1)"]) - outcome, hard, _, _, _ = failing_verifier.verify(tmp) - self.assertEqual(outcome, OUTCOME_FAIL) - self.assertEqual(hard, 0.0) + fixture_root = os.path.join(tmp, "fixtures") + real_repo = os.path.join(fixture_root, "real_repo") + wt_dir = os.path.join(fixture_root, "wt") + head_sha = _init_git_repo(real_repo, {"tracked.txt": "v1\n"}) + subprocess.run( + ["git", "-C", real_repo, "worktree", "add", "--detach", wt_dir, head_sha], + check=True, + capture_output=True, + ) + hook_sentinel = os.path.join(real_repo, ".git", "hooks", "pwn_marker") + fake_script = _write_fake_agent_script( + os.path.join(tmp, "wt_agent.py"), + ( + "import os, subprocess, sys\n" + "common = subprocess.check_output(['git', 'rev-parse', '--git-common-dir'], text=True).strip()\n" + "main_head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()\n" + "with open('edited.txt', 'w') as f:\n" + " f.write('updated\\n')\n" + "subprocess.run(['git', 'add', 'edited.txt'], check=True)\n" + "subprocess.run(['git', '-c', 'user.name=A', '-c', 'user.email=a@a', 'commit', '-m', 'agent'], check=True)\n" + "new_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()\n" + "subprocess.run(['git', 'update-ref', 'refs/heads/main', new_sha], check=True)\n" + "with open(os.path.join(common, 'hooks', 'pwn_marker'), 'w') as hf:\n" + " hf.write('touched')\n" + ), + ) - # Timeout command - timeout_verifier = TaskVerifier( - test_cmd=[sys.executable, "-c", "import time; time.sleep(2)"], - timeout=0.2, + # 1. Unpinned worktree fixture must be rejected because .git is a linked file + be_unpinned = AgenticReplayBackend( + AgenticReplayConfig(enabled=True, fixture_root=fixture_root), + agent_runner=lambda _ws, _env: {"response": "ok", "actions": 1}, + default_verifier=TaskVerifier(verifier_fn=lambda _p: (True, "ok")), ) - outcome, hard, _, _, reason = timeout_verifier.verify(tmp) - self.assertEqual(outcome, OUTCOME_INCONCLUSIVE) - self.assertEqual(reason, REASON_TIMEOUT) + unpinned_res = be_unpinned.replay_agentic_task( + TaskRecord(id="wt-unpinned", project=real_repo, intent="edit"), + fixture_dir="wt", + ) + self.assertEqual(unpinned_res.outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(unpinned_res.inconclusive_reason, REASON_FIXTURE_FAILURE) + + # 2. Pinned worktree fixture materializes without .git pointer and isolates real_repo + def _run_wt(ws_path, env, ctx): + ws = ctx["workspace"] + to, rc, out, err = ws.run_subprocess([sys.executable, fake_script], env=env) + return {"response": out, "actions": 1, "returncode": rc} + + be_pinned = AgenticReplayBackend( + AgenticReplayConfig(enabled=True, fixture_root=fixture_root, pinned_commit=head_sha), + agent_runner=_run_wt, + default_verifier=TaskVerifier(expected_files={"edited.txt": "updated"}), + ) + pinned_res = be_pinned.replay_agentic_task( + TaskRecord(id="wt-pinned", project=real_repo, intent="edit"), + fixture_dir="wt", + ) + self.assertEqual(pinned_res.outcome, OUTCOME_PASS, pinned_res.fail_reason) + self.assertIn("edited.txt", pinned_res.diff) + # Real repository's refs/heads/main and hooks dir are untouched + real_main = subprocess.run( + ["git", "-C", real_repo, "rev-parse", "refs/heads/main"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + self.assertEqual(real_main, head_sha) + self.assertFalse(os.path.exists(hook_sentinel)) + subprocess.run(["git", "-C", real_repo, "worktree", "remove", "--force", wt_dir], capture_output=True) -class TestAgenticReplayBackend(unittest.TestCase): - def test_successful_agentic_replay(self): - backend = AgenticReplayBackend() - task = TaskRecord(id="task_build", project="test", intent="fix build config") + def test_alternates_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + repo = os.path.join(tmp, "repo") + sha = _init_git_repo(repo) + alt_file = os.path.join(repo, ".git", "objects", "info", "alternates") + os.makedirs(os.path.dirname(alt_file), exist_ok=True) + with open(alt_file, "w", encoding="utf-8") as f: + f.write("/some/other/objects\n") + + with self.assertRaises(FixtureSetupError): + AgenticWorkspace(base_dir=repo, fixture_root=tmp).setup() + with self.assertRaises(FixtureSetupError): + AgenticWorkspace(base_dir=repo, pinned_commit=sha, fixture_root=tmp).setup() + + def test_fake_agent_git_config_diff_external_and_host_gpgsign(self): + with tempfile.TemporaryDirectory() as tmp: + broken_global_cfg = os.path.join(tmp, "host_gitconfig") + with open(broken_global_cfg, "w", encoding="utf-8") as f: + f.write("[commit]\n\tgpgsign = true\n[gpg]\n\tprogram = nonexistent-gpg-bin\n") + + hijack_marker = os.path.join(tmp, "DIFF_EXTERNAL_EXECUTED") + fake_agent = _write_fake_agent_script( + os.path.join(tmp, "hijack_agent.py"), + ( + "import os, sys\n" + f"marker = {hijack_marker!r}\n" + "cfg_path = os.path.join('.git', 'config')\n" + "with open(cfg_path, 'a', encoding='utf-8') as f:\n" + " f.write(f\"\\n[diff]\\n\\texternal = {sys.executable} -c \\\"open({marker!r}, 'w').write('pwnd')\\\"\\n\")\n" + "with open('feature.txt', 'w', encoding='utf-8') as f:\n" + " f.write('real feature\\n')\n" + ), + ) - def agent_runner(workspace_path, env): - target_file = os.path.join(workspace_path, "build.txt") - with open(target_file, "w", encoding="utf-8") as f: - f.write("build=success\n") - return ["write_file", "run_test"] + with mock.patch.dict(os.environ, {"GIT_CONFIG_GLOBAL": broken_global_cfg}): + be = AgenticReplayBackend( + AgenticReplayConfig(enabled=True), + agent_runner=lambda _ws, env, ctx: ( + lambda res: {"response": res[2], "actions": 1, "returncode": res[1]} + )(ctx["workspace"].run_subprocess([sys.executable, fake_agent], env=env)), + default_verifier=TaskVerifier(expected_files={"feature.txt": "real feature"}), + ) + res = be.replay_agentic_task(TaskRecord(id="git-cfg", project=tmp, intent="add feature")) + + self.assertEqual(res.outcome, OUTCOME_PASS, res.fail_reason) + self.assertIn("+real feature", res.diff) + self.assertFalse(os.path.exists(hijack_marker)) + + def test_fake_agent_mined_intent_flags_and_fixture_root_containment(self): + with tempfile.TemporaryDirectory() as tmp: + fixture_root = os.path.join(tmp, "fixtures") + allowed_dir = os.path.join(fixture_root, "proj") + outside_dir = os.path.join(tmp, "outside_secret") + os.makedirs(allowed_dir) + os.makedirs(outside_dir) + with open(os.path.join(allowed_dir, "allowed.txt"), "w", encoding="utf-8") as f: + f.write("ok\n") + with open(os.path.join(outside_dir, "secret.txt"), "w", encoding="utf-8") as f: + f.write("topsecret\n") + + fake_agent = _write_fake_agent_script( + os.path.join(tmp, "prompt_reader.py"), + ( + "import json, os, sys\n" + "stdin_intent = sys.stdin.read()\n" + "with open('seen_intent.txt', 'w', encoding='utf-8') as f:\n" + " f.write(stdin_intent)\n" + "payload_idx = sys.argv.index('--payload') + 1\n" + "payload = json.load(open(sys.argv[payload_idx], encoding='utf-8'))\n" + "with open('seen_skill.txt', 'w', encoding='utf-8') as f:\n" + " f.write(payload['skill'] + '|' + payload['memory'])\n" + ), + ) - result = backend.replay_agentic_task( - task, - skill="use correct build flags", - memory="", - expected_files={"build.txt": "build=success"}, - agent_runner=agent_runner, - ) + # 1. Task with flag-like intent ("--help") & context_excerpt pointing at existing outside_dir: + # outside_dir must NOT be treated as a fixture! Intent and skill must arrive via stdin/file. + def _runner(_ws_path, env, ctx): + ws = ctx["workspace"] + to, rc, out, err = ws.run_subprocess( + [sys.executable, fake_agent, "--payload", ctx["prompt_file"]], + stdin_text=ctx["intent"], + env=env, + ) + self.assertFalse(os.path.exists(os.path.join(ws.workspace_path, "secret.txt"))) + return {"response": out or "done", "actions": 1, "returncode": rc} + + be = AgenticReplayBackend( + AgenticReplayConfig(enabled=True, fixture_root=fixture_root), + agent_runner=_runner, + default_verifier=TaskVerifier(expected_files={ + "allowed.txt": "ok", + "seen_intent.txt": "--help --malicious", + "seen_skill.txt": "# Skill Body|# Memory Body", + }), + ) + task = TaskRecord( + id="flag-intent", + project=tmp, + intent="--help --malicious", + context_excerpt=outside_dir, + ) + res = be.replay_agentic_task( + task, + skill="# Skill Body", + memory="# Memory Body", + fixture_dir="proj", + ) + self.assertEqual(res.outcome, OUTCOME_PASS, res.fail_reason) - self.assertEqual(result.outcome, OUTCOME_PASS) - self.assertEqual(result.hard, 1.0) - self.assertEqual(result.soft, 1.0) - self.assertIn("build.txt", result.diff) - self.assertEqual(result.tools_called, ["write_file", "run_test"]) + # 2. Explicit fixture_dir outside fixture_root fails closed with REASON_FIXTURE_FAILURE + escape_res = be.replay_agentic_task( + task, + fixture_dir=os.path.join("..", "outside_secret"), + ) + self.assertEqual(escape_res.outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(escape_res.inconclusive_reason, REASON_FIXTURE_FAILURE) - def test_failing_agentic_replay(self): - backend = AgenticReplayBackend() - task = TaskRecord(id="task_fail", project="test", intent="break test") + def test_timeout_kills_process_tree_and_caps_output_and_diff(self): + with tempfile.TemporaryDirectory() as tmp: + pidfile = os.path.join(tmp, "grandchild.pid") + spawner = _write_fake_agent_script( + os.path.join(tmp, "spawner.py"), + ( + "import subprocess, sys, time\n" + "p = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])\n" + f"open({pidfile!r}, 'w').write(str(p.pid))\n" + "time.sleep(30)\n" + ), + ) + noisy = _write_fake_agent_script( + os.path.join(tmp, "noisy.py"), + ( + "import sys\n" + "sys.stdout.write('X' * 50000)\n" + "open('huge.txt', 'w').write('Y\\n' * 10000)\n" + ), + ) - def agent_runner(workspace_path, env): - return ["run_tests"] + with AgenticWorkspace(max_output_bytes=1024, max_diff_bytes=2048) as ws: + timed_out, rc, _out, _err = ws.run_subprocess( + [sys.executable, spawner], timeout_sec=0.4 + ) + self.assertTrue(timed_out) + self.assertNotEqual(rc, 0) + if os.path.exists(pidfile): + gpid = int(open(pidfile, encoding="utf-8").read().strip()) + self.assertFalse(_pid_alive(gpid), "grandchild process should be terminated") + + to2, rc2, out2, _ = ws.run_subprocess( + [sys.executable, noisy], timeout_sec=5.0 + ) + self.assertFalse(to2) + self.assertEqual(rc2, 0) + self.assertLess(len(out2), 2048) + self.assertIn("truncated", out2) + diff = ws.get_diff() + self.assertLess(len(diff), 4096) + self.assertIn("diff truncated", diff) + + +class TestTaskVerifierAndOutcomes(unittest.TestCase): + def test_verifier_checks_and_no_verifier_inconclusive(self): + with tempfile.TemporaryDirectory() as tmp: + # No verifier configured must be inconclusive (insufficient_evidence), NOT pass + empty_v = TaskVerifier() + outcome, hard, soft, _rat, reason = empty_v.verify(tmp) + self.assertEqual(outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(reason, REASON_INSUFFICIENT_EVIDENCE) + self.assertEqual(hard, 0.0) + self.assertEqual(soft, 0.0) - result = backend.replay_agentic_task( - task, - skill="", - memory="", - test_cmd=[sys.executable, "-c", "sys.exit(1)"], - agent_runner=agent_runner, - ) + verifier = TaskVerifier(verifier_fn=lambda _p: (True, "all checks passed")) + self.assertEqual(verifier.verify(tmp)[:3], (OUTCOME_PASS, 1.0, 1.0)) - self.assertEqual(result.outcome, OUTCOME_FAIL) - self.assertEqual(result.hard, 0.0) - self.assertIn("failed", result.fail_reason.lower()) + failing = TaskVerifier(verifier_fn=lambda _p: (False, "assertion failed")) + self.assertEqual(failing.verify(tmp)[:3], (OUTCOME_FAIL, 0.0, 0.0)) - def test_insufficient_decidable_actions(self): - config = AgenticReplayConfig(min_decidable_actions=2) - backend = AgenticReplayBackend(config=config) - task = TaskRecord(id="task_thin", project="test", intent="simple query") + exp = TaskVerifier(expected_files={"out.json": '"ok"'}) + self.assertEqual(exp.verify(tmp)[0], OUTCOME_FAIL) + with open(os.path.join(tmp, "out.json"), "w", encoding="utf-8") as f: + f.write('{"status": "ok"}') + self.assertEqual(exp.verify(tmp)[0], OUTCOME_PASS) - def zero_action_runner(workspace_path, env): - return [] # 0 actions, below threshold of 2 + timeout_v = TaskVerifier(test_cmd=[sys.executable, "-c", "import time; time.sleep(5)"], timeout_sec=0.3) + t_out = timeout_v.verify(tmp) + self.assertEqual(t_out[0], OUTCOME_INCONCLUSIVE) + self.assertEqual(t_out[4], REASON_TIMEOUT) - result = backend.replay_agentic_task( - task, - skill="", - memory="", - agent_runner=zero_action_runner, + def test_nonzero_agent_exit_is_infra_and_bad_pin_is_fixture_failure(self): + with tempfile.TemporaryDirectory() as tmp: + fixture_root = os.path.join(tmp, "fixtures") + repo = os.path.join(fixture_root, "repo") + _init_git_repo(repo) + + be_bad_pin = AgenticReplayBackend( + AgenticReplayConfig( + enabled=True, + fixture_root=fixture_root, + pinned_commit="deadbeefdeadbeefdeadbeef", + ), + agent_runner=lambda _ws, _env: {"response": "ok", "actions": 1}, + default_verifier=TaskVerifier(verifier_fn=lambda _p: (True, "ok")), + ) + pairs = replay_batch(be_bad_pin, [TaskRecord(id="badpin", project=repo, intent="test")], "", "") + self.assertEqual(len(pairs), 1) + self.assertEqual(pairs[0][1].outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(pairs[0][1].inconclusive_reason, REASON_FIXTURE_FAILURE) + + be_exit = AgenticReplayBackend( + AgenticReplayConfig(enabled=True), + agent_runner=lambda _ws, _env: {"response": "crash", "actions": 1, "returncode": 2}, + default_verifier=TaskVerifier(verifier_fn=lambda _p: (True, "ok")), + ) + res = be_exit.replay_agentic_task(TaskRecord(id="crash", project=tmp, intent="fail")) + self.assertEqual(res.outcome, OUTCOME_INCONCLUSIVE) + self.assertEqual(res.inconclusive_reason, REASON_INFRA) + + def test_budgets_and_thread_safety(self): + be = AgenticReplayBackend( + AgenticReplayConfig( + enabled=True, + max_turns=2, + max_tokens=100, + max_replays_per_run=3, + min_decidable_actions=1, + ), + agent_runner=lambda _ws, _env: {"response": "ok", "actions": 1, "turns": 1, "tokens": 0}, + default_verifier=TaskVerifier(verifier_fn=lambda _p: (True, "ok")), ) - - self.assertEqual(result.outcome, OUTCOME_INCONCLUSIVE) - self.assertEqual(result.inconclusive_reason, REASON_INSUFFICIENT_EVIDENCE) - - def test_budget_exhaustion(self): - config = AgenticReplayConfig(max_replays_per_run=1) - backend = AgenticReplayBackend(config=config) - task1 = TaskRecord(id="task_1", project="test", intent="first task") - task2 = TaskRecord(id="task_2", project="test", intent="second task") - - res1 = backend.replay_agentic_task(task1, skill="", memory="") - self.assertNotEqual(res1.inconclusive_reason, REASON_BUDGET_EXHAUSTED) - - # Second replay exceeds budget cap of 1 - res2 = backend.replay_agentic_task(task2, skill="", memory="") - self.assertEqual(res2.outcome, OUTCOME_INCONCLUSIVE) - self.assertEqual(res2.inconclusive_reason, REASON_BUDGET_EXHAUSTED) - - def test_tiered_reflection_excludes_inconclusive(self): - backend = AgenticReplayBackend() - task_real_fail = TaskRecord(id="tf", project="p", intent="real fail") - task_infra_fail = TaskRecord(id="ti", project="p", intent="infra fail") - - r_real_fail = ReplayResult(id="tf", hard=0.0, outcome=OUTCOME_FAIL) - r_infra_fail = ReplayResult( - id="ti", - hard=0.0, - outcome=OUTCOME_INCONCLUSIVE, - inconclusive_reason=REASON_TIMEOUT, + # Parallel replay batch of 6 tasks against cap 3 must yield strictly 3 pass + 3 budget_exhausted + tasks = [TaskRecord(id=f"p{i}", project="/p", intent="run") for i in range(6)] + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(lambda t: be.replay_agentic_task(t), tasks)) + passes = [r for r in results if r.outcome == OUTCOME_PASS] + budget = [r for r in results if r.inconclusive_reason == REASON_BUDGET_EXHAUSTED] + self.assertEqual(len(passes), 3) + self.assertEqual(len(budget), 3) + self.assertEqual(sum(r.tokens for r in results), 0) # No hardcoded fake 500-token constant + + # Check turn cap and token cap enforcement + be2 = AgenticReplayBackend( + AgenticReplayConfig(enabled=True, max_turns=2, max_tokens=50, max_replays_per_run=5), + agent_runner=lambda _ws, _env: {"response": "ok", "actions": 1, "turns": 3, "tokens": 10}, + default_verifier=TaskVerifier(verifier_fn=lambda _p: (True, "ok")), ) + turn_over = be2.replay_agentic_task(TaskRecord(id="t-turn", project="/p", intent="run")) + self.assertEqual(turn_over.inconclusive_reason, REASON_BUDGET_EXHAUSTED) - failures = [(task_real_fail, r_real_fail), (task_infra_fail, r_infra_fail)] - # Reflect should only process real failures, discarding the infra failure - edits = backend.reflect( + be3 = AgenticReplayBackend( + AgenticReplayConfig(enabled=True, max_turns=5, max_tokens=50, max_replays_per_run=5), + agent_runner=lambda _ws, _env: {"response": "ok", "actions": 1, "turns": 1, "tokens": 60}, + default_verifier=TaskVerifier(verifier_fn=lambda _p: (True, "ok")), + ) + tok_over = be3.replay_agentic_task(TaskRecord(id="t-tok", project="/p", intent="run")) + self.assertEqual(tok_over.inconclusive_reason, REASON_BUDGET_EXHAUSTED) + + +class TestReflectionAndPairwiseGateSemantics(unittest.TestCase): + def test_tiered_reflection_excludes_inconclusive_via_spy(self): + base = MockBackend() + spy_reflect = mock.MagicMock(wraps=base.reflect) + base.reflect = spy_reflect # type: ignore[method-assign] + backend = AgenticReplayBackend(AgenticReplayConfig(enabled=True), base_backend=base) + + t_fail = TaskRecord(id="f1", project="/p", intent="fail deterministically") + t_timeout = TaskRecord(id="to1", project="/p", intent="timed out") + failures = [ + (t_fail, ReplayResult(id="f1", hard=0.0, outcome=OUTCOME_FAIL)), + (t_timeout, ReplayResult(id="to1", hard=0.0, outcome=OUTCOME_INCONCLUSIVE, inconclusive_reason=REASON_TIMEOUT)), + ] + successes = [ + (TaskRecord(id="s1", project="/p", intent="ok"), ReplayResult(id="s1", hard=1.0, outcome=OUTCOME_PASS)), + (TaskRecord(id="s_inc", project="/p", intent="bad"), ReplayResult(id="s_inc", hard=1.0, outcome=OUTCOME_INCONCLUSIVE)), + ] + + backend.reflect( failures, - [], - skill="baseline", - memory="", + successes, + skill="# s", + memory="# m", edit_budget=2, evolve_skill=True, evolve_memory=False, ) - self.assertIsInstance(edits, list) - - def test_aggregate_scores_exclude_inconclusive(self): - task1 = TaskRecord(id="t1", project="p", intent="i1") - task2 = TaskRecord(id="t2", project="p", intent="i2") - task3 = TaskRecord(id="t3", project="p", intent="i3") - - r1 = ReplayResult(id="t1", hard=1.0, soft=1.0, outcome=OUTCOME_PASS) - r2 = ReplayResult(id="t2", hard=0.0, soft=0.0, outcome=OUTCOME_FAIL) - r3 = ReplayResult( - id="t3", - hard=0.0, - soft=0.0, - outcome=OUTCOME_INCONCLUSIVE, - inconclusive_reason=REASON_TIMEOUT, + spy_reflect.assert_called_once() + passed_failures = spy_reflect.call_args.args[0] + passed_successes = spy_reflect.call_args.args[1] + self.assertEqual([t.id for t, _r in passed_failures], ["f1"]) + self.assertEqual([t.id for t, _r in passed_successes], ["s1"]) + + def test_aggregate_and_pairwise_scores(self): + t1 = TaskRecord(id="t1", project="/p", intent="1") + t2 = TaskRecord(id="t2", project="/p", intent="2") + t3 = TaskRecord(id="t3", project="/p", intent="3") + pairs = [ + (t1, ReplayResult(id="t1", hard=1.0, soft=1.0, outcome=OUTCOME_PASS)), + (t2, ReplayResult(id="t2", hard=0.0, soft=0.0, outcome=OUTCOME_FAIL)), + (t3, ReplayResult(id="t3", hard=0.0, soft=0.0, outcome=OUTCOME_INCONCLUSIVE, inconclusive_reason=REASON_INFRA)), + ] + self.assertAlmostEqual(aggregate_scores(pairs)[0], 1 / 3) + self.assertAlmostEqual(aggregate_scores(pairs, exclude_inconclusive=True)[0], 0.5) + + base_pairs = [ + (t1, ReplayResult(id="t1", hard=0.0, soft=0.0, outcome=OUTCOME_FAIL)), + (t2, ReplayResult(id="t2", hard=1.0, soft=1.0, outcome=OUTCOME_PASS)), + (t3, ReplayResult(id="t3", hard=1.0, soft=1.0, outcome=OUTCOME_PASS)), + ] + cand_pairs = [ + (t1, ReplayResult(id="t1", hard=1.0, soft=1.0, outcome=OUTCOME_PASS)), + (t2, ReplayResult(id="t2", hard=1.0, soft=1.0, outcome=OUTCOME_PASS)), + (t3, ReplayResult(id="t3", hard=0.0, soft=0.0, outcome=OUTCOME_INCONCLUSIVE, inconclusive_reason=REASON_BUDGET_EXHAUSTED)), + ] + b_pair, c_pair, n_dec, n_tot, cov = pairwise_aggregate_scores(base_pairs, cand_pairs) + self.assertEqual((n_dec, n_tot), (2, 3)) + self.assertAlmostEqual(cov, 2 / 3) + self.assertAlmostEqual(b_pair[0], 0.5) # baseline scored ONLY on {t1, t2} + self.assertAlmostEqual(c_pair[0], 1.0) # candidate scored ONLY on {t1, t2} + + def test_consolidate_4_task_budget_exhaustion_abstains_pairwise(self): + # Point 8 scenario: max_replays_per_run=5 on 4 val tasks + 1 train task. + # Baseline val gets 4 passes (replays 1..4), train uses replay 5 -> + # candidate trial & final val get inconclusive budget_exhausted. + val_tasks = [TaskRecord(id=f"v{i}", project="/p", intent="solve", split="val") for i in range(4)] + train_task = TaskRecord(id="tr0", project="/p", intent="learn", split="train") + tasks = [train_task, *val_tasks] + + class _ProposedEditBackend(MockBackend): + def reflect( + self, failures, successes, skill, memory, *, + edit_budget=4, evolve_skill=True, evolve_memory=True, + ): + if evolve_skill: + return [EditRecord(target="skill", op="add", content="New rule", rationale="fix")] + return [] + + def _runner(ws_path, _env, ctx): + return {"response": "ok", "actions": 1} + + # Replays 1..4 = baseline val pass; replay 5 = train pass; + # candidate trial runs v0 (replay 6) then v1..v3 budget_exhausted! + be_partial = AgenticReplayBackend( + AgenticReplayConfig(enabled=True, max_replays_per_run=6), + base_backend=_ProposedEditBackend(), + agent_runner=_runner, + default_verifier=TaskVerifier(verifier_fn=lambda _p: (True, "pass")), ) + res = consolidate( + be_partial, + tasks, + skill="# Base", + memory="", + evolve_memory=False, + gate_metric="hard", + min_decidable_coverage=0.5, + ) + self.assertFalse(res.accepted) + self.assertEqual(res.gate_action, "abstain_inconclusive") + self.assertEqual(res.new_skill, "# Base") + self.assertEqual(res.applied_edits, []) + # Crucially: pairwise baseline/candidate never drop to 0.25 due to inconclusive zeros + self.assertGreaterEqual(res.baseline_score, 0.0) + trial_skill = [g for g in res.gate_trials if g["target"] == "skill"][0] + # Candidate trial had v0 pass (replay 6) and v1..v3 budget_exhausted => baseline_score==1.0, candidate_score==1.0, abstained! + self.assertAlmostEqual(trial_skill["baseline_score"], 1.0) + self.assertAlmostEqual(trial_skill["candidate_score"], 1.0) + self.assertTrue(trial_skill["abstained_inconclusive"]) + self.assertAlmostEqual(trial_skill["decidable_coverage"], 0.25) + + def test_consolidate_pairwise_accepts_above_coverage_floor_with_no_regression(self): + val_tasks = [TaskRecord(id=f"v{i}", project="/p", intent=f"val {i}", split="val") for i in range(4)] + train_task = TaskRecord(id="tr0", project="/p", intent="learn", split="train") + tasks = [train_task, *val_tasks] + + class _EditBackend(MockBackend): + def reflect( + self, failures, successes, skill, memory, *, + edit_budget=4, evolve_skill=True, evolve_memory=True, + ): + if evolve_skill: + return [EditRecord(target="skill", op="add", content="Improved rule", rationale="improve v0")] + return [] + + def _batch_side_effect(_backend, batch_tasks, skill, _memory, **_kw): + out = [] + improved = "Improved rule" in skill + for t in batch_tasks: + if t.id == "v0": + out.append((t, ReplayResult(id=t.id, hard=1.0 if improved else 0.0, soft=1.0 if improved else 0.0))) + elif t.id in {"v1", "v2"}: + out.append((t, ReplayResult(id=t.id, hard=1.0, soft=1.0))) + elif t.id == "v3": + # Candidate hits timeout on v3; baseline passed v3. Pairwise excludes v3 from both sides! + if improved: + out.append((t, ReplayResult(id=t.id, hard=0.0, outcome=OUTCOME_INCONCLUSIVE, inconclusive_reason=REASON_TIMEOUT))) + else: + out.append((t, ReplayResult(id=t.id, hard=1.0, soft=1.0))) + else: + out.append((t, ReplayResult(id=t.id, hard=0.0, outcome=OUTCOME_FAIL))) + return out + + with mock.patch("skillopt_sleep.consolidate.replay_batch", side_effect=_batch_side_effect): + res = consolidate( + _EditBackend(), + tasks, + skill="# Base", + memory="", + evolve_memory=False, + gate_metric="hard", + gate_no_regression=True, + min_decidable_coverage=0.5, + ) - pairs = [(task1, r1), (task2, r2), (task3, r3)] - - # Standard aggregation: 1 / 3 = 0.3333 - hard_all, soft_all = aggregate_scores(pairs) - self.assertAlmostEqual(hard_all, 1.0 / 3.0) - - # Excluding inconclusive: 1 / 2 = 0.5 - hard_filtered, soft_filtered = aggregate_scores(pairs, exclude_inconclusive=True) - self.assertAlmostEqual(hard_filtered, 0.5) - - def test_get_backend_agentic(self): - be = get_backend("agentic") - self.assertIsInstance(be, AgenticReplayBackend) - self.assertEqual(be.name, "agentic") - - be2 = get_backend("agentic_replay") - self.assertIsInstance(be2, AgenticReplayBackend) + self.assertTrue(res.accepted) + self.assertEqual(res.gate_action, "accept_new_best") + # Baseline on {v0, v1, v2} = 2/3, Candidate on {v0, v1, v2} = 3/3 (coverage 0.75 >= 0.5) + self.assertAlmostEqual(res.baseline_score, 2 / 3) + self.assertAlmostEqual(res.candidate_score, 1.0) + final_trial = res.gate_trials[-1] + v3_row = [r for r in final_trial["task_deltas"] if r["task_id"] == "v3"][0] + self.assertEqual(v3_row["status"], "inconclusive") if __name__ == "__main__":