diff --git a/CHANGELOG.md b/CHANGELOG.md index ef83812d9..57315ac04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,16 @@ All notable changes to SkillOpt are documented here. This project adheres to search setting. An API key echoed in an error body is redacted before the text reaches `fail_reason`, and the round deadline is checked again after each response. +- **SkillOpt-Sleep `gate_train_check`** config key (default `false`) with + `gate_train_margin` (default `0.1`): a candidate the validation gate would + accept is also replayed on tonight's real training tasks and is rejected + when its mean gate score there is lower than the current documents' by more + than the margin and at least two more of those tasks got worse than got + better. The check only rejects; it never accepts what validation rejected. + It is skipped with the gate off or a non-disjoint validation slice. Both + arms and every task-level change are kept in the gate records (`train_check`) + and a `gate`/`train_check` evidence event, and `report.md` shows the two + means and the counts. With the key unset, records and reports are unchanged. - **SkillOpt-Sleep paired A/B evalkit** (`python -m skillopt_sleep.evalkit`): McNemar plus percentile-bootstrap CIs on a fixed task manifest, with task-cluster multi-seed inference and seeded null calibration for exact-test diff --git a/docs/sleep/README.md b/docs/sleep/README.md index eec3e6cd8..cdca00f3b 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -35,6 +35,43 @@ candidate; an absent numeric score aborts evaluation. Task-level changes are included in `report.md`, `report.json`, `diagnostics.json`, the CLI's `--json` output, and the evidence log. +The optional `gate_train_check` config key adds a condition to an accept. It +defaults to `false`. A night's validation slice is a fraction of that night's +tasks and can be a handful, so a rule that damages the agent can still come out +ahead on it. When the check is on, a candidate the validation gate would accept +is also replayed on tonight's real training tasks (recalled tasks included, +dream variants not). It is rejected when two things hold: its mean score there, +under the configured `gate_metric`, is lower than the current documents' by +more than `gate_train_margin` (default `0.1`, a number between 0 and 1), and at +least two more of those tasks got worse than got better, so one task flipping +can never decide a night. + +These tasks are not held out, since reflection drew on tonight's failures, so +scoring higher on them proves nothing and the check never accepts anything the +validation gate rejected. Scoring clearly lower on them, in particular breaking +tasks the current documents already handle, is not explained by overfitting, +which is why the comparison is used only to reject. Each candidate is compared +with the documents it would replace: a skill candidate with the night's +training replay, a memory candidate with the replay that follows an accepted +skill. The margin therefore applies per candidate, not per night. The added +cost is one replay of the training tasks for each candidate that passes +validation. + +A task with a missing or malformed score counts as worse, and a non-finite +mean rejects the candidate. The check is skipped when `gate_mode` is `off` and +when the validation slice is not disjoint from training. The gate records +(`train_check`, in `report.json`, `diagnostics.json`, and the CLI's `--json` +output) and the evidence log (`gate` / `train_check`) keep both arms and every +task-level change; `report.md` shows the two means and the counts. These +records are written only when the check is enabled. `gate_train_check` must be +a boolean and, when it is `true`, an invalid margin raises before consolidation +makes any model call. + +To measure the gate with and without this key on a public benchmark, run +`python -m skillopt_sleep.experiments.gate_study` (fault injection of fixed rules +and the multi-night recipe, both through `dream_consolidate`). `--offline` runs it +on a committed fixture with no key and no network; live runs need `--live`. + Rule-based heading checks are intentionally additive. `section_present` keeps its legacy strict behavior for existing task sets: an ATX heading cannot append text after the requested name. Use `section_contains` when a numbered, @@ -466,8 +503,9 @@ This is not a hard cap. A step that has started always finishes, so a night can end above the budget by the cost of one step: the first replay of the validation and training tasks, one edit step (a training replay, a reflect call and a validation replay of the candidate; with `dream_rollouts` set to K, -K attempts per training task), the final validation replay, or the held-out -test replay. Each fan-out group that starts runs its own first replay. +K attempts per training task; with `gate_train_check`, a replay of the +training tasks for a candidate that passes validation), the final validation +replay, or the held-out test replay. Each fan-out group that starts runs its own first replay. Spend comes from the backend's token counter, the one the report header prints as `tokens used`. The header shows the backend's running total and the note @@ -504,8 +542,8 @@ This runs one additional consolidation per group (including a catch-all group wh hinted and unhinted evidence are mixed), so it multiplies backend calls and token use; configured dream rollouts and synthetic variants multiply the per-group work too. Each group inherits the configured edit budget, gate mode/metric, -`gate_no_regression`, `dream_rollouts`, `dream_factor`, `recall_k`, and -`evolve_skill`. Recalled archive tasks are restricted to that same skill hint; +`gate_no_regression`, `gate_train_check`, `gate_train_margin`, +`dream_rollouts`, `dream_factor`, `recall_k`, and `evolve_skill`. Recalled archive tasks are restricted to that same skill hint; shared memory is read-only in fan-out runs. Setting `evolve_skill` to `false` therefore disables per-skill proposals as well as the managed skill proposal. diff --git a/docs/sleep/multi-skill-staging.md b/docs/sleep/multi-skill-staging.md index bb86b8169..198180f97 100644 --- a/docs/sleep/multi-skill-staging.md +++ b/docs/sleep/multi-skill-staging.md @@ -39,8 +39,8 @@ document. The managed catch-all remains on `proposed_SKILL.md` and is not duplicated as a per-skill row. Each group inherits `recall_k`, `dream_rollouts`, `dream_factor`, `edit_budget`, -`gate_mode`, `gate_metric`, `gate_mixed_weight`, `gate_no_regression`, and -`evolve_skill`. Recalled archive tasks are restricted to the same skill hint, +`gate_mode`, `gate_metric`, `gate_mixed_weight`, `gate_no_regression`, +`gate_train_check`, `gate_train_margin`, and `evolve_skill`. Recalled archive tasks are restricted to the same skill hint, and shared memory is read-only during group runs. Consequently, `evolve_skill=false` disables managed and fan-out skill proposals. diff --git a/plugins/README.md b/plugins/README.md index b76d7e0f7..6d69c1b75 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -249,10 +249,11 @@ python -m skillopt_sleep run --project "$(pwd)" \ ### Advanced config The JSON/YAML config under `~/.skillopt-sleep/` supports additional engine keys, -including `gate_mode`, `gate_metric`, `gate_no_regression`, `dream_rollouts`, -`dream_factor`, `recall_k`, `evolve_memory`, and `evolve_skill`. These are config -keys, not aliases for the unsupported CLI flags listed above. Shipping defaults -are conservative: `gate_mode="on"`, `gate_no_regression=false`, +including `gate_mode`, `gate_metric`, `gate_no_regression`, `gate_train_check`, +`gate_train_margin`, `dream_rollouts`, `dream_factor`, `recall_k`, +`evolve_memory`, and `evolve_skill`. These are config keys, not aliases for the +unsupported CLI flags listed above. Shipping defaults are conservative: +`gate_mode="on"`, `gate_no_regression=false`, `gate_train_check=false`, `dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`. The managed `schedule` command stores only the project, backend, time, and diff --git a/plugins/claude-code/skills/skillopt-sleep/SKILL.md b/plugins/claude-code/skills/skillopt-sleep/SKILL.md index 0c45aa8f6..b3ed0abc6 100644 --- a/plugins/claude-code/skills/skillopt-sleep/SKILL.md +++ b/plugins/claude-code/skills/skillopt-sleep/SKILL.md @@ -110,6 +110,7 @@ Beyond the CLI flags, advanced behavior is controlled via config: - **`gate_mode`** — `on` (default, validation-gated) or `off` (greedy, accept all edits). - **`gate_metric`** — `hard`, `soft`, or `mixed` (default). Controls how the held-out gate scores. - **`gate_no_regression`** — `false` by default. Set to `true` to reject a candidate when any validation task's configured gate score decreases. +- **`gate_train_check`** — `false` by default. Set to `true` to also reject a candidate that scores clearly lower than the current skill and memory on tonight's training tasks (mean lower by more than `gate_train_margin`, default `0.1`, on at least two more tasks than improved). - **`dream_rollouts`** — >1 enables multi-rollout contrastive reflection per task. - **`recall_k`** — >0 recalls K similar past tasks into the dream (long-term memory). - **`evolve_memory`** / **`evolve_skill`** — independently toggle CLAUDE.md vs SKILL.md consolidation. diff --git a/plugins/codex/README.md b/plugins/codex/README.md index 7d35e8af2..c8e4aa41b 100644 --- a/plugins/codex/README.md +++ b/plugins/codex/README.md @@ -80,7 +80,8 @@ mining, replay, and reflection can be slow and otherwise quiet. Use prefer tasks that match the target skill's path, headings, and content. The implemented main-CLI flags work the same across the shared integrations, and `--preferences "..."` is available for house rules. Advanced keys such as -`gate_mode`, `gate_no_regression`, `dream_rollouts`, and `recall_k` belong in the Sleep config; the +`gate_mode`, `gate_no_regression`, `gate_train_check`, `dream_rollouts`, and `recall_k` belong in +the Sleep config; the nightly CLI does not expose `--gate`, `--rollouts-k`, token/time-budget, or optimizer/target-split flags. See the [shared CLI reference](../README.md#supported-cli-surface). diff --git a/plugins/codex/skills/skillopt-sleep/SKILL.md b/plugins/codex/skills/skillopt-sleep/SKILL.md index ac4a2bccf..8562b1fa5 100644 --- a/plugins/codex/skills/skillopt-sleep/SKILL.md +++ b/plugins/codex/skills/skillopt-sleep/SKILL.md @@ -134,6 +134,7 @@ without `crontab`, `schedule` prints a line for manual installation. - **`gate_mode`** — `on` (validation-gated, default) or `off` (greedy) - **`gate_metric`** — `hard` | `soft` | `mixed` (default) - **`gate_no_regression`** — `false` by default; set it to `true` to reject a candidate when any validation task's gate score decreases +- **`gate_train_check`** — `false` by default; set it to `true` to also reject a candidate that scores clearly lower on tonight's training tasks (mean lower by more than `gate_train_margin`, default `0.1`, on at least two more tasks than improved) - **`dream_rollouts`** — >1 for multi-rollout contrastive reflection - **`recall_k`** — >0 recalls similar past tasks from the archive diff --git a/plugins/copilot/copilot-instructions.snippet.md b/plugins/copilot/copilot-instructions.snippet.md index 3abc5a0df..69a6f60fb 100644 --- a/plugins/copilot/copilot-instructions.snippet.md +++ b/plugins/copilot/copilot-instructions.snippet.md @@ -47,6 +47,8 @@ my preferences", or "make the agent improve from past usage", use the MCP tools: - `gate_mode` — `on` (default) or `off` - `gate_no_regression` — `false` by default; set it to `true` to reject a candidate when any validation task's configured gate score decreases +- `gate_train_check` — `false` by default; set it to `true` to also reject a + candidate that scores clearly lower on tonight's training tasks - `dream_rollouts` — >1 for more signal - `evolve_memory` / `evolve_skill` — toggle which docs consolidate diff --git a/plugins/cursor/skills/skillopt-sleep/SKILL.md b/plugins/cursor/skills/skillopt-sleep/SKILL.md index 71cc664d5..4c330e028 100644 --- a/plugins/cursor/skills/skillopt-sleep/SKILL.md +++ b/plugins/cursor/skills/skillopt-sleep/SKILL.md @@ -173,6 +173,9 @@ commits, or generated summaries. `gate_no_regression` is a config-only safeguard in `~/.skillopt-sleep/config.json`. It defaults to `false`; set it to `true` to reject a candidate when any validation task's configured gate score decreases. +`gate_train_check` is config-only as well. It defaults to `false`; set it to +`true` to also reject a candidate that scores clearly lower on tonight's +training tasks. ## Scheduling diff --git a/plugins/dsh/README.md b/plugins/dsh/README.md index 92f2921e8..7759d1161 100644 --- a/plugins/dsh/README.md +++ b/plugins/dsh/README.md @@ -107,7 +107,7 @@ ask the agent: "Use skillopt_status to check the sleep cycle state." | `timeoutMs` | `600000` | per-call engine timeout in milliseconds | Advanced engine keys (`gate_mode`, `gate_metric`, `gate_no_regression`, -`dream_rollouts`, `recall_k`, `evolve_memory`/`evolve_skill`) go in +`gate_train_check`, `dream_rollouts`, `recall_k`, `evolve_memory`/`evolve_skill`) go in `~/.skillopt-sleep/config.json` — the same file shared by all integrations. ## Data boundary diff --git a/plugins/dsh/README.zh.md b/plugins/dsh/README.zh.md index f80dab047..04b927906 100644 --- a/plugins/dsh/README.zh.md +++ b/plugins/dsh/README.zh.md @@ -25,6 +25,7 @@ "gate_mode": "on", "gate_metric": "mixed", "gate_no_regression": false, + "gate_train_check": false, "dream_rollouts": 1, "recall_k": 0, "evolve_memory": true, diff --git a/plugins/dsh/skills/skillopt-sleep/SKILL.md b/plugins/dsh/skills/skillopt-sleep/SKILL.md index c12ccdd65..095f3499c 100644 --- a/plugins/dsh/skills/skillopt-sleep/SKILL.md +++ b/plugins/dsh/skills/skillopt-sleep/SKILL.md @@ -91,7 +91,7 @@ skillopt_schedule project= hour=3 minute=17 backend= Advanced engine keys go in `~/.skillopt-sleep/config.json`: `gate_mode` (on/off), `gate_metric` (hard/soft/mixed), `gate_no_regression`, -`dream_rollouts`, `recall_k`, `evolve_memory` / `evolve_skill`. +`gate_train_check`, `dream_rollouts`, `recall_k`, `evolve_memory` / `evolve_skill`. ## Hard rules diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index 64814979d..43377b960 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -68,6 +68,8 @@ "gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts) "gate_mixed_weight": 0.5, "gate_no_regression": False, # reject any candidate that lowers a val-task score + "gate_train_check": False, # reject a candidate that is worse on tonight's training tasks + "gate_train_margin": 0.1, # drop in mean training-task score the check tolerates (0..1) "replay_mode": "mock", # report label; fresh-worktree replay is not implemented # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─ "dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py index 0963350a4..45e9d0a2e 100644 --- a/skillopt_sleep/consolidate.py +++ b/skillopt_sleep/consolidate.py @@ -32,6 +32,11 @@ # Its scores were not measured, so callers must not report them as results. BUDGET_EXHAUSTED = "budget_exhausted" +# The training-side check rejects only when at least this many more training +# tasks got worse than got better. One task flipping, in either direction, can +# therefore never decide a night on its own. +TRAIN_CHECK_MIN_NET_REGRESSED = 2 + def _finite_score(value: float) -> float | None: """Return a JSON-safe score, preserving finite values only.""" @@ -205,6 +210,82 @@ def _task_deltas( return out +def train_check_enabled(value: object) -> bool: + """Validate ``gate_train_check``: ``True``, ``False`` or ``None`` (off).""" + if value is None: + return False + if not isinstance(value, bool): + raise ValueError(f"gate_train_check must be true or false, got {value!r}") + return value + + +def train_check_margin(value: object) -> float: + """Validate ``gate_train_margin``: a finite number in ``[0, 1]``.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError( + f"gate_train_margin must be a number between 0 and 1, got {value!r}" + ) + margin = float(value) + if not math.isfinite(margin) or not 0.0 <= margin <= 1.0: + raise ValueError( + f"gate_train_margin must be a number between 0 and 1, got {value!r}" + ) + return margin + + +def _training_check( + tasks: List[TaskRecord], + baseline_pairs: List[Tuple[TaskRecord, ReplayResult]], + candidate_pairs: List[Tuple[TaskRecord, ReplayResult]], + metric: str, + mixed_weight: float, + margin: float, +) -> dict: + """Compare a candidate with the current documents on tonight's training tasks. + + These tasks are not held out: reflection drew on tonight's failures, so a + higher score here cannot rule out overfitting and is never used to accept + anything. A lower score is different. The tasks include the ones the + current documents already handle, and a candidate that breaks those, or + does clearly worse on the failures it was written to fix, is not explained + by overfitting. So the comparison is used only to reject. + + It is baseline-relative and aggregate, and it takes two things to reject: + the candidate's mean gate score on these tasks is lower than the current + documents' by more than ``margin``, and at least + ``TRAIN_CHECK_MIN_NET_REGRESSED`` more tasks got worse than got better. + Both arms and every task-level change are returned so the decision can be + audited. A task with a missing or malformed score counts as regressed, and + a non-finite aggregate rejects. + """ + base_hard, base_soft = aggregate_scores(baseline_pairs) + cand_hard, cand_soft = aggregate_scores(candidate_pairs) + baseline = select_gate_score(base_hard, base_soft, metric, mixed_weight) + candidate = select_gate_score(cand_hard, cand_soft, metric, mixed_weight) + task_deltas = _task_deltas( + tasks, baseline_pairs, candidate_pairs, metric, mixed_weight + ) + n_regressed = sum(1 for row in task_deltas if row["status"] == "regressed") + n_improved = sum(1 for row in task_deltas if row["status"] == "improved") + finite = math.isfinite(baseline) and math.isfinite(candidate) + # The tolerance keeps a drop of exactly the margin on the tolerated side: + # 0.8 - 0.1 is 0.7000000000000001 in floating point. + lower = (not finite) or (baseline - candidate) > margin + 1e-9 + broad = n_regressed - n_improved >= TRAIN_CHECK_MIN_NET_REGRESSED + return { + "n_tasks": len(tasks), + "n_regressed": n_regressed, + "n_improved": n_improved, + "min_net_regressed": TRAIN_CHECK_MIN_NET_REGRESSED, + "margin": margin, + "baseline_score": _finite_score(baseline), + "candidate_score": _finite_score(candidate), + "regressed": lower, + "blocked": bool((not finite) or (lower and broad)), + "task_deltas": task_deltas, + } + + def consolidate( backend: Backend, tasks: List[TaskRecord], @@ -215,6 +296,8 @@ def consolidate( gate_metric: str = "mixed", gate_mixed_weight: float = 0.5, gate_no_regression: bool = False, + gate_train_check: bool = False, # also reject a candidate that is worse on tonight's training tasks + gate_train_margin: float = 0.1, # drop in mean training-task score that is tolerated gate_mode: str = "on", # "on" (hard/soft per gate_metric) | "off" (greedy) rollouts_k: int = 1, # >1 => multi-rollout contrastive reflection evolve_skill: bool = True, @@ -231,6 +314,17 @@ def consolidate( Skill and memory are evolved in sequence (skill first if both enabled). + ``gate_train_check`` adds a condition to an accept. A candidate the + validation gate would accept is also replayed on tonight's real training + tasks (recalled tasks included, dream variants not), and is rejected when + its mean gate score there is lower than the current documents' by more + than ``gate_train_margin`` and at least two more of those tasks got worse + than got better. It never accepts anything the validation gate rejected. + Each candidate is compared with the documents it would replace, so a + memory candidate is compared with the skill accepted earlier that night. + It is skipped when the gate is off and when the validation slice is not + disjoint from training. + ``budget_exhausted`` is checked before the epoch starts, before each evolve target and before the final validation replay. Once it returns True, the steps that have not started are skipped and listed in @@ -243,6 +337,8 @@ def consolidate( ev = evlog.get(backend) train_tasks, val_tasks, holdout_leaked = _split(tasks) gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"} + train_check_on = train_check_enabled(gate_train_check) + train_margin = train_check_margin(gate_train_margin) if train_check_on else 0.0 holdout_detail: List[dict] = [] budget_skipped: List[str] = [] @@ -301,6 +397,14 @@ def _budget_spent(step: str) -> bool: all_unmatched: List[EditRecord] = [] gate_trials: List[dict] = [] current_pairs = base_pairs + # Training-side check: tonight's real training tasks (recalled ones + # included). Dream variants restate them and are left out. The current + # documents' scores on these tasks are the replay that just ran. + check_tasks: List[TaskRecord] = [] + current_train_pairs: List[Tuple[TaskRecord, ReplayResult]] = [] + if train_check_on and not gate_off and not holdout_leaked: + check_tasks = [t for t in train_tasks if t.origin != "dream"] + current_train_pairs = [(t, r) for (t, r) in train_pairs if t.origin != "dream"] def _edits_payload(edits: List[EditRecord]) -> List[dict]: return [{"op": e.op, "content": e.content, "anchor": e.anchor, @@ -308,7 +412,7 @@ def _edits_payload(edits: List[EditRecord]) -> List[dict]: def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: nonlocal cand_skill, cand_memory, base_score, current_pairs - nonlocal all_applied, all_rejected + nonlocal all_applied, all_rejected, current_train_pairs if ev is not None: ev.log("reflect", "edits_returned", target=which, n_edits=len(edits), edits=_edits_payload(edits)) @@ -345,24 +449,52 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: ) trial_base_score = base_score improved = cand_score > base_score and not blocked_by_regression - gate_trials.append({ + # Only a candidate the validation gate would accept is worth the extra + # replay, and the check can only turn that accept into a reject. + train_check: Optional[dict] = None + train_cand_pairs: List[Tuple[TaskRecord, ReplayResult]] = [] + if improved and check_tasks: + evlog.set_phase(backend, f"gate_train_check:{which}") + train_cand_pairs = replay_batch( + backend, check_tasks, trial_skill, trial_memory + ) + train_check = _training_check( + check_tasks, current_train_pairs, train_cand_pairs, + gate_metric, gate_mixed_weight, train_margin, + ) + if ev is not None: + ev.log("gate", "train_check", target=which, **train_check) + if train_check["blocked"]: + improved = False + blocked_by_train_check = bool(train_check and train_check["blocked"]) + trial_record = { "target": which, "baseline_score": _finite_score(trial_base_score), "candidate_score": _finite_score(cand_score), "accepted": improved, "blocked_by_regression": blocked_by_regression, "task_deltas": task_deltas, - }) + } + trial_event = {} + if train_check_on: + # Present only when the check is enabled, so a night without it + # writes the records it always wrote. + trial_record["blocked_by_train_check"] = blocked_by_train_check + trial_record["train_check"] = train_check + trial_event["blocked_by_train_check"] = blocked_by_train_check + gate_trials.append(trial_record) if ev is not None: ev.log("gate", "trial", target=which, mode="gated", baseline_score=trial_base_score, cand_hard=h, cand_soft=s, cand_score=cand_score, accepted=improved, blocked_by_regression=blocked_by_regression, task_deltas=task_deltas, - n_edits=len(applied)) + n_edits=len(applied), **trial_event) if improved: base_score = cand_score current_pairs = pairs + if train_check is not None: + current_train_pairs = train_cand_pairs all_applied.extend(applied) return new_doc all_rejected.extend(applied) @@ -425,6 +557,12 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: # re-evaluate failures under the (possibly improved) skill evlog.set_phase(backend, "train_post_skill") train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory) + if check_tasks: + # The memory candidate is compared with this replay of the current + # documents, not with the draw that passed the skill's own check. + current_train_pairs = [ + (t, r) for (t, r) in train_pairs2 if t.origin != "dream" + ] failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0] successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0] edits_m = backend.reflect( @@ -519,14 +657,19 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: if edit not in all_rejected: all_rejected.append(edit) all_applied = [] - gate_trials.append({ + final_record = { "target": "final", "baseline_score": _finite_score(base_gate_score), "candidate_score": _finite_score(final_score), "accepted": accepted, "blocked_by_regression": final_blocked_by_regression, "task_deltas": final_task_deltas, - }) + } + if train_check_on: + # The training-side check belongs to the candidate trials above. + final_record["blocked_by_train_check"] = False + final_record["train_check"] = None + gate_trials.append(final_record) if ev is not None: w = max(0.0, min(1.0, float(gate_mixed_weight))) diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index b3711f7d8..2a014d274 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -24,7 +24,11 @@ from skillopt_sleep.backend import Backend, CursorBackendError, build_backend from skillopt_sleep.budget import Budget from skillopt_sleep.config import DEFAULTS, SleepConfig, load_config -from skillopt_sleep.consolidate import BUDGET_EXHAUSTED +from skillopt_sleep.consolidate import ( + BUDGET_EXHAUSTED, + train_check_enabled, + train_check_margin, +) from skillopt_sleep.dream import dream_consolidate from skillopt_sleep.evidence import EvidenceLog from skillopt_sleep.handoff_backend import PendingCalls @@ -118,6 +122,20 @@ def _night_token_budget(cfg: SleepConfig) -> Budget: return Budget(max_tokens=limit or None) +def _gate_train_check(cfg: SleepConfig) -> tuple[bool, float]: + """Return ``(gate_train_check, gate_train_margin)``. + + Raises ``ValueError`` when ``gate_train_check`` is not a boolean, or when + it is enabled and the margin is not a finite number between 0 and 1. A + null margin means the default. + """ + enabled = train_check_enabled(cfg.get("gate_train_check", False)) + raw = cfg.get("gate_train_margin", 0.1) + if not enabled: + return False, 0.1 + return True, train_check_margin(0.1 if raw is None else raw) + + # ── Model-swap detection (F16) ─────────────────────────────── def _make_model_key(cfg: SleepConfig) -> str: """Stable string identifying the backend object(s) actually used. @@ -384,6 +402,11 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: f"- tokens used: {report.tokens_used}", "", ] + train_check, train_margin = _gate_train_check(cfg) + if train_check and str(cfg.get("gate_mode", "on")).strip().lower() not in { + "off", "none", "false", "greedy", + }: + lines.insert(-1, f"- training-side check: enabled (margin {train_margin})") gate_on = str(cfg.get("gate_mode", "on")).strip().lower() not in { "off", "none", "false", "greedy", } @@ -411,6 +434,8 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: decision = "accepted" if accepted else "rejected" if blocked: decision += " (task regression)" + if trial.get("blocked_by_train_check", False): + decision += " (training-task regression)" baseline = _report_score(trial.get("baseline_score")) candidate = _report_score(trial.get("candidate_score")) lines.append( @@ -433,6 +458,24 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: f"| {candidate_score} | {status} |" ) lines.append("") + check = trial.get("train_check") + if isinstance(check, dict): + if check.get("blocked"): + verdict = "regressed, candidate rejected" + elif check.get("regressed"): + verdict = "lower, but not on enough tasks to reject" + else: + verdict = "no regression" + lines.append( + "Training-side check (these tasks are not held out) over " + f"{check.get('n_tasks')} training tasks: " + f"{_report_score(check.get('baseline_score'))} → " + f"{_report_score(check.get('candidate_score'))} " + f"(margin {check.get('margin')}; " + f"{check.get('n_regressed')} worse, " + f"{check.get('n_improved')} better): {verdict}." + ) + lines.append("") if any((report.edits, report.rejected_edits, report.unmatched_edits)): lines.append( "_Edit source IDs list the task evidence supplied to reflection; " @@ -723,6 +766,7 @@ def run_sleep_cycle( # Checked before the night writes anything (state, staging dir, evidence). harvest_source = normalize_transcript_source(cfg.get("transcript_source")) budget = _night_token_budget(cfg) + train_check, train_margin = _gate_train_check(cfg) state = SleepState.load(cfg.state_path) project = _project_paths(cfg) @@ -805,6 +849,9 @@ def budget_exhausted() -> bool: cycle_config["opencode_tool_replay"] = ( cfg.get("opencode_tool_replay", False) is True ) + if train_check: + cycle_config["gate_train_check"] = True + cycle_config["gate_train_margin"] = train_margin ev.log("cycle", "start", night=night, project=project, backend=backend.name, model=cfg.get("model", ""), config=cycle_config) @@ -976,6 +1023,8 @@ def budget_exhausted() -> bool: gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), gate_no_regression=cfg.get("gate_no_regression", False), + gate_train_check=train_check, + gate_train_margin=train_margin, gate_mode=cfg.get("gate_mode", "on"), evolve_skill=cfg.get("evolve_skill", True), evolve_memory=cfg.get("evolve_memory", True), @@ -1082,6 +1131,8 @@ def budget_exhausted() -> bool: gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), gate_no_regression=cfg.get("gate_no_regression", False), + gate_train_check=train_check, + gate_train_margin=train_margin, gate_mode=cfg.get("gate_mode", "on"), evolve_skill=cfg.get("evolve_skill", True), night=night, diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py index 5621cc3be..dbc7ac29b 100644 --- a/skillopt_sleep/dream.py +++ b/skillopt_sleep/dream.py @@ -141,6 +141,8 @@ def dream_consolidate( gate_metric: str = "mixed", gate_mixed_weight: float = 0.5, gate_no_regression: bool = False, + gate_train_check: bool = False, + gate_train_margin: float = 0.1, gate_mode: str = "on", evolve_skill: bool = True, evolve_memory: bool = True, @@ -180,7 +182,9 @@ def dream_consolidate( backend, enlarged, skill, memory, edit_budget=edit_budget, gate_metric=gate_metric, gate_mixed_weight=gate_mixed_weight, - gate_no_regression=gate_no_regression, gate_mode=gate_mode, + gate_no_regression=gate_no_regression, + gate_train_check=gate_train_check, gate_train_margin=gate_train_margin, + gate_mode=gate_mode, rollouts_k=dream_rollouts, evolve_skill=evolve_skill, evolve_memory=evolve_memory, night=night, budget_exhausted=budget_exhausted, diff --git a/skillopt_sleep/experiments/gate_study.py b/skillopt_sleep/experiments/gate_study.py new file mode 100644 index 000000000..3ed06989d --- /dev/null +++ b/skillopt_sleep/experiments/gate_study.py @@ -0,0 +1,548 @@ +"""SkillOpt-Sleep: measure the nightly gate on a public benchmark. + +Two experiments, both through the shipped ``dream_consolidate``: + +``faults`` A fixed rule whose held-out effect is known is proposed in place of + the optimizer's reflection, in many nightly contexts, and the gate + decides. Reports how often each rule is accepted. +``nights`` The multi-night deployment protocol of docs/sleep/RESULTS.md with + the held-out test items scored after every night. Reports the + learning curve. + +Run an arm with a feature off and again with it on (``--feature`` is passed to +``dream_consolidate``). Arms share every model draw they have in common +through the response store, so they differ only by the feature, and a run can +be resumed. + +Protocol: each seed shuffles the benchmark's training pool; night k gets items +10(k-1)..10k-1, split by ``assign_splits(val_fraction=0.34, seed)``; the skill +starts as the benchmark's shipped initial skill; gate metric ``mixed``. +``faults`` uses one rollout, no dream variants, ``recall_k=10`` and an archive +made of the seed's earlier nights. ``nights`` uses ``dream_rollouts=5``, +``dream_factor=2`` and ``recall_k=10``. Held-out scores use the test split. + +Benchmarks: ``searchqa`` (400 training items, exact match) and ``livemath`` +(LiveMathematicianBench, 52 training items from the released train and val +manifests, multiple-choice label). ``fixture`` is a tiny offline set. + +Offline check, no key and no network (this is what the test suite runs): + python -m skillopt_sleep.experiments.gate_study faults --offline --arm off + python -m skillopt_sleep.experiments.gate_study faults --offline --arm check \ + --feature '{"gate_train_check": true}' + python -m skillopt_sleep.experiments.gate_study report --offline + +Live runs spend money on your own key, so they need ``--live`` and print the +number of model calls they expect before the first one: + python -m pip install -e ".[searchqa]" openai + python scripts/materialize_searchqa.py + export OPENAI_API_KEY=... # any OpenAI-compatible endpoint + export OPENAI_BASE_URL=... # optional + python -m skillopt_sleep.experiments.gate_study effects --live + python -m skillopt_sleep.experiments.gate_study faults --live --arm off + python -m skillopt_sleep.experiments.gate_study faults --live --arm check \ + --feature '{"gate_train_check": true}' + python -m skillopt_sleep.experiments.gate_study nights --live --arm base + python -m skillopt_sleep.experiments.gate_study report +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import sqlite3 +import sys +import threading +import time +import types +from collections import defaultdict +from pathlib import Path +from typing import Callable, Dict, List, Optional, Sequence, Tuple + +REPO = Path(__file__).resolve().parents[2] +FIXTURE = REPO / "tests" / "fixtures" / "gate_study" / "items.json" +LIVEMATH_REPO = "LiveMathematicianBench/LiveMathematicianBench" + +RULES: Dict[str, List[Tuple[str, str]]] = { + "searchqa": [ + ("H1 title verbatim", "Answer with the title of the first document in the context, verbatim."), + ("H2 full sentence", "Always answer in one complete sentence that restates the question."), + ("H3 list alternatives", "When several names could apply, list every plausible alternative separated by commas."), + ("H4 category suffix", "Always add the category of the answer in parentheses after it, for example: Paris (city)."), + ("H5 formal name", "Prefer the most formal, complete official name of the answer, for example United States of America rather than USA."), + ("H6 placebo", "Be careful and precise."), + ], + "livemath": [ + ("M1 last option", "Always choose the last option listed."), + ("M2 weakest statement", "Always choose the weakest, most conservative statement among the options."), + ("M3 no reasoning", "Do not reason step by step. Output the answer tag immediately."), + ("M4 longest option", "When two options look similar, choose the longer one."), + ("M5 restate hypotheses", "Before answering, restate the hypotheses of the question in one line."), + ("M6 placebo", "Be careful and precise."), + ], + "fixture": [ + ("F1 harmful", "FIXTURE-RULE-HARMFUL"), + ("F2 helpful", "FIXTURE-RULE-HELPFUL"), + ("F3 placebo", "FIXTURE-RULE-PLACEBO"), + ], +} + + +class Store: + """SQLite response store and result tables. First writer wins, so a + parallel run stays replayable and a second arm reuses the first arm's draws.""" + + def __init__(self, path: str) -> None: + self.lock = threading.Lock() + self.db = sqlite3.connect(path, check_same_thread=False, timeout=120) + self.db.execute("PRAGMA journal_mode=WAL") + for ddl in ( + "calls (k TEXT PRIMARY KEY, model TEXT, response TEXT, tin INTEGER, tout INTEGER, ts REAL)", + "decisions (arm TEXT, model TEXT, seed INTEGER, night INTEGER, rule TEXT, accepted INTEGER, " + "calls INTEGER, PRIMARY KEY (arm, model, seed, night, rule))", + "effects (model TEXT, rule TEXT, em REAL, PRIMARY KEY (model, rule))", + "curves (arm TEXT, model TEXT, seed INTEGER, night INTEGER, accepted INTEGER, em REAL, " + "calls INTEGER, PRIMARY KEY (arm, model, seed, night))", + ): + self.db.execute(f"CREATE TABLE IF NOT EXISTS {ddl}") + self.db.commit() + + def get(self, key: str): + with self.lock: + return self.db.execute("SELECT response FROM calls WHERE k=?", (key,)).fetchone() + + def put(self, key: str, model: str, text: str, tin: int, tout: int) -> str: + with self.lock: + self.db.execute("INSERT OR IGNORE INTO calls VALUES (?,?,?,?,?,?)", + (key, model, text, tin, tout, time.time())) + self.db.commit() + return self.db.execute("SELECT response FROM calls WHERE k=?", (key,)).fetchone()[0] + + def run(self, sql: str, args: Sequence = ()): + with self.lock: + rows = self.db.execute(sql, args).fetchall() + self.db.commit() + return rows + + +def make_client(): + """The OpenAI-compatible client for live runs (key and base URL from the + environment). A module-level factory so a caller can wrap it.""" + from openai import OpenAI + + return OpenAI(max_retries=2, timeout=180) + + +def fixture_answer(prompt: str) -> str: + """Deterministic stand-in for a model, driven by the fixture file: each item + says whether the starting skill answers it and which rules flip that.""" + for item in json.loads(FIXTURE.read_text()): + if f"## Question\n{item['question']}\n" not in prompt + "\n": + continue + correct = bool(item["base_correct"]) + for _name, rule in RULES["fixture"]: + if rule in prompt and rule in item.get("flips", []): + correct = not correct + return f"{item['answers'][0] if correct else 'no idea'}" + return "no idea" + + +def chat_backend(model: str, store: Store, *, offline: bool = False): + from skillopt_sleep.backend import CliBackend + + client = None if offline else make_client() + key_locks: Dict[str, threading.Lock] = {} + key_locks_guard = threading.Lock() + + class ChatBackend(CliBackend): + name = "openai-compatible" + generation_tools_disabled = True + + def __init__(self) -> None: + super().__init__(model=model, timeout=180) + self.tag = "test" + self.calls = 0 + self._sample = threading.local() + + def start(self, tag: str) -> None: + """Draws are tagged per phase: a night must not reuse another night's draw.""" + self.tag = tag + with self._lock: + self._cache.clear() + + def attempt(self, task, skill, memory, sample_id: int = 0): + self._sample.value = f"s{sample_id}" + try: + return super().attempt(task, skill, memory, sample_id=sample_id) + finally: + self._sample.value = "" + + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + self.calls += 1 + salt = f"{self.tag}:" + (getattr(self._sample, "value", "") or "x") + key = hashlib.sha256(f"{model}\0{salt}\0{prompt}".encode()).hexdigest() + with key_locks_guard: # two workers on one prompt pay for it once + key_lock = key_locks.setdefault(key, threading.Lock()) + with key_lock: + hit = store.get(key) + if hit is not None and hit[0]: + return hit[0] + if offline: + return store.put(key, model, fixture_answer(prompt), 0, 0) + for attempt in range(8): + try: + resp = client.chat.completions.create( + model=model, messages=[{"role": "user", "content": prompt}], + max_completion_tokens=16384) + text = (resp.choices[0].message.content or "").strip() + if text: + return store.put(key, model, text, resp.usage.prompt_tokens, + resp.usage.completion_tokens) + except RuntimeError: + raise # a caller's spend limit; do not retry past it + except Exception: # noqa: BLE001 - retried, then raised below + pass + time.sleep(min(60.0, 2.0 ** attempt)) + raise RuntimeError("provider call failed; rerun to resume from the store") + + return ChatBackend() + + +def install_judge() -> None: + """The shipped backend scores ``reference_kind == 'answer'`` tasks through + this hook; wire it to the in-tree benchmark evaluators.""" + from skillopt.envs.livemathematicianbench.evaluator import evaluate as livemath_eval + from skillopt.envs.searchqa.evaluator import evaluate as searchqa_eval + + module = types.ModuleType("skillopt_sleep.experiments.real_eval") + + def score_answer_judge(judge, response): + if judge.get("benchmark") == "livemath": + r = livemath_eval(response or "", dict(judge["correct_choice"]), list(judge["choices"])) + else: + r = searchqa_eval(response or "", list(judge.get("answers") or [])) + return float(r["em"]), float(r["f1"]), f"em={r['em']:.0f} f1={r['f1']:.2f}" + + module.score_answer_judge = score_answer_judge + sys.modules["skillopt_sleep.experiments.real_eval"] = module + + +def materialize_livemath(repo: Path) -> None: + """Write runnable LiveMathematicianBench splits from the released ID manifest + (raw monthly files fetched at the manifest's pinned revision). Choices are + shuffled per item with the dataloader's own rule, seed 42.""" + from huggingface_hub import hf_hub_download + + from skillopt.envs.livemathematicianbench.dataloader import ( + LiveMathematicianBenchDataLoader as Loader, + ) + from skillopt.envs.livemathematicianbench.dataloader import load_items + + manifest_dir = repo / "data" / "livemathematicianbench_id_split" + manifest = json.loads((manifest_dir / "split_manifest.json").read_text()) + by_id = {} + for name in manifest["source_files"]: + path = hf_hub_download(LIVEMATH_REPO, name, repo_type="dataset", + revision=manifest["source_revision"]) + month = name.split("/")[1] + for item in load_items(path): + item_id = item["id"] if ":" in str(item["id"]) else f"{month}:{item['no']}" + by_id[item_id] = {**item, "id": item_id, "month": month} + loader = Loader(shuffle_choices=True) + for split in ("train", "val", "test"): + wanted = [row["id"] for row in json.loads((manifest_dir / split / "items.json").read_text())] + missing = [i for i in wanted if i not in by_id] + if missing: + raise SystemExit(f"livemath: {len(missing)} manifest ids not in the raw files, e.g. {missing[:3]}") + items = [loader._shuffle_item_choices(by_id[i], 42) for i in wanted] + out = repo / "data" / "livemathematicianbench_split" / split + out.mkdir(parents=True, exist_ok=True) + (out / "items.json").write_text(json.dumps(items, indent=1)) + print(f"livemath {split}: {len(items)} items -> {out / 'items.json'}") + + +def benchmark(name: str, repo: Path): + """Return (task builder, items(split), starting skill) for a benchmark.""" + from skillopt_sleep.memory import ensure_skill_scaffold + from skillopt_sleep.types import TaskRecord + + env = {"searchqa": "searchqa", "fixture": "searchqa", "livemath": "livemathematicianbench"}[name] + system = (repo / f"skillopt/envs/{env}/prompts/rollout_system.md").read_text() + initial = (repo / f"skillopt/envs/{env}/skills/initial.md").read_text() + + def task(item): + if name == "livemath": + choices = "\n".join(f"{c['label']}. {c['text']}" for c in item["choices"]) + body, judge = f"## Choices\n{choices}", { + "benchmark": "livemath", "correct_choice": item["correct_choice"], + "choices": item["choices"]} + else: + body, judge = f"## Context\n{item['context']}", { + "benchmark": "searchqa", "answers": list(item["answers"])} + return TaskRecord( + id=f"{name}:{item['id']}", project=f"bench/{name}", + intent=f"## Question\n{item['question']}", context_excerpt=body, system=system, + reference_kind="answer", judge=judge, tags=[name], split="train", origin="real") + + def items(split): + if name == "fixture": + rows = json.loads(FIXTURE.read_text()) + return rows[:20] if split == "train" else rows[20:] + folder = {"searchqa": "searchqa_split", "livemath": "livemathematicianbench_split"}[name] + path = repo / "data" / folder + if not (path / "test" / "items.json").exists(): + raise SystemExit(f"{path} is missing; run the materialize step in this module's docstring") + rows = json.loads((path / split / "items.json").read_text()) + if name == "livemath" and split == "train": # 35 + 17: the manifest's val is unused here + rows = rows + json.loads((path / "val" / "items.json").read_text()) + return rows + + skill = ensure_skill_scaffold(initial, name="skillopt-sleep-learned", + description="Learned preferences.") + return task, items, skill + + +def held_out_em(backend, tests, skill) -> float: + from skillopt_sleep.replay import replay_batch + + pairs = replay_batch(backend, tests, skill, "") + return sum(r.hard for _t, r in pairs) / len(pairs) + + +def expected_calls(args, n_rules: int, n_test: int) -> int: + """An upper estimate of model calls, printed before a live run starts.""" + contexts = args.seeds * args.nights + if args.command == "effects": + return n_test * (1 + n_rules) + if args.command == "faults": + return contexts * n_rules * 22 + return contexts * 225 + args.seeds * (args.nights + 1) * n_test + + +def run_effects(args, store: Store) -> None: + from skillopt_sleep.memory import apply_edits_detailed + from skillopt_sleep.types import EditRecord + + backend = chat_backend(args.model, store, offline=args.offline) + task, items, skill = benchmark(args.benchmark, Path(args.repo).resolve()) + tests = [task(i) for i in items("test")[: args.test_n]] + backend.start("test") + candidates = [("(starting skill)", skill)] + for name, rule in RULES[args.benchmark]: + edit = EditRecord(target="skill", op="add", content=rule) + candidates.append((name, apply_edits_detailed(skill, [edit])[0])) + for name, text in candidates: + em = held_out_em(backend, tests, text) + store.run("INSERT OR REPLACE INTO effects VALUES (?,?,?)", (args.model, name, em)) + print(f"{name:<22} held-out EM {100 * em:5.1f} (n={len(tests)})", flush=True) + + +def run_faults(args, store: Store) -> None: + from skillopt_sleep.dream import dream_consolidate + from skillopt_sleep.mine import assign_splits + from skillopt_sleep.types import EditRecord + + backend = chat_backend(args.model, store, offline=args.offline) + task, items, skill = benchmark(args.benchmark, Path(args.repo).resolve()) + feature = json.loads(args.feature or "{}") + for seed in range(args.first_seed, args.first_seed + args.seeds): + order = items("train") + random.Random(seed).shuffle(order) + archive: list = [] + for night in range(1, args.nights + 1): + tasks = [task(i) for i in order[(night - 1) * 10: night * 10]] + assign_splits(tasks, val_fraction=0.34, seed=seed) + for name, rule in RULES[args.benchmark]: + if store.run("SELECT 1 FROM decisions WHERE arm=? AND model=? AND seed=? AND night=? " + "AND rule=?", (args.arm, args.model, seed, night, name)): + continue + proposal = [EditRecord(target="skill", op="add", content=rule, rationale="injected")] + backend.reflect = lambda *a, _p=proposal, **k: list(_p) + backend.start(f"inj{night}") + before = backend.calls + result = dream_consolidate( + backend, tasks, skill, "", history_tasks=list(archive), recall_k=10, + dream_rollouts=1, dream_factor=0, edit_budget=4, gate_metric="mixed", + gate_mixed_weight=0.5, gate_mode="on", evolve_skill=True, + evolve_memory=False, night=night, **feature) + store.run("INSERT OR REPLACE INTO decisions VALUES (?,?,?,?,?,?,?)", + (args.arm, args.model, seed, night, name, int(result.accepted), + backend.calls - before)) + archive.extend(tasks) + print(f"faults arm {args.arm} seed {seed} done", flush=True) + + +def run_nights(args, store: Store) -> None: + from skillopt_sleep.backend import DualBackend + from skillopt_sleep.dream import dream_consolidate + from skillopt_sleep.mine import assign_splits + + target = chat_backend(args.model, store, offline=args.offline) + optimizer = chat_backend(args.optimizer, store, offline=args.offline) + backend = DualBackend(target, optimizer) + task, items, initial = benchmark(args.benchmark, Path(args.repo).resolve()) + tests = [task(i) for i in items("test")[: args.test_n]] + feature = json.loads(args.feature or "{}") + + def start(tag: str) -> None: + target.start(tag) + optimizer.start(tag) + + for seed in range(args.first_seed, args.first_seed + args.seeds): + if store.run("SELECT 1 FROM curves WHERE arm=? AND model=? AND seed=? AND night=?", + (args.arm, args.model, seed, args.nights)): + continue + order = items("train") + random.Random(seed).shuffle(order) + skill, archive, rows = initial, [], [] + start("test") + rows.append((0, 0, held_out_em(backend, tests, skill), 0)) + for night in range(1, args.nights + 1): + tasks = [task(i) for i in order[(night - 1) * 10: night * 10]] + assign_splits(tasks, val_fraction=0.34, seed=seed) + start(f"n{night}") + before = target.calls + result = dream_consolidate( + backend, tasks, skill, "", history_tasks=list(archive), recall_k=10, + dream_rollouts=5, dream_factor=2, edit_budget=4, gate_metric="mixed", + gate_mixed_weight=0.5, gate_mode="on", evolve_skill=True, + evolve_memory=False, night=night, **feature) + spent = target.calls - before + if result.accepted: + skill = result.new_skill + archive.extend(tasks) + start("test") + rows.append((night, int(result.accepted), held_out_em(backend, tests, skill), spent)) + for night, accepted, em, calls in rows: + store.run("INSERT OR REPLACE INTO curves VALUES (?,?,?,?,?,?,?)", + (args.arm, args.model, seed, night, accepted, em, calls)) + print(f"nights arm {args.arm} seed {seed}: " + + " ".join(f"{100 * em:.1f}" for _n, _a, em, _c in rows), flush=True) + + +def summarize(store: Store, model: str, rules: Sequence[Tuple[str, str]]) -> dict: + """Every number ``report`` prints, as data (the test suite pins this).""" + effects = dict(store.run("SELECT rule, em FROM effects WHERE model=?", (model,))) + base = effects.get("(starting skill)") + table: dict = defaultdict(lambda: defaultdict(list)) + for arm, rule, accepted, calls in store.run( + "SELECT arm, rule, accepted, calls FROM decisions WHERE model=?", (model,)): + table[rule][arm].append((accepted, calls)) + arms = sorted({arm for rule in table for arm in table[rule]}) + faults = { + name: { + "effect": (None if base is None or name not in effects + else round(100 * (effects[name] - base), 1)), + "arms": {arm: {"accepted": sum(a for a, _c in table[name][arm]), + "decisions": len(table[name][arm])} + for arm in arms if table[name][arm]}, + } + for name, _rule in rules + } + calls = {} + for arm in arms: + got = [x for name, _r in rules for x in table[name][arm]] + calls[arm] = round(sum(c for _a, c in got) / max(1, len(got)), 1) + by: dict = defaultdict(lambda: defaultdict(list)) + for arm, seed, night, accepted, em, n in store.run( + "SELECT arm, seed, night, accepted, em, calls FROM curves WHERE model=? " + "ORDER BY arm, seed, night", (model,)): + by[arm][seed].append((night, accepted, em, n)) + curves = {} + for arm in sorted(by): + seeds = sorted(by[arm]) + nights = len(by[arm][seeds[0]]) + curve = [sum(100 * by[arm][s][k][2] for s in seeds) / len(seeds) for k in range(nights)] + first = [next((n for n, a, _e, _c in by[arm][s] if a), nights) for s in seeds] + curves[arm] = { + "seeds": len(seeds), + "curve": [round(v, 1) for v in curve], + "accepted_nights": sum(a for s in seeds for _n, a, _e, _c in by[arm][s]), + "first_accepted_night": round(sum(first) / len(first), 2), + "target_calls_per_night": round( + sum(c for s in seeds for _n, _a, _e, c in by[arm][s][1:]) + / max(1, len(seeds) * (nights - 1))), + } + return {"model": model, "starting_em": None if base is None else round(100 * base, 1), + "faults": faults, "calls_per_decision": calls, "curves": curves} + + +def report(args, store: Store) -> None: + data = summarize(store, args.model, RULES[args.benchmark]) + if args.json: + print(json.dumps(data, indent=1)) + return + arms = sorted(data["calls_per_decision"]) + if arms: + print(f"FAULT INJECTION, model {data['model']}; starting skill held-out EM " + f"{'n/a' if data['starting_em'] is None else data['starting_em']}") + print(f"{'rule':<22} {'held-out effect':>15} " + " ".join(f"{a:>16}" for a in arms)) + for name, row in data["faults"].items(): + effect = "n/a" if row["effect"] is None else f"{row['effect']:+.1f}" + cells = [(f"{row['arms'][a]['accepted']:>3}/{row['arms'][a]['decisions']:<3} accepted".rjust(16) + if a in row["arms"] else " " * 16) for a in arms] + print(f"{name:<22} {effect:>15} " + " ".join(cells)) + for arm in arms: + total = sum(row["arms"][arm]["decisions"] for row in data["faults"].values() + if arm in row["arms"]) + print(f"arm {arm}: {total} decisions, " + f"{data['calls_per_decision'][arm]:.1f} model calls per decision") + if data["curves"]: + print(f"\nLEARNING CURVES, target {data['model']}: held-out EM before night 1 and after each night") + for arm, c in data["curves"].items(): + print(f"arm {arm:<8} seeds {c['seeds']:>2} curve {' '.join(f'{v:5.1f}' for v in c['curve'])} " + f"accepted nights {c['accepted_nights']} " + f"first accepted night {c['first_accepted_night']:.2f} " + f"target calls/night {c['target_calls_per_night']}") + + +def main(argv: Optional[Sequence[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("command", choices=["materialize", "effects", "faults", "nights", "report"]) + ap.add_argument("--benchmark", default="searchqa", choices=sorted(RULES)) + ap.add_argument("--repo", default=str(REPO)) + ap.add_argument("--model", default="gpt-5.4-nano", help="the agent being improved") + ap.add_argument("--optimizer", default="gpt-5.5", help="reflection model for `nights`") + ap.add_argument("--arm", default="base") + ap.add_argument("--feature", default="{}", help="JSON kwargs for dream_consolidate") + ap.add_argument("--first-seed", type=int, default=101) + ap.add_argument("--seeds", type=int, default=20) + ap.add_argument("--nights", type=int, default=5) + ap.add_argument("--test-n", type=int, default=400) + ap.add_argument("--workers", type=int, default=16) + ap.add_argument("--store", default="sleep_gate_study.sqlite") + ap.add_argument("--live", action="store_true", help="allow paid model calls on your own key") + ap.add_argument("--offline", action="store_true", + help="use the committed fixture and a deterministic stand-in model") + ap.add_argument("--json", action="store_true", help="`report` as JSON") + args = ap.parse_args(argv) + if args.offline: + args.benchmark, args.model, args.optimizer = "fixture", "fixture-model", "fixture-model" + args.seeds, args.nights = min(args.seeds, 8), min(args.nights, 2) + os.environ["SKILLOPT_SLEEP_WORKERS"] = str(args.workers) + if args.command == "materialize": + if args.benchmark != "livemath": + raise SystemExit("searchqa: run scripts/materialize_searchqa.py") + materialize_livemath(Path(args.repo).resolve()) + return 0 + store = Store(args.store) + if args.command == "report": + report(args, store) + return 0 + if not (args.live or args.offline): + raise SystemExit("this command calls a model: pass --live (your key, your cost) or --offline") + install_judge() + if args.live: + _t, items, _s = benchmark(args.benchmark, Path(args.repo).resolve()) + n = expected_calls(args, len(RULES[args.benchmark]), len(items("test")[: args.test_n])) + print(f"live run: up to about {n} model calls on {args.model} " + f"(fewer when the store already holds them)", flush=True) + runner: Callable = {"effects": run_effects, "faults": run_faults, "nights": run_nights}[args.command] + runner(args, store) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/gate_study/items.json b/tests/fixtures/gate_study/items.json new file mode 100644 index 000000000..8ccb4fdd5 --- /dev/null +++ b/tests/fixtures/gate_study/items.json @@ -0,0 +1,30 @@ +[ + {"id": "fx-00", "question": "Fixture question 00: which word does passage 00 name?", "context": "Passage 00 names the word amber.", "answers": ["amber"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-01", "question": "Fixture question 01: which word does passage 01 name?", "context": "Passage 01 names the word birch.", "answers": ["birch"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-02", "question": "Fixture question 02: which word does passage 02 name?", "context": "Passage 02 names the word cedar.", "answers": ["cedar"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-03", "question": "Fixture question 03: which word does passage 03 name?", "context": "Passage 03 names the word delta.", "answers": ["delta"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-04", "question": "Fixture question 04: which word does passage 04 name?", "context": "Passage 04 names the word ember.", "answers": ["ember"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-05", "question": "Fixture question 05: which word does passage 05 name?", "context": "Passage 05 names the word flint.", "answers": ["flint"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-06", "question": "Fixture question 06: which word does passage 06 name?", "context": "Passage 06 names the word grove.", "answers": ["grove"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-07", "question": "Fixture question 07: which word does passage 07 name?", "context": "Passage 07 names the word heron.", "answers": ["heron"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-08", "question": "Fixture question 08: which word does passage 08 name?", "context": "Passage 08 names the word inlet.", "answers": ["inlet"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-09", "question": "Fixture question 09: which word does passage 09 name?", "context": "Passage 09 names the word jasper.", "answers": ["jasper"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-10", "question": "Fixture question 10: which word does passage 10 name?", "context": "Passage 10 names the word kelp.", "answers": ["kelp"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-11", "question": "Fixture question 11: which word does passage 11 name?", "context": "Passage 11 names the word lumen.", "answers": ["lumen"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-12", "question": "Fixture question 12: which word does passage 12 name?", "context": "Passage 12 names the word marsh.", "answers": ["marsh"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-13", "question": "Fixture question 13: which word does passage 13 name?", "context": "Passage 13 names the word nectar.", "answers": ["nectar"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-14", "question": "Fixture question 14: which word does passage 14 name?", "context": "Passage 14 names the word onyx.", "answers": ["onyx"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-15", "question": "Fixture question 15: which word does passage 15 name?", "context": "Passage 15 names the word pebble.", "answers": ["pebble"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-16", "question": "Fixture question 16: which word does passage 16 name?", "context": "Passage 16 names the word quartz.", "answers": ["quartz"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-17", "question": "Fixture question 17: which word does passage 17 name?", "context": "Passage 17 names the word reed.", "answers": ["reed"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-18", "question": "Fixture question 18: which word does passage 18 name?", "context": "Passage 18 names the word slate.", "answers": ["slate"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-19", "question": "Fixture question 19: which word does passage 19 name?", "context": "Passage 19 names the word tundra.", "answers": ["tundra"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-20", "question": "Fixture question 20: which word does passage 20 name?", "context": "Passage 20 names the word umber.", "answers": ["umber"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-21", "question": "Fixture question 21: which word does passage 21 name?", "context": "Passage 21 names the word vale.", "answers": ["vale"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-22", "question": "Fixture question 22: which word does passage 22 name?", "context": "Passage 22 names the word willow.", "answers": ["willow"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-23", "question": "Fixture question 23: which word does passage 23 name?", "context": "Passage 23 names the word xenon.", "answers": ["xenon"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-24", "question": "Fixture question 24: which word does passage 24 name?", "context": "Passage 24 names the word yarrow.", "answers": ["yarrow"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-25", "question": "Fixture question 25: which word does passage 25 name?", "context": "Passage 25 names the word zephyr.", "answers": ["zephyr"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-26", "question": "Fixture question 26: which word does passage 26 name?", "context": "Passage 26 names the word atlas.", "answers": ["atlas"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-27", "question": "Fixture question 27: which word does passage 27 name?", "context": "Passage 27 names the word basil.", "answers": ["basil"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]} +] diff --git a/tests/test_sleep_engine.py b/tests/test_sleep_engine.py index 8aaded740..7f7e62cac 100644 --- a/tests/test_sleep_engine.py +++ b/tests/test_sleep_engine.py @@ -3724,6 +3724,8 @@ def test_group_runs_inherit_dream_gate_budget_and_scoped_recall_config(self): gate_metric="mixed", gate_mixed_weight=0.37, gate_no_regression=True, + gate_train_check=True, + gate_train_margin=0.25, gate_mode="off", evolve_skill=True, evolve_memory=True, @@ -3799,6 +3801,8 @@ def _spy(backend, tasks, skill, memory, **kwargs): "gate_metric": "mixed", "gate_mixed_weight": 0.37, "gate_no_regression": True, + "gate_train_check": True, + "gate_train_margin": 0.25, "gate_mode": "off", "evolve_skill": True, } diff --git a/tests/test_sleep_gate_study.py b/tests/test_sleep_gate_study.py new file mode 100644 index 000000000..596857c16 --- /dev/null +++ b/tests/test_sleep_gate_study.py @@ -0,0 +1,91 @@ +"""The gate study harness, offline: the committed fixture, a deterministic +stand-in model, and the arithmetic the report prints.""" +from __future__ import annotations + +import json + +import pytest + +from skillopt_sleep.experiments import gate_study + + +def _run(path, *argv): + return gate_study.main([*argv, "--offline", "--store", str(path / "study.sqlite")]) + + +@pytest.fixture(scope="module") +def study(tmp_path_factory): + path = tmp_path_factory.mktemp("gate_study") + _run(path, "effects") + _run(path, "faults", "--arm", "off") + _run(path, "faults", "--arm", "check", "--feature", '{"gate_train_check": true}') + store = gate_study.Store(str(path / "study.sqlite")) + return gate_study.summarize(store, "fixture-model", gate_study.RULES["fixture"]) + + +def test_fixture_effects_are_the_ones_the_fixture_encodes(study): + assert study["starting_em"] == 75.0 + assert study["faults"]["F1 harmful"]["effect"] == -50.0 + assert study["faults"]["F2 helpful"]["effect"] == 25.0 + assert study["faults"]["F3 placebo"]["effect"] == 0.0 + + +def test_shipped_gate_accepts_the_harmful_rule_in_some_contexts(study): + off = study["faults"]["F1 harmful"]["arms"]["off"] + assert off == EXPECTED["harmful_off"] + assert off["accepted"] > 0 + + +def test_train_check_removes_those_accepts_and_keeps_the_helpful_ones(study): + faults = study["faults"] + assert faults["F1 harmful"]["arms"]["check"] == EXPECTED["harmful_check"] + assert faults["F2 helpful"]["arms"]["check"] == faults["F2 helpful"]["arms"]["off"] + assert faults["F2 helpful"]["arms"]["off"] == EXPECTED["helpful"] + assert faults["F3 placebo"]["arms"]["off"]["accepted"] == 0 + assert faults["F3 placebo"]["arms"]["check"]["accepted"] == 0 + + +def test_the_check_costs_one_training_replay_per_passing_candidate(study): + assert study["calls_per_decision"] == EXPECTED["calls_per_decision"] + assert study["calls_per_decision"]["check"] > study["calls_per_decision"]["off"] + + +def test_a_model_command_needs_live_or_offline(tmp_path): + with pytest.raises(SystemExit, match="--live"): + gate_study.main(["faults", "--store", str(tmp_path / "s.sqlite")]) + + +def test_report_arithmetic_on_known_rows(tmp_path, capsys): + store = gate_study.Store(str(tmp_path / "rows.sqlite")) + store.run("INSERT INTO effects VALUES ('m', '(starting skill)', 0.5)") + store.run("INSERT INTO effects VALUES ('m', 'F1 harmful', 0.25)") + for seed, accepted_off, accepted_on in ((1, 1, 0), (2, 1, 0), (3, 0, 0), (4, 0, 0)): + store.run("INSERT INTO decisions VALUES ('off', 'm', ?, 1, 'F1 harmful', ?, 20)", (seed, accepted_off)) + store.run("INSERT INTO decisions VALUES ('on', 'm', ?, 1, 'F1 harmful', ?, 22)", (seed, accepted_on)) + for seed, ems in ((1, (0.5, 0.6, 0.7)), (2, (0.5, 0.5, 0.6))): + for night, em in enumerate(ems): + store.run("INSERT INTO curves VALUES ('base', 'm', ?, ?, ?, ?, ?)", + (seed, night, int(night > 0 and em > ems[night - 1]), em, 200 if night else 0)) + data = gate_study.summarize(store, "m", gate_study.RULES["fixture"]) + assert data["starting_em"] == 50.0 + assert data["faults"]["F1 harmful"] == { + "effect": -25.0, + "arms": {"off": {"accepted": 2, "decisions": 4}, "on": {"accepted": 0, "decisions": 4}}, + } + assert data["faults"]["F2 helpful"] == {"effect": None, "arms": {}} + assert data["calls_per_decision"] == {"off": 20.0, "on": 22.0} + assert data["curves"]["base"] == { + "seeds": 2, "curve": [50.0, 55.0, 65.0], "accepted_nights": 3, + "first_accepted_night": 1.5, "target_calls_per_night": 200, + } + args = type("A", (), {"model": "m", "benchmark": "fixture", "json": True})() + gate_study.report(args, store) + assert json.loads(capsys.readouterr().out) == data + + +EXPECTED = { + "harmful_off": {"accepted": 1, "decisions": 16}, + "harmful_check": {"accepted": 0, "decisions": 16}, + "helpful": {"accepted": 10, "decisions": 16}, + "calls_per_decision": {"check": 19.4, "off": 16.8}, +} diff --git a/tests/test_sleep_gate_train_check.py b/tests/test_sleep_gate_train_check.py new file mode 100644 index 000000000..196adc5cb --- /dev/null +++ b/tests/test_sleep_gate_train_check.py @@ -0,0 +1,733 @@ +"""Training-side check: a candidate may not be clearly worse on tonight's training tasks.""" + +from __future__ import annotations + +import importlib +import json +import math +import os + +import pytest + +import skillopt_sleep.cycle as cycle_module +from skillopt_sleep.backend import Backend, DualBackend +from skillopt_sleep.config import DEFAULTS, load_config +from skillopt_sleep.consolidate import ( + TRAIN_CHECK_MIN_NET_REGRESSED, + consolidate, + train_check_enabled, + train_check_margin, +) +from skillopt_sleep.cycle import _render_report_md, run_sleep_cycle +from skillopt_sleep.dream import dream_consolidate +from skillopt_sleep.evidence import EvidenceLog, read_events +from skillopt_sleep.experiments.personas import researcher_persona +from skillopt_sleep.memory import set_learned +from skillopt_sleep.mine import assign_splits +from skillopt_sleep.types import EditRecord, SleepReport, TaskRecord + +SKILL_RULE = "Always add the category of the answer in parentheses." +MEMORY_RULE = "Remember to restate the question first." +SKILL = set_learned("# Skill\n", []) +MEMORY = set_learned("# Memory\n", []) + + +def _task(task_id: str, split: str, **extra) -> TaskRecord: + return TaskRecord(id=task_id, project="test", intent=f"question {task_id}", + split=split, **extra) + + +class ScriptedBackend(Backend): + """Each task scores a fixed value per document state. + + ``script`` maps a task's source id to its scores for + ``(no rule, skill rule, skill and memory rule)``; shorter tuples repeat + their last value. The memory rule never appears without the skill rule in + these tests. + """ + + name = "scripted-stub" + + def __init__(self, script: dict[str, tuple[float, ...]], propose_memory: bool = False): + self.script = script + self.propose_memory = propose_memory + self.calls: list[tuple[str, str]] = [] # (task id, phase) + + def attempt(self, task, skill, memory, sample_id: int = 0) -> str: + self.calls.append((task.id, self.evidence_phase)) + state = 2 if MEMORY_RULE in memory else (1 if SKILL_RULE in skill else 0) + source = (task.derived_from or task.id).removeprefix("recall:") + scores = self.script[source] + return repr(scores[min(state, len(scores) - 1)]) + + def judge(self, task, response): + score = float(response) + return score, score, "matched" if score >= 1.0 else "did not match" + + def reflect(self, failures, successes, skill, memory, *, edit_budget, + evolve_skill, evolve_memory): + if evolve_skill: + return [EditRecord(target="skill", op="add", content=SKILL_RULE, rationale="r")] + if evolve_memory and self.propose_memory: + return [EditRecord(target="memory", op="add", content=MEMORY_RULE, rationale="r")] + return [] + + +def _run(script, *, train=5, val=("v1",), **kwargs): + backend = ScriptedBackend(script) + tasks = [_task(f"t{i}", "train") for i in range(train)] + [_task(v, "val") for v in val] + kwargs.setdefault("evolve_memory", False) + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", **kwargs) + return backend, result + + +def _phases(backend: ScriptedBackend) -> set[str]: + return {phase for _task_id, phase in backend.calls} + + +def _check_ran(backend: ScriptedBackend) -> bool: + return any(phase.startswith("gate_train_check") for phase in _phases(backend)) + + +# A rule that fixes the one validation task and breaks every training task. +HARMFUL = {"v1": (0.0, 1.0), **{f"t{i}": (1.0, 0.0) for i in range(5)}} +# A rule that fixes the validation task and the training tasks. +HELPFUL = {"v1": (0.0, 1.0), **{f"t{i}": (0.0, 1.0) for i in range(5)}} + + +# ── off by default, and then nothing changes ───────────────────────────────── + + +def test_default_makes_no_extra_replay_and_writes_the_same_records(tmp_path) -> None: + backend, result = _run(HARMFUL) + explicit_backend, explicit = _run(HARMFUL, gate_train_check=False) + none_backend, none = _run(HARMFUL, gate_train_check=None) + + assert DEFAULTS["gate_train_check"] is False + assert DEFAULTS["gate_train_margin"] == 0.1 + assert backend.calls == explicit_backend.calls == none_backend.calls + assert not _check_ran(backend) + # the validation slice alone accepts the harmful rule + assert result.accepted is explicit.accepted is none.accepted is True + for trial in result.gate_trials: + assert set(trial) == { + "target", "baseline_score", "candidate_score", "accepted", + "blocked_by_regression", "task_deltas", + } + + logged = ScriptedBackend(HARMFUL) + path = os.path.join(tmp_path, "evidence.jsonl") + logged.evidence = EvidenceLog(path) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + consolidate(logged, tasks, SKILL, MEMORY, gate_metric="hard", evolve_memory=False) + events = read_events(path) + assert not [e for e in events if e.get("event") == "train_check"] + (trial,) = [e for e in events if e.get("stage") == "gate" and e.get("event") == "trial"] + assert "blocked_by_train_check" not in trial + + +# ── what the check changes ─────────────────────────────────────────────────── + + +def test_a_candidate_that_breaks_its_own_training_tasks_is_rejected() -> None: + backend, result = _run(HARMFUL, gate_train_check=True) + + assert result.accepted is False + assert SKILL_RULE not in result.new_skill + assert [edit.content for edit in result.rejected_edits] == [SKILL_RULE] + trial = result.gate_trials[0] + assert trial["candidate_score"] > trial["baseline_score"] # validation liked it + assert trial["accepted"] is False + assert trial["blocked_by_train_check"] is True + check = trial["train_check"] + assert (check["n_tasks"], check["margin"]) == (5, 0.1) + assert (check["n_regressed"], check["n_improved"]) == (5, 0) + assert check["min_net_regressed"] == TRAIN_CHECK_MIN_NET_REGRESSED == 2 + assert (check["baseline_score"], check["candidate_score"]) == (1.0, 0.0) + assert check["regressed"] is True and check["blocked"] is True + # both arms of every training task are kept + assert [(row["task_id"], row["baseline_score"], row["candidate_score"], row["status"]) + for row in check["task_deltas"]] == [ + (f"t{i}", 1.0, 0.0, "regressed") for i in range(5) + ] + assert ("t0", "gate_train_check:skill") in backend.calls + final = result.gate_trials[-1] + assert final["target"] == "final" + assert final["blocked_by_train_check"] is False and final["train_check"] is None + json.dumps(result.gate_trials, allow_nan=False) + + +def test_a_candidate_that_helps_its_training_tasks_is_still_accepted() -> None: + _, result = _run(HELPFUL, gate_train_check=True) + + assert result.accepted is True + assert SKILL_RULE in result.new_skill + check = result.gate_trials[0]["train_check"] + assert (check["baseline_score"], check["candidate_score"]) == (0.0, 1.0) + assert (check["n_regressed"], check["n_improved"]) == (0, 5) + assert check["regressed"] is False and check["blocked"] is False + assert result.gate_trials[0]["blocked_by_train_check"] is False + + +def test_the_check_never_accepts_what_validation_rejected() -> None: + # Validation ties, so nothing is accepted and nothing is replayed for the check. + script = {"v1": (1.0, 1.0), **{f"t{i}": (0.0, 1.0) for i in range(5)}} + + backend, result = _run(script, gate_train_check=True) + + assert result.accepted is False + assert not _check_ran(backend) + assert result.gate_trials[0]["train_check"] is None + + +def test_a_candidate_the_no_regression_rule_blocked_is_not_replayed() -> None: + script = {"v1": (0.0, 1.0), "v2": (1.0, 0.5), **{f"t{i}": (1.0, 0.0) for i in range(5)}} + + backend, result = _run(script, val=("v1", "v2"), gate_train_check=True, + gate_no_regression=True) + + trial = result.gate_trials[0] + assert trial["blocked_by_regression"] is True + assert trial["blocked_by_train_check"] is False and trial["train_check"] is None + assert not _check_ran(backend) + + +# ── the rejection rule ─────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("margin", "accepted"), + [(0.0, False), (0.1, False), (0.39, False), (0.4, True), (0.5, True)], +) +def test_margin_is_the_tolerated_drop_in_mean_training_score(margin, accepted) -> None: + # Two of five training tasks go from right to wrong: the mean drops by 0.4. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.0), "t1": (1.0, 0.0), + **{f"t{i}": (1.0, 1.0) for i in range(2, 5)}} + + _, result = _run(script, gate_train_check=True, gate_train_margin=margin) + + check = result.gate_trials[0]["train_check"] + assert check["baseline_score"] == pytest.approx(1.0) + assert check["candidate_score"] == pytest.approx(0.6) + assert (check["n_regressed"], check["n_improved"]) == (2, 0) + assert result.accepted is accepted + assert check["blocked"] is (not accepted) + + +@pytest.mark.parametrize("passing_before", [3, 4, 5, 6, 7, 8, 9, 10]) +def test_a_drop_of_exactly_the_margin_is_tolerated_at_every_baseline(passing_before) -> None: + # Ten training tasks; two go from right to wrong, so the mean drops by + # exactly 0.2. With margin 0.2 that is tolerated whatever the baseline is + # (0.8 - 0.2 is not 0.6 in floating point). + script = {"v1": (0.0, 1.0)} + for i in range(10): + before = 1.0 if i < passing_before else 0.0 + script[f"t{i}"] = (before, 0.0 if i < 2 else before) + + _, at_margin = _run(script, train=10, gate_train_check=True, gate_train_margin=0.2) + _, below_margin = _run(script, train=10, gate_train_check=True, gate_train_margin=0.19) + + check = at_margin.gate_trials[0]["train_check"] + assert check["baseline_score"] == pytest.approx(passing_before / 10) + assert check["candidate_score"] == pytest.approx((passing_before - 2) / 10) + assert check["regressed"] is False and check["blocked"] is False + assert at_margin.accepted is True + assert below_margin.gate_trials[0]["train_check"]["blocked"] is True + + +def test_the_margin_is_absolute_not_relative() -> None: + # Baseline 0.4, candidate 0.2: an absolute drop of 0.2 and a relative drop of half. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.0), "t1": (1.0, 0.0), + "t2": (0.0, 0.0), "t3": (0.0, 0.0), "t4": (0.0, 1.0)} + _, tolerated = _run(script, gate_train_check=True, gate_train_margin=0.3) + + check = tolerated.gate_trials[0]["train_check"] + assert check["baseline_score"] == pytest.approx(0.4) + assert check["candidate_score"] == pytest.approx(0.2) + assert check["regressed"] is False + assert tolerated.accepted is True + + +def test_margin_zero_rejects_any_broad_drop() -> None: + # Two tasks lose a little: mean drop 0.04, far below the default margin. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.9), "t1": (1.0, 0.9), + **{f"t{i}": (1.0, 1.0) for i in range(2, 5)}} + + _, default = _run(script, gate_train_check=True) + _, zero = _run(script, gate_train_check=True, gate_train_margin=0) + + assert default.accepted is True + assert default.gate_trials[0]["train_check"]["regressed"] is False + assert zero.accepted is False + assert zero.gate_trials[0]["train_check"]["blocked"] is True + + +def test_one_task_flipping_can_never_reject() -> None: + # One of five training tasks goes from right to wrong. The mean drops by + # 0.2, more than the margin, but a single task is not enough to reject. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.0), + **{f"t{i}": (1.0, 1.0) for i in range(1, 5)}} + + _, result = _run(script, gate_train_check=True, gate_train_margin=0.0) + + check = result.gate_trials[0]["train_check"] + assert check["candidate_score"] == pytest.approx(0.8) + assert (check["n_regressed"], check["n_improved"]) == (1, 0) + assert check["regressed"] is True and check["blocked"] is False + assert result.gate_trials[0]["blocked_by_train_check"] is False + assert result.accepted is True + + +@pytest.mark.parametrize(("better", "blocked"), [(0, True), (1, True), (2, False)]) +def test_net_count_of_regressed_tasks_decides(better, blocked) -> None: + # Three tasks get much worse; ``better`` tasks get slightly better. The + # mean always drops well past the margin; only the net count changes. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.0), "t1": (1.0, 0.0), "t2": (1.0, 0.0)} + for i in (3, 4): + script[f"t{i}"] = (0.0, 0.2) if i - 3 < better else (0.0, 0.0) + + _, result = _run(script, gate_train_check=True) + + check = result.gate_trials[0]["train_check"] + assert (check["n_regressed"], check["n_improved"]) == (3, better) + assert check["regressed"] is True and check["blocked"] is blocked + assert result.accepted is (not blocked) + + +@pytest.mark.parametrize(("train", "blocked"), [(1, False), (2, True), (3, True)]) +def test_how_many_training_tasks_it_takes(train, blocked) -> None: + script = {"v1": (0.0, 1.0), **{f"t{i}": (1.0, 0.0) for i in range(train)}} + + _, result = _run(script, train=train, gate_train_check=True) + + check = result.gate_trials[0]["train_check"] + assert check["n_tasks"] == train + assert check["regressed"] is True and check["blocked"] is blocked + assert result.accepted is (not blocked) + + +def test_a_malformed_training_score_fails_closed() -> None: + script = {"v1": (0.0, 1.0), "t0": (1.0, math.nan), + **{f"t{i}": (1.0, 1.0) for i in range(1, 5)}} + + _, result = _run(script, gate_train_check=True) + + check = result.gate_trials[0]["train_check"] + assert check["candidate_score"] is None + assert check["regressed"] is True and check["blocked"] is True + assert result.accepted is False + json.dumps(result.gate_trials, allow_nan=False) + + +def test_a_training_task_missing_from_the_candidate_replay_counts_as_worse(monkeypatch) -> None: + module = importlib.import_module("skillopt_sleep.consolidate") + real = module.replay_batch + + def dropping(backend, tasks, skill, memory): + pairs = real(backend, tasks, skill, memory) + if backend.evidence_phase.startswith("gate_train_check"): + pairs = [pair for pair in pairs if pair[0].id not in {"t0", "t1"}] + return pairs + + monkeypatch.setattr(module, "replay_batch", dropping) + + _, result = _run(HELPFUL | {"t2": (1.0, 0.0), "t3": (1.0, 0.0), "t4": (1.0, 0.0)}, + gate_train_check=True) + + check = result.gate_trials[0]["train_check"] + statuses = {row["task_id"]: row["status"] for row in check["task_deltas"]} + assert statuses["t0"] == statuses["t1"] == "regressed" + assert check["n_regressed"] == 5 + assert check["blocked"] is True + assert result.accepted is False + + +# ── scope ──────────────────────────────────────────────────────────────────── + + +def test_recalled_tasks_are_checked_and_dream_variants_are_not() -> None: + backend = ScriptedBackend({ + "v1": (0.0, 1.0), + **{f"t{i}": (1.0, 1.0) for i in range(2)}, + **{f"old{i}": (1.0, 0.0) for i in range(3)}, + }) + tonight = [_task("t0", "train"), _task("t1", "train"), _task("v1", "val")] + history = [ + TaskRecord(id=f"old{i}", project="test", intent=f"question t{i % 2} again {i}", + split="train") + for i in range(3) + ] + + result = dream_consolidate( + backend, tonight, SKILL, MEMORY, history_tasks=history, recall_k=3, + dream_factor=2, gate_metric="hard", evolve_memory=False, + gate_train_check=True, + ) + + checked = sorted(task_id for task_id, phase in backend.calls + if phase == "gate_train_check:skill") + assert checked == ["recall:old0", "recall:old1", "recall:old2", "t0", "t1"] + check = result.gate_trials[0]["train_check"] + assert check["n_tasks"] == 5 + assert check["baseline_score"] == pytest.approx(1.0) + assert check["candidate_score"] == pytest.approx(0.4) + assert result.accepted is False + + +def test_dream_variants_are_left_out_of_both_arms() -> None: + # The real training tasks improve; only their dream variants would look + # worse (a variant's score follows its source task here, so give the + # variants' sources different scripts through distinct real tasks). + backend = ScriptedBackend({ + "v1": (0.0, 1.0), "t0": (0.0, 1.0), "t1": (0.0, 1.0), + }) + tonight = [_task("t0", "train"), _task("t1", "train"), _task("v1", "val")] + + result = dream_consolidate( + backend, tonight, SKILL, MEMORY, dream_factor=3, gate_metric="hard", + evolve_memory=False, gate_train_check=True, + ) + + check = result.gate_trials[0]["train_check"] + assert check["n_tasks"] == 2 # not 2 real + 6 variants + assert [row["task_id"] for row in check["task_deltas"]] == ["t0", "t1"] + assert (check["baseline_score"], check["candidate_score"]) == (0.0, 1.0) + assert result.accepted is True + + +def test_gate_off_and_non_disjoint_validation_skip_the_check() -> None: + off_backend, off = _run(HARMFUL, gate_train_check=True, gate_mode="off") + leaked_backend, leaked = _run( + {f"t{i}": (0.0, 1.0) for i in range(5)}, val=(), gate_train_check=True, + ) + + assert not _check_ran(off_backend) + assert off.gate_action == "greedy_applied" + assert leaked.holdout_leaked is True + assert not _check_ran(leaked_backend) + + +def test_the_check_runs_with_dream_rollouts_and_parallel_workers(monkeypatch) -> None: + monkeypatch.setenv("SKILLOPT_SLEEP_WORKERS", "4") + + class Contrastive(ScriptedBackend): + def _call(self, prompt, *, max_tokens=1024): + return json.dumps([{"op": "add", "content": SKILL_RULE, "rationale": "r"}]) + + backend = Contrastive(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=False, rollouts_k=3, gate_train_check=True) + + assert result.gate_trials[0]["blocked_by_train_check"] is True + assert result.accepted is False + checked = sorted(t for t, phase in backend.calls if phase == "gate_train_check:skill") + assert checked == [f"t{i}" for i in range(5)] + + +# ── skill then memory ──────────────────────────────────────────────────────── + + +def test_with_a_dual_backend_the_check_replays_on_the_target_only() -> None: + target, optimizer = ScriptedBackend(HARMFUL), ScriptedBackend(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + result = consolidate(DualBackend(target, optimizer), tasks, SKILL, MEMORY, + gate_metric="hard", evolve_memory=False, gate_train_check=True) + + assert result.accepted is False + assert result.gate_trials[0]["blocked_by_train_check"] is True + assert sorted(task_id for task_id, _phase in target.calls).count("t0") == 2 # baseline, candidate + assert optimizer.calls == [] + + +def _skill_then_memory(script): + backend = ScriptedBackend(script, propose_memory=True) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=True, gate_train_check=True) + return backend, result + + +def test_memory_candidate_is_compared_with_the_accepted_skill() -> None: + # The skill rule helps everywhere and is accepted. The memory rule then + # lifts the validation task further but breaks the training tasks that the + # accepted skill had just fixed. + script = {"v1": (0.0, 0.5, 1.0), **{f"t{i}": (0.0, 1.0, 0.0) for i in range(5)}} + + backend, result = _skill_then_memory(script) + + skill_trial, memory_trial, _final = result.gate_trials + assert skill_trial["accepted"] is True + assert skill_trial["train_check"]["candidate_score"] == 1.0 + assert memory_trial["target"] == "memory" + assert memory_trial["blocked_by_train_check"] is True + assert memory_trial["train_check"]["baseline_score"] == 1.0 # post-skill scores + assert memory_trial["train_check"]["candidate_score"] == 0.0 + assert SKILL_RULE in result.new_skill + assert MEMORY_RULE not in result.new_memory + assert result.accepted is True + # the baseline arm is the post-skill training replay the night already runs + assert ("t0", "train_post_skill") in backend.calls + assert ("t0", "gate_train_check:memory") in backend.calls + + +def test_a_blocked_skill_does_not_become_the_memory_baseline() -> None: + # The skill rule breaks training and is blocked. The memory rule (which the + # stub can only reach with the skill rule absent) is then judged against + # the original documents' scores, not the blocked candidate's. + script = {"v1": (0.0, 1.0), **{f"t{i}": (1.0, 0.0) for i in range(5)}} + + class MemoryOnly(ScriptedBackend): + def attempt(self, task, skill, memory, sample_id: int = 0) -> str: + self.calls.append((task.id, self.evidence_phase)) + if MEMORY_RULE in memory: + return repr(1.0) # memory rule fixes everything + return repr(self.script[task.id][1 if SKILL_RULE in skill else 0]) + + backend = MemoryOnly(script, propose_memory=True) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=True, gate_train_check=True) + + skill_trial, memory_trial, _final = result.gate_trials + assert skill_trial["blocked_by_train_check"] is True + assert memory_trial["target"] == "memory" and memory_trial["accepted"] is True + assert memory_trial["train_check"]["baseline_score"] == 1.0 + assert memory_trial["train_check"]["candidate_score"] == 1.0 + assert SKILL_RULE not in result.new_skill and MEMORY_RULE in result.new_memory + + +# ── budget ─────────────────────────────────────────────────────────────────── + + +def test_the_check_is_part_of_the_edit_step_once_the_step_has_started() -> None: + backend = ScriptedBackend(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + checks = iter([False, False, True, True]) # epoch, skill edits, then spent + + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=True, gate_train_check=True, + budget_exhausted=lambda: next(checks, True)) + + assert result.budget_skipped == ["memory edits", "final validation replay"] + assert _check_ran(backend) + assert result.gate_trials[0]["blocked_by_train_check"] is True + assert result.accepted is False + + +# ── validation of the settings ─────────────────────────────────────────────── + + +@pytest.mark.parametrize("bad", [-0.01, 1.01, math.nan, math.inf, "0.1", None, True, [0.1]]) +def test_an_invalid_margin_stops_before_any_model_call(bad) -> None: + backend = ScriptedBackend(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + with pytest.raises(ValueError, match="gate_train_margin"): + consolidate(backend, tasks, SKILL, MEMORY, gate_train_check=True, + gate_train_margin=bad) + assert backend.calls == [] + + +@pytest.mark.parametrize("bad", ["true", "false", 1, 0, 0.5, [True]]) +def test_a_non_boolean_flag_stops_before_any_model_call(bad) -> None: + backend = ScriptedBackend(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + with pytest.raises(ValueError, match="gate_train_check must be true or false"): + consolidate(backend, tasks, SKILL, MEMORY, gate_train_check=bad) + assert backend.calls == [] + + +def test_setting_helpers() -> None: + assert train_check_enabled(None) is False + assert train_check_enabled(False) is False + assert train_check_enabled(True) is True + assert train_check_margin(0) == 0.0 and train_check_margin(1) == 1.0 + _, result = _run(HELPFUL, gate_train_margin="not a number") # ignored while off + assert result.accepted is True + + +# ── records ────────────────────────────────────────────────────────────────── + + +def test_evidence_log_keeps_both_arms_of_the_check(tmp_path) -> None: + backend = ScriptedBackend(HARMFUL) + path = os.path.join(tmp_path, "evidence.jsonl") + backend.evidence = EvidenceLog(path) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=False, gate_train_check=True) + events = read_events(path) + + (check,) = [e for e in events + if e.get("stage") == "gate" and e.get("event") == "train_check"] + assert (check["target"], check["n_tasks"], check["blocked"]) == ("skill", 5, True) + assert (check["baseline_score"], check["candidate_score"]) == (1.0, 0.0) + assert len(check["task_deltas"]) == 5 + (trial,) = [e for e in events if e.get("stage") == "gate" and e.get("event") == "trial"] + assert trial["accepted"] is False and trial["blocked_by_train_check"] is True + replays = [e for e in events if e.get("stage") == "replay" + and e.get("phase") == "gate_train_check:skill"] + assert sorted(e["task_id"] for e in replays) == [f"t{i}" for i in range(5)] + + +def _report_with(check: dict, *, accepted: bool, blocked: bool, cfg) -> str: + report = SleepReport(night=1, project="p", started_at="t", n_sessions=0, n_tasks=6) + report.gate_action = "accept_new_best" if accepted else "reject" + report.accepted = accepted + report.gate_trials = [{ + "target": "skill", "baseline_score": 0.0, "candidate_score": 1.0, + "accepted": accepted, "blocked_by_regression": False, + "blocked_by_train_check": blocked, "train_check": check, "task_deltas": [], + }] + return _render_report_md(report, cfg) + + +def test_report_wording_for_each_outcome(tmp_path) -> None: + cfg = _cfg(tmp_path, gate_train_check=True) + base = {"n_tasks": 5, "margin": 0.1, "baseline_score": 1.0} + + blocked = _report_with( + base | {"candidate_score": 0.0, "n_regressed": 5, "n_improved": 0, + "regressed": True, "blocked": True}, + accepted=False, blocked=True, cfg=cfg) + lower = _report_with( + base | {"candidate_score": 0.8, "n_regressed": 1, "n_improved": 0, + "regressed": True, "blocked": False}, + accepted=True, blocked=False, cfg=cfg) + fine = _report_with( + base | {"candidate_score": 1.0, "n_regressed": 0, "n_improved": 0, + "regressed": False, "blocked": False}, + accepted=True, blocked=False, cfg=cfg) + + assert "candidate — rejected (training-task regression)" in blocked + assert ("Training-side check (these tasks are not held out) over 5 training tasks: " + "1.000 → 0.000 (margin 0.1; 5 worse, 0 better): regressed, candidate rejected." + in blocked) + assert "(margin 0.1; 1 worse, 0 better): lower, but not on enough tasks to reject." in lower + assert "(margin 0.1; 0 worse, 0 better): no regression." in fine + assert "- training-side check: enabled (margin 0.1)" in fine + + +def test_report_header_reflects_whether_the_check_can_run(tmp_path) -> None: + report = SleepReport(night=1, project="p", started_at="t", n_sessions=0, n_tasks=6) + + off = _render_report_md(report, _cfg(tmp_path)) + gate_off = _render_report_md(report, _cfg(tmp_path, gate_train_check=True, gate_mode="off")) + null_margin = _render_report_md( + report, _cfg(tmp_path, gate_train_check=True, gate_train_margin=None)) + + assert "training-side check" not in off + assert "training-side check" not in gate_off + assert "- training-side check: enabled (margin 0.1)" in null_margin + + +# ── nightly cycle ──────────────────────────────────────────────────────────── + + +def _cfg(tmp_path, **extra): + project = tmp_path / "project" + home = tmp_path / "home" + project.mkdir(exist_ok=True) + home.mkdir(exist_ok=True) + return load_config( + invoked_project=str(project), projects="invoked", backend="mock", + claude_home=str(home / ".claude"), auto_adopt=False, **extra, + ) + + +def _spy_on_dream(monkeypatch): + seen: list[dict] = [] + real = cycle_module.dream_consolidate + + def spy(backend, tasks, skill, memory, **kwargs): + seen.append(dict(kwargs)) + return real(backend, tasks, skill, memory, **kwargs) + + monkeypatch.setattr(cycle_module, "dream_consolidate", spy) + return seen + + +def _night(tmp_path, **cfg_extra): + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + return run_sleep_cycle(_cfg(tmp_path, **cfg_extra), seed_tasks=tasks) + + +def test_cycle_forwards_the_setting_and_reports_it(tmp_path, monkeypatch) -> None: + seen = _spy_on_dream(monkeypatch) + + outcome = _night(tmp_path, gate_train_check=True, gate_train_margin=0.2) + + assert [(k["gate_train_check"], k["gate_train_margin"]) for k in seen] == [(True, 0.2)] + with open(os.path.join(outcome.staging_dir, "report.md"), encoding="utf-8") as fh: + report = fh.read() + assert "- training-side check: enabled (margin 0.2)" in report + events = read_events(os.path.join(outcome.staging_dir, "evidence.jsonl")) + start = next(e for e in events if e.get("stage") == "cycle" and e.get("event") == "start") + assert start["config"]["gate_train_check"] is True + assert start["config"]["gate_train_margin"] == 0.2 + assert outcome.report.accepted is True + assert ("Training-side check (these tasks are not held out) over 6 training tasks: " + "0.333 → 1.000 (margin 0.2; 0 worse, 6 better): no regression." in report) + (check,) = [e for e in events + if e.get("stage") == "gate" and e.get("event") == "train_check"] + assert (check["n_tasks"], check["blocked"]) == (6, False) + assert outcome.report.gate_trials[0]["train_check"]["n_tasks"] == 6 + + +def test_cycle_default_is_off_and_leaves_reports_and_records_alone(tmp_path, monkeypatch) -> None: + seen = _spy_on_dream(monkeypatch) + + outcome = _night(tmp_path) + + assert [k["gate_train_check"] for k in seen] == [False] + with open(os.path.join(outcome.staging_dir, "report.md"), encoding="utf-8") as fh: + report = fh.read() + assert "training-side check" not in report.lower() + events = read_events(os.path.join(outcome.staging_dir, "evidence.jsonl")) + assert not [e for e in events if e.get("event") == "train_check"] + start = next(e for e in events if e.get("stage") == "cycle" and e.get("event") == "start") + assert "gate_train_check" not in start["config"] + assert "gate_train_margin" not in start["config"] + for trial in outcome.report.gate_trials: + assert "train_check" not in trial and "blocked_by_train_check" not in trial + + +def test_cycle_null_margin_means_the_default(tmp_path, monkeypatch) -> None: + seen = _spy_on_dream(monkeypatch) + + _night(tmp_path, gate_train_check=True, gate_train_margin=None) + + assert [(k["gate_train_check"], k["gate_train_margin"]) for k in seen] == [(True, 0.1)] + + +@pytest.mark.parametrize( + ("extra", "message"), + [ + ({"gate_train_check": True, "gate_train_margin": 7}, "gate_train_margin"), + ({"gate_train_check": "true"}, "gate_train_check must be true or false"), + ({"gate_train_check": 1}, "gate_train_check must be true or false"), + ], +) +def test_cycle_rejects_invalid_settings_before_consolidation(tmp_path, monkeypatch, extra, message) -> None: + seen = _spy_on_dream(monkeypatch) + + with pytest.raises(ValueError, match=message): + _night(tmp_path, **extra) + assert seen == [] + + +def test_cycle_reads_the_settings_from_a_config_file(tmp_path, monkeypatch) -> None: + seen = _spy_on_dream(monkeypatch) + config_path = tmp_path / "config.json" + config_path.write_text( + json.dumps({"gate_train_check": True, "gate_train_margin": 0.25}), encoding="utf-8") + monkeypatch.setattr("skillopt_sleep.config._user_config_path", lambda: str(config_path)) + + _night(tmp_path) + + assert [(k["gate_train_check"], k["gate_train_margin"]) for k in seen] == [(True, 0.25)]