From 64fbe48beabdc82ff4ccb0f908611cf8bb96d739 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:48:07 +0000 Subject: [PATCH 1/2] fix(searchqa): treat a single string gold answer as one answer evaluate() and the EM, F1 and sub_em helpers iterate gold_answers. When an item's answers field was a single string instead of a list, they iterated its characters: the correct answer "Paris" scored EM 0, while "P" or "s" scored EM 1 and F1 1. hard is EM and soft is F1, so these items gave an inverted training reward and wrong reported scores. Wrap a string gold answer in a list at the start of evaluate(), which is the entry point both rollout paths call. List gold answers are unchanged. --- CHANGELOG.md | 3 +++ skillopt/envs/searchqa/evaluator.py | 7 ++++++- tests/test_searchqa_evaluator.py | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef83812d..d9e031a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,6 +180,9 @@ All notable changes to SkillOpt are documented here. This project adheres to Code, Codex and Pi transcript timestamps as instants rather than strings. East of UTC, sessions that ended in the hours after a harvest were never harvested (thanks @1AbdullahButt). +- Treat a SearchQA gold answer given as a single string as one answer instead + of a sequence of characters, which scored the correct answer as wrong and a + one-letter answer as correct (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/envs/searchqa/evaluator.py b/skillopt/envs/searchqa/evaluator.py index 3c29c89a..666e0eff 100644 --- a/skillopt/envs/searchqa/evaluator.py +++ b/skillopt/envs/searchqa/evaluator.py @@ -93,11 +93,16 @@ def sub_em(prediction: str, gold_answers: list[str]) -> float: return 0.0 -def evaluate(prediction_text: str, gold_answers: list[str]) -> dict: +def evaluate(prediction_text: str, gold_answers: list[str] | str) -> dict: """Evaluate a single QA prediction against gold answers. + A single string gold answer is treated as a one-item list, so it is not + iterated character by character. + Returns dict with: em, f1, sub_em, predicted_answer, gold_answers. """ + if isinstance(gold_answers, str): + gold_answers = [gold_answers] answer = extract_answer(prediction_text) return { "em": exact_match(answer, gold_answers), diff --git a/tests/test_searchqa_evaluator.py b/tests/test_searchqa_evaluator.py index 75fa9eea..8eea9a27 100644 --- a/tests/test_searchqa_evaluator.py +++ b/tests/test_searchqa_evaluator.py @@ -26,3 +26,17 @@ def test_sub_em_empty_prediction_matches_empty_gold(): result = evaluate("", [""]) assert result["sub_em"] == result["em"] == result["f1"] == 1.0 + + +@pytest.mark.parametrize("response", ["P", "s"]) +def test_single_string_gold_is_not_split_into_characters(response): + result = evaluate(response, "Paris") + + assert result["em"] == result["f1"] == 0.0 + + +def test_single_string_gold_matches_the_full_answer(): + result = evaluate("Paris", "Paris") + + assert result["em"] == result["f1"] == result["sub_em"] == 1.0 + assert result["gold_answers"] == ["Paris"] From c875c290596d77645e98cf42b19a559fd2988033 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:14:40 +0000 Subject: [PATCH 2/2] fix(searchqa): record a string gold answer as a list on rollout rows evaluate() now wraps a string gold answer, but the rollout kept the raw value: process_one stored item["answers"] as gold_answers, and run_batch's timeout and error rows stored it as gold_answer. For an item with "answers": "Paris" the score used ["Paris"] while the saved row recorded "Paris", so anything that later iterated the row's gold answers would get characters again. Normalize once with a _gold_answers() helper and use it for both rows. The evaluate() guard stays for direct callers. --- CHANGELOG.md | 5 ++- skillopt/envs/searchqa/rollout.py | 10 ++++- tests/test_searchqa_rollout_gold_answers.py | 41 +++++++++++++++++++++ 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 tests/test_searchqa_rollout_gold_answers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d9e031a0..e28afca5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -181,8 +181,9 @@ All notable changes to SkillOpt are documented here. This project adheres to East of UTC, sessions that ended in the hours after a harvest were never harvested (thanks @1AbdullahButt). - Treat a SearchQA gold answer given as a single string as one answer instead - of a sequence of characters, which scored the correct answer as wrong and a - one-letter answer as correct (thanks @Shizoqua). + of a sequence of characters, in scoring and in the gold answers recorded on + rollout rows. The correct answer was scored as wrong and a one-letter answer + as correct (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/envs/searchqa/rollout.py b/skillopt/envs/searchqa/rollout.py index 7f96393d..549ec017 100644 --- a/skillopt/envs/searchqa/rollout.py +++ b/skillopt/envs/searchqa/rollout.py @@ -27,6 +27,12 @@ _MAX_CONTEXT_CHARS = 6000 +def _gold_answers(item: dict) -> list: + """Return the item's gold answers as a list; a single string is one answer.""" + answers = item.get("answers", []) + return [answers] if isinstance(answers, str) else answers + + def _raise_on_systemic_failure(results: list[dict]) -> None: """Abort when all rollout rows failed before any agent response.""" if not results or not all(row.get("agent_ok") is False for row in results): @@ -184,7 +190,7 @@ def process_one( pred_dir = confined_task_output_dir(out_root, item_id) question = item["question"] context = item.get("context", "") - gold_answers = item.get("answers", []) + gold_answers = _gold_answers(item) result = { "id": item_id, @@ -415,7 +421,7 @@ def _timeout_result(item: dict) -> dict: "fail_reason": f"task-timeout-{task_timeout}s", "agent_ok": False, "n_turns": 0, - "gold_answer": item.get("answers", []), + "gold_answer": _gold_answers(item), "phase": "timeout", } diff --git a/tests/test_searchqa_rollout_gold_answers.py b/tests/test_searchqa_rollout_gold_answers.py new file mode 100644 index 00000000..78ec5f27 --- /dev/null +++ b/tests/test_searchqa_rollout_gold_answers.py @@ -0,0 +1,41 @@ +"""SearchQA rollout rows record a single string gold answer as a one-item list.""" + +from __future__ import annotations + +from skillopt.envs.searchqa import rollout + +ITEM = { + "id": "q1", + "question": "What is the capital of France?", + "context": "Paris is the capital.", + "answers": "Paris", +} + + +def test_process_one_records_string_gold_answer_as_a_list(tmp_path, monkeypatch): + monkeypatch.setattr(rollout, "is_target_exec_backend", lambda: False) + monkeypatch.setattr(rollout, "chat_target", lambda **kwargs: ("Paris", {})) + + row = rollout.process_one(dict(ITEM), str(tmp_path), "") + + assert row["gold_answers"] == ["Paris"] + assert row["hard"] == 1 + + +def test_error_row_records_string_gold_answer_as_a_list(tmp_path, monkeypatch): + good = {"id": "ok", "agent_ok": True, "hard": 1, "soft": 1.0} + + def fake_process_one(item, *args, **kwargs): + if item["id"] == "q1": + raise ValueError("boom") + return dict(good) + + monkeypatch.setattr(rollout, "process_one", fake_process_one) + + rows = rollout.run_batch( + [dict(ITEM), {"id": "ok", "question": "q", "answers": ["x"]}], str(tmp_path), "", workers=1 + ) + + (error_row,) = [row for row in rows if row["id"] == "q1"] + assert error_row["phase"] == "error" + assert error_row["gold_answer"] == ["Paris"]