fix(docvqa): keep the answer when a stray <answer> follows it

CHANGELOG.md

@@ -180,6 +180,9 @@ All notable changes to SkillOpt are documented here. This project adheres to
   Code, Codex and Pi transcript timestamps as instants rather than strings.
   East of UTC, sessions that ended in the hours after a harvest were never
   harvested (thanks @1AbdullahButt).
+- Keep the DocVQA answer when a stray `<answer>` or `</answer>` follows the
+  final answer, instead of scoring the wrong text and giving a correct answer
+  ANLS 0 (thanks @Shizoqua).
 
 ### Security
 - Prevent managed-identity credentials from being sent to non-Azure or

skillopt/envs/docvqa/evaluator.py

@@ -2,11 +2,16 @@
 
 import ast
 import json
+import re
 from collections.abc import Iterable
 from typing import Any
 
 DEFAULT_ANLS_THRESHOLD = 0.5
 
+# A complete <answer>...</answer> pair whose body contains no other "<answer>",
+# so nested input keeps the innermost answer and stray tags elsewhere are ignored.
+_ANSWER_RE = re.compile(r"<answer>((?:(?!<answer>).)*?)</answer>", re.DOTALL | re.IGNORECASE)
+
 
 def _normalize_text(value: Any) -> str:
     if value is None:
@@ -91,11 +96,9 @@ def _extract_answer_strings(raw: Any) -> list[str]:
 
 
 def extract_answer(text: str) -> str:
-    lower = text.lower()
-    start = lower.rfind("<answer>")
-    end = lower.rfind("</answer>")
-    if start != -1 and end != -1 and end > start:
-        return text[start + len("<answer>"):end].strip()
+    matches = _ANSWER_RE.findall(text)
+    if matches:
+        return matches[-1].strip()
     lines = [line.strip() for line in text.splitlines() if line.strip()]
     return lines[-1] if lines else text.strip()
 

tests/test_docvqa_evaluator.py

@@ -98,9 +98,25 @@ def test_extract_answer_strings_accepts_dataset_shapes(raw, expected):
     [
         ("reasoning\n<ANSWER>Foo</ANSWER>", "Foo"),
         ("<answer>x</answer> then <answer>y</answer>", "y"),
+        ("<answer>x<answer>y</answer>", "y"),
         ("first line\n\nlast line  ", "last line"),
         ("", ""),
     ],
 )
 def test_extract_answer(text, expected):
     assert extract_answer(text) == expected
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "<answer>42</answer> (formatted with the <answer> tag as asked)",
+        "The total is <answer>42</answer>\nNote: only one <answer> block.",
+        "<answer>42</answer>\n<answer>",
+        "<answer>42</answer> (wrapped in </answer> as asked)",
+        "<answer>42</answer>\n</answer>",
+    ],
+)
+def test_stray_answer_tag_after_the_last_complete_answer_is_ignored(text):
+    assert extract_answer(text) == "42"
+    assert evaluate(text, ["42"])["anls"] == 1.0