From 6c6c6f201d144669f55ba63d37f50d787e9ec7a7 Mon Sep 17 00:00:00 2001 From: gezerui0124 Date: Sat, 31 Oct 2026 03:22:54 +0800 Subject: [PATCH 1/2] fix(skillsbench): reject incomplete court forms --- tasks/samples/skillsbench/README.md | 30 +++-- .../verifier/test_outputs.py | 11 +- tests/test_skillsbench_court_form_verifier.py | 115 ++++++++++++++++++ 3 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 tests/test_skillsbench_court_form_verifier.py diff --git a/tasks/samples/skillsbench/README.md b/tasks/samples/skillsbench/README.md index 0584d5d7b..defccd361 100644 --- a/tasks/samples/skillsbench/README.md +++ b/tasks/samples/skillsbench/README.md @@ -7,15 +7,19 @@ evaluation (not just a smoke test) with one command. They are vendored from [**benchflow-ai/skillsbench**](https://github.com/benchflow-ai/skillsbench) (Apache-2.0), pinned at commit [`95b3e60`](https://github.com/benchflow-ai/skillsbench/tree/95b3e60de7a075090e2f9137f32e0d24da1a289b). -Each sample keeps the upstream prompt, inputs, and verifier; only the container -base and the coder_eval task wiring were added. The one local change is to the -`3d-scan-calc` verifier, which is patched to grade against fixed expected -values instead of re-deriving them from the agent-writable `scan_data.stl` -(issue #233). The other verifiers are unchanged and can still read -agent-writable inputs: `dialogue-parser` takes part of its ground truth from -`/app/script.txt`. - -That patch closes only the `scan_data.stl` vector. Grading still runs in the +Each sample keeps the upstream prompt and inputs. The container base and +coder_eval task wiring were added locally, along with two verifier patches: + +- `3d-scan-calc` grades against fixed expected values instead of re-deriving + them from the agent-writable `scan_data.stl` (issue #233). +- `court-form-filling` fails when required checkbox data or fields that must + remain blank are missing (issue #234). Its blank-field checks also replace + a nonexistent county field with the omitted third trial court address field. + +The `dialogue-parser` verifier is unchanged and takes part of its ground truth +from the agent-writable `/app/script.txt`. + +The `3d-scan-calc` patch closes only the `scan_data.stl` vector. Grading still runs in the agent's container, so the `pytest` on `PATH`, auto-loaded `pytest11` plugins, site-packages, and the `$TASK_DIR` verifier copy remain tamperable by the agent. See the follow-up issue @@ -56,7 +60,7 @@ coder-eval run tasks/samples/skillsbench/*/*.yaml Each task builds its container, runs the agent (Claude Sonnet 4.6 by default) in an isolated sandbox, then scores the result with the upstream verifier (for -`3d-scan-calc`, the locally patched one described above). Budget roughly 1–3 +`3d-scan-calc` and `court-form-filling`, the local patches described above). Budget roughly 1–3 minutes per task. ## Interpreting the results @@ -101,6 +105,6 @@ is non-deterministic, so exact numbers vary from run to run: These tasks are derived from benchflow-ai/skillsbench, licensed under Apache-2.0. The prompts, input data, and verifiers are the work of their original authors -(credited in each upstream task's `task.md`). The `3d-scan-calc` verifier is -patched locally to grade against fixed expected values. See the pinned commit -above for the upstream source of truth. +(credited in each upstream task's `task.md`). The `3d-scan-calc` and +`court-form-filling` verifiers have the local patches described above. See the +pinned commit above for the upstream source of truth. diff --git a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py index e5c5b5281..44bf75b5e 100644 --- a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py +++ b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py @@ -268,7 +268,6 @@ def normalize_text(text: str) -> str: ("SC-100[0].Page1[0].CaptionRight[0].CN[0].CaseName[0]", "page1_case_name"), ("SC-100[0].Page1[0].CaptionRight[0].CN[0].CaseNumber[0]", "page1_case_number"), ("SC-100[0].Page1[0].CaptionRight[0].County[0].CourtInfo[0]", "page1_court_info"), - ("SC-100[0].Page1[0].CaptionRight[0].County[0].County[0]", "page1_county"), # Trial date section 1 ("SC-100[0].Page1[0].Order[0].List1[0].LI1[0].TrialDate1[0]", "page1_trial_date1"), ("SC-100[0].Page1[0].Order[0].List1[0].LI1[0].TrialTime1[0]", "page1_trial_time1"), @@ -283,6 +282,7 @@ def normalize_text(text: str) -> str: ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDate3[0]", "page1_trial_date3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialTIme3[0]", "page1_trial_time3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDepartment3[0]", "page1_trial_dept3"), + ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateCourtAdd3[0]", "page1_court_addr3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateClerkSign[0]", "page1_clerk_sign"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateClerkSignDate[0]", "page1_clerk_date"), # Second plaintiff/defendant (only one of each in this case) @@ -421,9 +421,6 @@ def test_checkbox_value(self, checkbox_data, field_pattern, description, expecte } if not matching_fields: - # If no checkboxes found at all, it might be extraction failure - if not checkbox_data: - pytest.skip("No checkbox data could be extracted from PDF (XFA extraction may not be supported)") pytest.fail( f"Checkbox '{description}' (pattern: {field_pattern}) not found. " f"Available checkboxes: {list(checkbox_data.keys())}" @@ -459,9 +456,6 @@ def test_checkbox_unchecked(self, checkbox_data, field_pattern, description): } if not matching_fields: - # If no checkboxes found at all, it might be extraction failure - if not checkbox_data: - pytest.skip("No checkbox data could be extracted from PDF") # Field not found means it's likely unchecked (not in filled data) return @@ -479,6 +473,7 @@ class TestEmptyFields: ids=[f[1] for f in EMPTY_FIELDS]) def test_field_empty(self, pdf_field_values, field_name, description): """Verify field is empty or has minimal placeholder.""" - value = pdf_field_values.get(field_name, "") + assert field_name in pdf_field_values, f"Field '{description}' not found in PDF form" + value = pdf_field_values[field_name] is_empty = not value or value in ["", "None"] or len(value) < 3 assert is_empty, f"Field '{description}' should be empty, got: '{value}'" diff --git a/tests/test_skillsbench_court_form_verifier.py b/tests/test_skillsbench_court_form_verifier.py new file mode 100644 index 000000000..465e117a5 --- /dev/null +++ b/tests/test_skillsbench_court_form_verifier.py @@ -0,0 +1,115 @@ +"""Regression tests for the vendored court-form-filling verifier.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +_VERIFIER = ( + Path(__file__).resolve().parents[1] + / "tasks" + / "samples" + / "skillsbench" + / "court-form-filling" + / "verifier" + / "test_outputs.py" +) + + +@pytest.mark.parametrize( + ("check_class", "fixture_name", "values", "exit_code", "summary"), + [ + ("TestCheckboxes", "checkbox_data", "{}", 1, "6 failed"), + ("TestUncheckedCheckboxes", "checkbox_data", "{}", 0, "6 passed"), + ("TestEmptyFields", "pdf_field_values", "{}", 1, "20 failed"), + ( + "TestEmptyFields", + "pdf_field_values", + '{name: "" for name, _ in EMPTY_FIELDS[1:]}', + 1, + "1 failed, 19 passed", + ), + ( + "TestEmptyFields", + "pdf_field_values", + '{name: "filled" for name, _ in EMPTY_FIELDS}', + 1, + "20 failed", + ), + ( + "TestEmptyFields", + "pdf_field_values", + '{name: "" for name, _ in EMPTY_FIELDS}', + 0, + "20 passed", + ), + ( + "TestCheckboxes", + "checkbox_data", + "{pattern: expected for pattern, _, expected, _ in REQUIRED_CHECKBOXES}", + 0, + "6 passed", + ), + ( + "TestCheckboxes", + "checkbox_data", + '{pattern: "wrong" for pattern, _, _, _ in REQUIRED_CHECKBOXES}', + 1, + "6 failed", + ), + ( + "TestUncheckedCheckboxes", + "checkbox_data", + '{pattern: "/1" for pattern, _ in UNCHECKED_CHECKBOXES}', + 1, + "6 failed", + ), + ], + ids=[ + "missing-required-boxes", + "unchecked-boxes", + "missing-fields", + "one-missing-field", + "nonblank-fields", + "blank-fields", + "correct-boxes", + "wrong-boxes", + "unexpected-checked-boxes", + ], +) +def test_verifier_outcome( + tmp_path: Path, check_class: str, fixture_name: str, values: str, exit_code: int, summary: str +) -> None: + verifier = tmp_path / "test_outputs.py" + verifier.write_text( + _VERIFIER.read_text() + f"\n\n@pytest.fixture\ndef {fixture_name}():\n return {values}\n", + encoding="utf-8", + ) + config = tmp_path / "pytest.ini" + config.write_text("[pytest]\n", encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-c", + str(config), + f"{verifier}::{check_class}", + "-q", + "-p", + "no:cacheprovider", + ], + cwd=tmp_path, + env={**os.environ, "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == exit_code, result.stdout + result.stderr + assert summary in result.stdout, result.stdout + assert "skipped" not in result.stdout, result.stdout From 2ec6190b06f99b11740c656bc7b41b43ab55a2af Mon Sep 17 00:00:00 2001 From: gezerui0124 Date: Sat, 31 Oct 2026 09:25:59 +0800 Subject: [PATCH 2/2] fix(skillsbench): verify real forms and preserve AcroForm errors --- .claude/notes/lint-rules.md | 9 + pyproject.toml | 6 +- tasks/samples/skillsbench/README.md | 24 +- .../court-form-filling/environment/Dockerfile | 2 +- .../verifier/test_outputs.py | 43 ++- tests/test_custom_lint.py | 69 +++++ tests/test_skillsbench_court_form_verifier.py | 246 ++++++++++-------- uv.lock | 16 ++ 8 files changed, 274 insertions(+), 141 deletions(-) diff --git a/.claude/notes/lint-rules.md b/.claude/notes/lint-rules.md index 1044b58d9..32a6f2fa3 100644 --- a/.claude/notes/lint-rules.md +++ b/.claude/notes/lint-rules.md @@ -961,3 +961,12 @@ the module as a parameter instead of a third copy. Taking the class name from the model follows CE056 importing `IN_CONTAINER_ENV` and CE057 deriving its target set from `SIDECAR_MODULES`. Alias resolution removes the *local binding* spelling, not every rename; taking the name from the model covers the rest. + +## CE071 — patched binary verifiers must not skip + +The court-form verifier skipped checks when checkbox data or blank-field keys +were absent. Its shell runner treated pytest's successful exit as reward 1. +The guard rejects skip and xfail calls or marks in the two locally patched +binary-reward SkillsBench verifiers. It also rejects bare `except:` and +`except BaseException:` handlers. Real PDF regression tests +also anchor the court-form field names and extraction to the shipped input. diff --git a/pyproject.toml b/pyproject.toml index 2cb30ce71..d7628741a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ Changelog = "https://github.com/UiPath/coder_eval/blob/main/CHANGELOG.md" [project.optional-dependencies] dev = [ + "pypdf[crypto]==6.21.4", "pytest-asyncio>=1.3.0", "pytest>=9.0.2", "pytest-mock>=3.15.1", @@ -241,8 +242,8 @@ constraint-dependencies = [ [tool.ruff] line-length = 120 target-version = "py313" -# Vendored third-party sample tasks (benchflow-ai/skillsbench) are kept verbatim; -# they are not framework code and are not linted/formatted. +# Vendored third-party sample tasks (benchflow-ai/skillsbench) include local +# verifier patches documented in their README. Keep them outside framework formatting. extend-exclude = ["tasks/samples"] [tool.ruff.lint] @@ -310,6 +311,7 @@ external = [ "CE065", "CE066", "CE068", + "CE071", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/tasks/samples/skillsbench/README.md b/tasks/samples/skillsbench/README.md index defccd361..bb07d7fdb 100644 --- a/tasks/samples/skillsbench/README.md +++ b/tasks/samples/skillsbench/README.md @@ -15,14 +15,18 @@ coder_eval task wiring were added locally, along with two verifier patches: - `court-form-filling` fails when required checkbox data or fields that must remain blank are missing (issue #234). Its blank-field checks also replace a nonexistent county field with the omitted third trial court address field. + AcroForm read and field dereference errors propagate to the test runner. + Its environment pins pypdf 6.21.4 (upstream 5.1.0) to remove known advisories. The `dialogue-parser` verifier is unchanged and takes part of its ground truth -from the agent-writable `/app/script.txt`. - -The `3d-scan-calc` patch closes only the `scan_data.stl` vector. Grading still runs in the -agent's container, so the `pytest` on `PATH`, auto-loaded `pytest11` plugins, -site-packages, and the `$TASK_DIR` verifier copy remain tamperable by the -agent. See the follow-up issue +from the agent-writable `/app/script.txt`. The `court-form-filling` verifier +also reads the agent-writable `/root/sc100-blank.pdf` for its input-versus-output +identity check. + +The `3d-scan-calc` patch closes only the `scan_data.stl` vector. Grading still +runs in the agent's container, so the `pytest` on `PATH`, auto-loaded `pytest11` +plugins, site-packages, and the `$TASK_DIR` verifier copy remain tamperable by +the agent. See the follow-up issue [#238](https://github.com/UiPath/coder_eval/issues/238). ## What they do @@ -60,8 +64,8 @@ coder-eval run tasks/samples/skillsbench/*/*.yaml Each task builds its container, runs the agent (Claude Sonnet 4.6 by default) in an isolated sandbox, then scores the result with the upstream verifier (for -`3d-scan-calc` and `court-form-filling`, the local patches described above). Budget roughly 1–3 -minutes per task. +`3d-scan-calc` and `court-form-filling`, the local patches described above). +Budget roughly 1–3 minutes per task. ## Interpreting the results @@ -94,6 +98,10 @@ is non-deterministic, so exact numbers vary from run to run: | `court-form-filling` | 1.00 | | `dialogue-parser` | ~0.83 | +The court-form score above predates the verifier fix for issue #234 and the +pypdf update. Those historical scores are not comparable with scores from +the updated sample. + ## How it works (brief) - **`environment/Dockerfile`** rebases the upstream task onto `coder-eval-agent` and bakes the inputs at the paths the prompt expects. diff --git a/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile b/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile index b571aebc3..54c85bda2 100644 --- a/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile +++ b/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends poppler-utils \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir \ - pypdf==5.1.0 \ + pypdf==6.21.4 \ fillpdf==0.7.3 \ pdfrw==0.4 \ PyPDF2==3.0.1 \ diff --git a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py index 44bf75b5e..83b81f3e6 100644 --- a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py +++ b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py @@ -1,3 +1,4 @@ +# Modified from benchflow-ai/skillsbench@95b3e60: require form fields and preserve AcroForm read and field dereference errors (issue #234). """ Tests for court-form-filling task (SC-100 Small Claims Court form). @@ -63,26 +64,23 @@ def extract_checkbox_values(pdf_path: Path) -> dict: checkbox_values.update(parse_xfa_checkboxes(xfa_xml)) # Method 2: Try pypdf get_fields() for AcroForms - try: - from pypdf import PdfReader - reader = PdfReader(str(pdf_path)) - fields = reader.get_fields() or {} - - for name, field in fields.items(): - # Check if it's a checkbox/radio field - if "Checkbox" in name or field.get("/FT") in ["/Btn"]: - value = field.get("/V", "") - if value: - if hasattr(value, "get_object"): - value = str(value.get_object()) - else: - value = str(value) - # Normalize value - value = value.strip("/") - if value and value not in ["Off", "None", ""]: - checkbox_values[name] = value - except Exception: - pass + from pypdf import PdfReader + reader = PdfReader(str(pdf_path)) + fields = reader.get_fields() or {} + + for name, field in fields.items(): + # Check if it's a checkbox/radio field + if "Checkbox" in name or field.get("/FT") in ["/Btn"]: + value = field.get("/V", "") + if value: + if hasattr(value, "get_object"): + value = str(value.get_object()) + else: + value = str(value) + # Normalize value + value = value.strip("/") + if value and value not in ["Off", "None", ""]: + checkbox_values[name] = value return checkbox_values @@ -347,10 +345,7 @@ def pdf_field_values(): value = field.get("/V", "") if value: if hasattr(value, "get_object"): - try: - value = str(value.get_object()) - except: - value = str(value) + value = str(value.get_object()) else: value = str(value) field_values[name] = value if value else "" diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index c166dfd5a..7eb1e52df 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -5399,3 +5399,72 @@ def test_shares_manifest_skill_dirs_with_the_runtime_allowlist(self): from coder_eval.isolation import eval_material assert eval_material.manifest_skill_dirs is manifest_skill_dirs + + +@pytest.mark.lint +class TestCE071PatchedBinaryVerifiersDoNotSkip: + """CE071: patched binary verifiers must not skip, xfail, or catch BaseException. + + Scope: the locally patched 3d-scan-calc and court-form-filling verifiers. + Blind spots: imported aliases and dynamically constructed calls are not resolved. + """ + + @staticmethod + def _offenders(source: str) -> list[int]: + import ast + + return [ + node.lineno + for node in ast.walk(ast.parse(source)) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "pytest" + and node.func.attr in {"skip", "importorskip", "xfail"} + ) + or ( + isinstance(node, ast.Attribute) + and node.attr in {"skip", "skipif", "xfail"} + and isinstance(node.value, ast.Attribute) + and node.value.attr == "mark" + and isinstance(node.value.value, ast.Name) + and node.value.value.id == "pytest" + ) + or ( + isinstance(node, ast.ExceptHandler) + and (node.type is None or (isinstance(node.type, ast.Name) and node.type.id == "BaseException")) + ) + ] + + @pytest.mark.parametrize("slug", ["3d-scan-calc", "court-form-filling"]) + def test_patched_verifier(self, slug: str) -> None: + path = ( + Path(__file__).resolve().parents[1] + / "tasks" + / "samples" + / "skillsbench" + / slug + / "verifier/test_outputs.py" + ) + offenders = self._offenders(path.read_text(encoding="utf-8")) + assert not offenders, f"CE071: silent outcome or broad catch in {path} at lines {offenders}" + + @pytest.mark.parametrize( + "source", + [ + "pytest.skip('missing')", + "pytest.importorskip('pypdf')", + "pytest.xfail('missing')", + "@pytest.mark.skip\ndef test(): pass", + "@pytest.mark.skipif(True, reason='missing')\ndef test(): pass", + "@pytest.mark.xfail\ndef test(): pass", + "try: read()\nexcept: pass", + "try: read()\nexcept BaseException: pass", + ], + ) + def test_detects_silent_outcomes(self, source: str) -> None: + assert self._offenders(source) + + def test_allows_failure_and_narrow_exception(self) -> None: + assert not self._offenders("try: read()\nexcept ValueError: pytest.fail('invalid')") diff --git a/tests/test_skillsbench_court_form_verifier.py b/tests/test_skillsbench_court_form_verifier.py index 465e117a5..fb3910b72 100644 --- a/tests/test_skillsbench_court_form_verifier.py +++ b/tests/test_skillsbench_court_form_verifier.py @@ -2,114 +2,148 @@ from __future__ import annotations -import os -import subprocess -import sys +import importlib.util from pathlib import Path +from unittest.mock import Mock import pytest +from pypdf import PdfReader, PdfWriter +from pypdf.errors import PdfReadError -_VERIFIER = ( - Path(__file__).resolve().parents[1] - / "tasks" - / "samples" - / "skillsbench" - / "court-form-filling" - / "verifier" - / "test_outputs.py" -) - - -@pytest.mark.parametrize( - ("check_class", "fixture_name", "values", "exit_code", "summary"), - [ - ("TestCheckboxes", "checkbox_data", "{}", 1, "6 failed"), - ("TestUncheckedCheckboxes", "checkbox_data", "{}", 0, "6 passed"), - ("TestEmptyFields", "pdf_field_values", "{}", 1, "20 failed"), - ( - "TestEmptyFields", - "pdf_field_values", - '{name: "" for name, _ in EMPTY_FIELDS[1:]}', - 1, - "1 failed, 19 passed", - ), - ( - "TestEmptyFields", - "pdf_field_values", - '{name: "filled" for name, _ in EMPTY_FIELDS}', - 1, - "20 failed", - ), - ( - "TestEmptyFields", - "pdf_field_values", - '{name: "" for name, _ in EMPTY_FIELDS}', - 0, - "20 passed", - ), - ( - "TestCheckboxes", - "checkbox_data", - "{pattern: expected for pattern, _, expected, _ in REQUIRED_CHECKBOXES}", - 0, - "6 passed", - ), - ( - "TestCheckboxes", - "checkbox_data", - '{pattern: "wrong" for pattern, _, _, _ in REQUIRED_CHECKBOXES}', - 1, - "6 failed", - ), - ( - "TestUncheckedCheckboxes", - "checkbox_data", - '{pattern: "/1" for pattern, _ in UNCHECKED_CHECKBOXES}', - 1, - "6 failed", - ), - ], - ids=[ - "missing-required-boxes", - "unchecked-boxes", - "missing-fields", - "one-missing-field", - "nonblank-fields", - "blank-fields", - "correct-boxes", - "wrong-boxes", - "unexpected-checked-boxes", - ], -) -def test_verifier_outcome( - tmp_path: Path, check_class: str, fixture_name: str, values: str, exit_code: int, summary: str -) -> None: - verifier = tmp_path / "test_outputs.py" - verifier.write_text( - _VERIFIER.read_text() + f"\n\n@pytest.fixture\ndef {fixture_name}():\n return {values}\n", - encoding="utf-8", - ) - config = tmp_path / "pytest.ini" - config.write_text("[pytest]\n", encoding="utf-8") - result = subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "-c", - str(config), - f"{verifier}::{check_class}", - "-q", - "-p", - "no:cacheprovider", - ], - cwd=tmp_path, - env={**os.environ, "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, - capture_output=True, - text=True, - timeout=30, - ) - assert result.returncode == exit_code, result.stdout + result.stderr - assert summary in result.stdout, result.stdout - assert "skipped" not in result.stdout, result.stdout +_TASK_DIR = Path(__file__).resolve().parents[1] / "tasks" / "samples" / "skillsbench" / "court-form-filling" +_BLANK_PDF = _TASK_DIR / "environment" / "sc100-blank.pdf" +_SPEC = importlib.util.spec_from_file_location("court_form_verifier", _TASK_DIR / "verifier" / "test_outputs.py") +assert _SPEC is not None and _SPEC.loader is not None +_VERIFIER = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_VERIFIER) + + +@pytest.fixture(autouse=True) +def reject_verifier_skips(monkeypatch: pytest.MonkeyPatch) -> None: + """A direct verifier call must fail if it tries to skip a check.""" + monkeypatch.setattr(pytest, "skip", pytest.fail) + + +def test_verifier_skip_fails() -> None: + with pytest.raises(pytest.fail.Exception, match="missing"): + pytest.skip("missing") + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_missing_fails(case: tuple) -> None: + with pytest.raises(pytest.fail.Exception, match="not found"): + _VERIFIER.TestCheckboxes().test_checkbox_value({}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_wrong_value_fails(case: tuple) -> None: + with pytest.raises(AssertionError, match="expected value"): + _VERIFIER.TestCheckboxes().test_checkbox_value({case[0]: "wrong"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_correct_value_passes(case: tuple) -> None: + _VERIFIER.TestCheckboxes().test_checkbox_value({case[0]: case[2]}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.UNCHECKED_CHECKBOXES, ids=lambda case: case[1]) +def test_unchecked_checkbox_absent_passes(case: tuple) -> None: + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked({}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.UNCHECKED_CHECKBOXES, ids=lambda case: case[1]) +def test_unchecked_checkbox_checked_fails(case: tuple) -> None: + with pytest.raises(AssertionError, match="should be unchecked"): + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked({case[0]: "/1"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_missing_fails(case: tuple) -> None: + values = {name: "" for name, _ in _VERIFIER.EMPTY_FIELDS if name != case[0]} + with pytest.raises(AssertionError, match="not found"): + _VERIFIER.TestEmptyFields().test_field_empty(values, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_filled_fails(case: tuple) -> None: + with pytest.raises(AssertionError, match="should be empty"): + _VERIFIER.TestEmptyFields().test_field_empty({case[0]: "filled"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_blank_passes(case: tuple) -> None: + _VERIFIER.TestEmptyFields().test_field_empty({case[0]: ""}, *case) + + +def test_expected_fields_exist_in_shipped_pdf() -> None: + fields = PdfReader(_BLANK_PDF).get_fields() + assert fields is not None + for name, _ in _VERIFIER.EMPTY_FIELDS: + assert name in fields, name + assert fields[name]["/FT"] == "/Tx", name + for pattern, *_ in [*_VERIFIER.REQUIRED_CHECKBOXES, *_VERIFIER.UNCHECKED_CHECKBOXES]: + matches = [field for name, field in fields.items() if pattern in name] + assert matches, pattern + assert all(field["/FT"] == "/Btn" for field in matches), pattern + + +def test_correct_pdf_field_roundtrip_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + reader = PdfReader(_BLANK_PDF) + fields = reader.get_fields() + assert fields is not None + values = { + name: expected + for pattern, _, expected, _ in _VERIFIER.REQUIRED_CHECKBOXES + for name, field in fields.items() + if pattern in name and expected in field.get("/_States_", []) + } + output = tmp_path / "filled.pdf" + writer = PdfWriter(clone_from=reader) + writer.update_page_form_field_values(list(writer.pages), values, auto_regenerate=False) + writer.write(output) + monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", output) + checkboxes = _VERIFIER.checkbox_data.__wrapped__() + field_values = _VERIFIER.pdf_field_values.__wrapped__() + for case in _VERIFIER.REQUIRED_CHECKBOXES: + _VERIFIER.TestCheckboxes().test_checkbox_value(checkboxes, *case) + for case in _VERIFIER.UNCHECKED_CHECKBOXES: + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked(checkboxes, *case) + for case in _VERIFIER.EMPTY_FIELDS: + _VERIFIER.TestEmptyFields().test_field_empty(field_values, *case) + + +def test_blank_pdf_required_checkboxes_fail() -> None: + checkboxes = _VERIFIER.extract_checkbox_values(_BLANK_PDF) + for case in _VERIFIER.REQUIRED_CHECKBOXES: + with pytest.raises(pytest.fail.Exception, match="not found"): + _VERIFIER.TestCheckboxes().test_checkbox_value(checkboxes, *case) + + +def test_pdf_without_form_fields_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + output = tmp_path / "no-form.pdf" + writer = PdfWriter() + writer.add_blank_page(width=612, height=792) + writer.write(output) + monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", output) + values = _VERIFIER.pdf_field_values.__wrapped__() + assert values == {} + for case in _VERIFIER.EMPTY_FIELDS: + with pytest.raises(AssertionError, match="not found"): + _VERIFIER.TestEmptyFields().test_field_empty(values, *case) + + +def test_checkbox_extraction_preserves_pdf_error(tmp_path: Path) -> None: + output = tmp_path / "invalid.pdf" + output.write_bytes(b"not a PDF") + with pytest.raises(PdfReadError): + _VERIFIER.extract_checkbox_values(output) + + +def test_field_extraction_preserves_dereference_error(monkeypatch: pytest.MonkeyPatch) -> None: + value = Mock() + value.get_object.side_effect = RuntimeError("broken field reference") + monkeypatch.setattr("pypdf.PdfReader", Mock(return_value=Mock(get_fields=lambda: {"field": {"/V": value}}))) + monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", _BLANK_PDF) + with pytest.raises(RuntimeError, match="broken field reference"): + _VERIFIER.pdf_field_values.__wrapped__() diff --git a/uv.lock b/uv.lock index 3dfa40491..39eec8516 100644 --- a/uv.lock +++ b/uv.lock @@ -596,6 +596,7 @@ dev = [ { name = "mcp" }, { name = "pip-audit" }, { name = "pre-commit" }, + { name = "pypdf", extra = ["crypto"] }, { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -636,6 +637,7 @@ requires-dist = [ { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.5.1" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, + { name = "pypdf", extras = ["crypto"], marker = "extra == 'dev'", specifier = "==6.21.4" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.408" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" }, @@ -2406,6 +2408,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pypdf" +version = "6.21.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyright" version = "1.1.408"