From 9c5244e5b806ab02a7965ca122b0ad71fcfdae5f Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Thu, 19 Nov 2026 23:25:35 -0500 Subject: [PATCH] Add student_source() for inspecting the submission StudentFixture.student_source() returns the student's submitted source as a string, a documented way to inspect the submission itself (for example with ast) instead of reaching for the student_code_file path. For a notebook it returns the graded cells' code, the same source that runs in the sandbox, so .py and .ipynb questions use the same call. Co-Authored-By: Claude Opus 4.8 --- converting.md | 1 + quick_start.md | 21 ++++++++++++ src/pytest_prairielearn_grader/fixture.py | 18 ++++++++++ .../test_student_source/__init__.py | 0 .../test_student_source/expected_outcome.json | 29 ++++++++++++++++ .../test_student_source/scenario.py | 28 ++++++++++++++++ .../test_student_source/student_code.py | 2 ++ .../test_student_source_notebook/__init__.py | 0 .../expected_outcome.json | 20 +++++++++++ .../test_student_source_notebook/scenario.py | 18 ++++++++++ .../student_code.ipynb | 33 +++++++++++++++++++ 11 files changed, 170 insertions(+) create mode 100644 tests/scenario_root/test_student_source/__init__.py create mode 100644 tests/scenario_root/test_student_source/expected_outcome.json create mode 100644 tests/scenario_root/test_student_source/scenario.py create mode 100644 tests/scenario_root/test_student_source/student_code.py create mode 100644 tests/scenario_root/test_student_source_notebook/__init__.py create mode 100644 tests/scenario_root/test_student_source_notebook/expected_outcome.json create mode 100644 tests/scenario_root/test_student_source_notebook/scenario.py create mode 100644 tests/scenario_root/test_student_source_notebook/student_code.ipynb diff --git a/converting.md b/converting.md index 2b0035e..06fc8b2 100644 --- a/converting.md +++ b/converting.md @@ -187,6 +187,7 @@ regrade of a variant, and `question.html` could show it with `{{params.x}}`. | `total_iters = 3` and `@not_repeated` | `@pytest.mark.parametrize` over the inputs, or a loop of `subtests.test()` blocks for partial credit | | `Feedback.not_allowed` | `import_blacklist=["numpy.linalg.inv"]` in the `ConfigObject`. See [Update the setup code](#4-update-the-setup-code). | | `repeated_setup()` | Not needed: setup code runs again for each sandbox. | +| Reading `/grade/student/user_code.py` to inspect the submission with `ast` | `sandbox.student_source()`. See [Inspecting the Student's Source](quick_start.md#inspecting-the-students-source). | ### Checks diff --git a/quick_start.md b/quick_start.md index 6cdb017..8666967 100644 --- a/quick_start.md +++ b/quick_start.md @@ -634,6 +634,27 @@ def test_output_manually(sandbox: StudentFixture) -> None: assert_true("Hello, Alice!" in output, msg="Greeting message not found in output") ``` +### Inspecting the Student's Source + +Most tests check what student code _computes_. When you need to check how the submission is +_written_ — for example, that a result was built a particular way — use `sandbox.student_source()` +to get the submitted source as a string and inspect it, usually with the `ast` module. + +```python +import ast + + +@pytest.mark.grading_data(name="Uses matrix multiplication", points=1) +def test_uses_matmul(sandbox: StudentFixture) -> None: + tree = ast.parse(sandbox.student_source()) + uses_matmul = any(isinstance(node, ast.BinOp) and isinstance(node.op, ast.MatMult) for node in ast.walk(tree)) + assert_true(uses_matmul, msg="Use the @ operator for matrix multiplication.") +``` + +`student_source()` returns only the student's file, not any `leading_code.py` or `trailing_code.py` +the question adds around it. For a notebook submission it returns the graded cells' code (selected +by `notebook_cell_tag` when one is set), the same code that runs in the sandbox. + ### Module-Scoped Sandbox Use `module_sandbox` instead of `sandbox` when you want student code state to persist diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 4f68ff0..7a62318 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -551,6 +551,24 @@ def query_setup(self, var_to_query: str) -> Any: raise NameError(f"Query for setup variable '{var_to_query}' failed") return deserialize_object_unsafe(response["value_encoded"]) + def student_source(self) -> str: + """Return the source code the student submitted. + + Use this to inspect the submission itself, for example parsing it with + :mod:`ast` to check how a result was produced. For a ``.py`` submission + this is the file's text. For a notebook submission it is the code from + the graded cells, the same code that runs in the sandbox, selected by + ``notebook_cell_tag`` when one is set. + + This is only the student's file, not any ``leading_code.py`` or + ``trailing_code.py`` the question adds around it. + """ + if not self.student_code_file.is_file(): + raise FileNotFoundError(f"No student submission found at {self.student_code_file}") + if self.student_code_file.suffix.lower() == ".ipynb": + return _extract_notebook_code(self.student_code_file, self.notebook_cell_tag) + return self.student_code_file.read_text(encoding="utf-8") + # TODO add functions that let instructors use the student fixture # use the stuff pete set up here: https://github.com/reteps/pytest-autograder-prototype def __repr__(self) -> str: diff --git a/tests/scenario_root/test_student_source/__init__.py b/tests/scenario_root/test_student_source/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_student_source/expected_outcome.json b/tests/scenario_root/test_student_source/expected_outcome.json new file mode 100644 index 0000000..4726be8 --- /dev/null +++ b/tests/scenario_root/test_student_source/expected_outcome.json @@ -0,0 +1,29 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "source available", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_student_source.py::test_source_text_is_the_submission[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "ast inspection", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_student_source.py::test_source_can_be_parsed_and_inspected[student_code]" + } + ] + }, + "expected_outcome": { + "passed": 2, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_student_source/scenario.py b/tests/scenario_root/test_student_source/scenario.py new file mode 100644 index 0000000..6dd5a27 --- /dev/null +++ b/tests/scenario_root/test_student_source/scenario.py @@ -0,0 +1,28 @@ +"""Reading the student's submitted source to inspect it with ast. + +This mirrors the legacy CS357 sparse questions that parse the submission to +check how a result was produced (for example, detecting matrix multiplication). +""" + +import ast + +import pytest + +from pytest_prairielearn_grader.fixture import StudentFixture + + +@pytest.mark.grading_data(name="source available", points=1) +def test_source_text_is_the_submission(sandbox: StudentFixture) -> None: + source = sandbox.student_source() + assert "def combine(a, b):" in source + + +@pytest.mark.grading_data(name="ast inspection", points=1) +def test_source_can_be_parsed_and_inspected(sandbox: StudentFixture) -> None: + tree = ast.parse(sandbox.student_source()) + + function_names = {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} + assert "combine" in function_names + + uses_matmul = any(isinstance(node, ast.BinOp) and isinstance(node.op, ast.MatMult) for node in ast.walk(tree)) + assert uses_matmul diff --git a/tests/scenario_root/test_student_source/student_code.py b/tests/scenario_root/test_student_source/student_code.py new file mode 100644 index 0000000..ad16c1f --- /dev/null +++ b/tests/scenario_root/test_student_source/student_code.py @@ -0,0 +1,2 @@ +def combine(a, b): + return a @ b diff --git a/tests/scenario_root/test_student_source_notebook/__init__.py b/tests/scenario_root/test_student_source_notebook/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_student_source_notebook/expected_outcome.json b/tests/scenario_root/test_student_source_notebook/expected_outcome.json new file mode 100644 index 0000000..d9bb7ae --- /dev/null +++ b/tests/scenario_root/test_student_source_notebook/expected_outcome.json @@ -0,0 +1,20 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "notebook source", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_student_source_notebook.py::test_notebook_source_is_graded_cells[student_code]" + } + ] + }, + "expected_outcome": { + "passed": 1, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_student_source_notebook/scenario.py b/tests/scenario_root/test_student_source_notebook/scenario.py new file mode 100644 index 0000000..166ada7 --- /dev/null +++ b/tests/scenario_root/test_student_source_notebook/scenario.py @@ -0,0 +1,18 @@ +"""student_source() on a notebook returns the graded cells' code, not scratch cells.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject( + student_code_pattern="student_code.ipynb", + notebook_cell_tag="#grade", +) + + +@pytest.mark.grading_data(name="notebook source", points=1) +def test_notebook_source_is_graded_cells(sandbox: StudentFixture) -> None: + source = sandbox.student_source() + assert "def solution():" in source + assert "SHOULD_NOT_APPEAR" not in source diff --git a/tests/scenario_root/test_student_source_notebook/student_code.ipynb b/tests/scenario_root/test_student_source_notebook/student_code.ipynb new file mode 100644 index 0000000..7f7546f --- /dev/null +++ b/tests/scenario_root/test_student_source_notebook/student_code.ipynb @@ -0,0 +1,33 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "cell0", + "metadata": {}, + "outputs": [], + "source": [ + "scratch_marker = \"SHOULD_NOT_APPEAR\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell1", + "metadata": {}, + "outputs": [], + "source": [ + "#grade\n", + "def solution():\n", + " return 42" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}