Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions converting.md
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ regrade of a variant, and `question.html` could show it with `{{params.x}}`.
| `total_iters = 3` and `@not_repeated` | `@pytest.mark.parametrize` over the inputs, or a loop of `subtests.test()` blocks for partial credit |
| `Feedback.not_allowed` | `import_blacklist=["numpy.linalg.inv"]` in the `ConfigObject`. See [Update the setup code](#4-update-the-setup-code). |
| `repeated_setup()` | Not needed: setup code runs again for each sandbox. |
| Reading `/grade/student/user_code.py` to inspect the submission with `ast` | `sandbox.student_source()`. See [Inspecting the Student's Source](quick_start.md#inspecting-the-students-source). |

### Checks

Expand Down
21 changes: 21 additions & 0 deletions quick_start.md
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,27 @@ def test_output_manually(sandbox: StudentFixture) -> None:
assert_true("Hello, Alice!" in output, msg="Greeting message not found in output")
```

### Inspecting the Student's Source

Most tests check what student code _computes_. When you need to check how the submission is
_written_ — for example, that a result was built a particular way — use `sandbox.student_source()`
to get the submitted source as a string and inspect it, usually with the `ast` module.

```python
import ast


@pytest.mark.grading_data(name="Uses matrix multiplication", points=1)
def test_uses_matmul(sandbox: StudentFixture) -> None:
tree = ast.parse(sandbox.student_source())
uses_matmul = any(isinstance(node, ast.BinOp) and isinstance(node.op, ast.MatMult) for node in ast.walk(tree))
assert_true(uses_matmul, msg="Use the @ operator for matrix multiplication.")
```

`student_source()` returns only the student's file, not any `leading_code.py` or `trailing_code.py`
the question adds around it. For a notebook submission it returns the graded cells' code (selected
by `notebook_cell_tag` when one is set), the same code that runs in the sandbox.

### Module-Scoped Sandbox

Use `module_sandbox` instead of `sandbox` when you want student code state to persist
Expand Down
18 changes: 18 additions & 0 deletions src/pytest_prairielearn_grader/fixture.py
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,24 @@ def query_setup(self, var_to_query: str) -> Any:
raise NameError(f"Query for setup variable '{var_to_query}' failed")
return deserialize_object_unsafe(response["value_encoded"])

def student_source(self) -> str:
"""Return the source code the student submitted.

Use this to inspect the submission itself, for example parsing it with
:mod:`ast` to check how a result was produced. For a ``.py`` submission
this is the file's text. For a notebook submission it is the code from
the graded cells, the same code that runs in the sandbox, selected by
``notebook_cell_tag`` when one is set.

This is only the student's file, not any ``leading_code.py`` or
``trailing_code.py`` the question adds around it.
"""
if not self.student_code_file.is_file():
raise FileNotFoundError(f"No student submission found at {self.student_code_file}")
if self.student_code_file.suffix.lower() == ".ipynb":
return _extract_notebook_code(self.student_code_file, self.notebook_cell_tag)
return self.student_code_file.read_text(encoding="utf-8")

# TODO add functions that let instructors use the student fixture
# use the stuff pete set up here: https://github.com/reteps/pytest-autograder-prototype
def __repr__(self) -> str:
Expand Down
Empty file.
29 changes: 29 additions & 0 deletions tests/scenario_root/test_student_source/expected_outcome.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"expected_data_object": {
"score": 1.0,
"tests": [
{
"max_points": 1,
"message": "",
"name": "source available",
"outcome": "passed",
"points": 1.0,
"points_frac": 1.0,
"test_id": "test_student_source.py::test_source_text_is_the_submission[student_code]"
},
{
"max_points": 1,
"message": "",
"name": "ast inspection",
"outcome": "passed",
"points": 1.0,
"points_frac": 1.0,
"test_id": "test_student_source.py::test_source_can_be_parsed_and_inspected[student_code]"
}
]
},
"expected_outcome": {
"passed": 2,
"failed": 0
}
}
28 changes: 28 additions & 0 deletions tests/scenario_root/test_student_source/scenario.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"""Reading the student's submitted source to inspect it with ast.

This mirrors the legacy CS357 sparse questions that parse the submission to
check how a result was produced (for example, detecting matrix multiplication).
"""

import ast

import pytest

from pytest_prairielearn_grader.fixture import StudentFixture


@pytest.mark.grading_data(name="source available", points=1)
def test_source_text_is_the_submission(sandbox: StudentFixture) -> None:
source = sandbox.student_source()
assert "def combine(a, b):" in source


@pytest.mark.grading_data(name="ast inspection", points=1)
def test_source_can_be_parsed_and_inspected(sandbox: StudentFixture) -> None:
tree = ast.parse(sandbox.student_source())

function_names = {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)}
assert "combine" in function_names

uses_matmul = any(isinstance(node, ast.BinOp) and isinstance(node.op, ast.MatMult) for node in ast.walk(tree))
assert uses_matmul
2 changes: 2 additions & 0 deletions tests/scenario_root/test_student_source/student_code.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def combine(a, b):
return a @ b
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"expected_data_object": {
"score": 1.0,
"tests": [
{
"max_points": 1,
"message": "",
"name": "notebook source",
"outcome": "passed",
"points": 1.0,
"points_frac": 1.0,
"test_id": "test_student_source_notebook.py::test_notebook_source_is_graded_cells[student_code]"
}
]
},
"expected_outcome": {
"passed": 1,
"failed": 0
}
}
18 changes: 18 additions & 0 deletions tests/scenario_root/test_student_source_notebook/scenario.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
"""student_source() on a notebook returns the graded cells' code, not scratch cells."""

import pytest

from pytest_prairielearn_grader import ConfigObject
from pytest_prairielearn_grader.fixture import StudentFixture

autograder_config = ConfigObject(
student_code_pattern="student_code.ipynb",
notebook_cell_tag="#grade",
)


@pytest.mark.grading_data(name="notebook source", points=1)
def test_notebook_source_is_graded_cells(sandbox: StudentFixture) -> None:
source = sandbox.student_source()
assert "def solution():" in source
assert "SHOULD_NOT_APPEAR" not in source
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "cell0",
"metadata": {},
"outputs": [],
"source": [
"scratch_marker = \"SHOULD_NOT_APPEAR\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cell1",
"metadata": {},
"outputs": [],
"source": [
"#grade\n",
"def solution():\n",
" return 42"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading