From 18862a5da49d20820fc11ae19d9d2524cd4ec103 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:54:17 +0530 Subject: [PATCH] DOC Count fairness categories with get_experiment_summary in the benchmark notebook The notebook flattened each experiment's score_category, which is the string "unknown" for an unscored experiment, so that experiment became rows for the letters u, n, k, o and w, and an experiment scored without a category was dropped. It now reads score_distribution from get_experiment_summary, which counts both as "unknown". Fixes #3674 Co-Authored-By: Claude Opus 5.5 --- doc/code/executor/6_benchmark.ipynb | 4 ++-- doc/code/executor/6_benchmark.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/code/executor/6_benchmark.ipynb b/doc/code/executor/6_benchmark.ipynb index ccf116c61a..184bd05026 100644 --- a/doc/code/executor/6_benchmark.ipynb +++ b/doc/code/executor/6_benchmark.ipynb @@ -1109,8 +1109,8 @@ " # Analyze pronoun distribution\n", " context = fairness_benchmark.get_last_context()\n", " if context:\n", - " score_categories = [cat for r in context.experiment_results for cat in r[\"score_category\"]]\n", - " for category, count in Counter(score_categories).items():\n", + " summary = fairness_benchmark.get_experiment_summary(context=context)\n", + " for category, count in summary[\"score_distribution\"].items():\n", " results_summary.append(\n", " {\n", " \"Story Type\": story_type,\n", diff --git a/doc/code/executor/6_benchmark.py b/doc/code/executor/6_benchmark.py index 8b481aaf10..8737bf2c46 100644 --- a/doc/code/executor/6_benchmark.py +++ b/doc/code/executor/6_benchmark.py @@ -231,8 +231,8 @@ # Analyze pronoun distribution context = fairness_benchmark.get_last_context() if context: - score_categories = [cat for r in context.experiment_results for cat in r["score_category"]] - for category, count in Counter(score_categories).items(): + summary = fairness_benchmark.get_experiment_summary(context=context) + for category, count in summary["score_distribution"].items(): results_summary.append( { "Story Type": story_type,