Repository: jupyter/nbgrader (language: python)

Issue:
New Autograde Preprocessor: IgnorePattern
Creating a new preprocessor, which is useful for removing specific warning cells that might result in a false-negative autograde result. This is used when the assignment is already out and released, but you notice a problem in a test after the fact. This provides a much easier way of recovering compared to editing the assignment.

We at @aaltoscicomp came across an issue where Torch raised a specific warning on stderr (due to some avx1 incompatibility in the node on which the grading would happen), and the auto-grader would mark the solution as wrong. Since the warning was not a Python Warning object (NNPACK.cpp warning), it cannot be suppressed the usual way. Since this warning can't be easily predicted in advance, the assignment couldn't be made to compensate for it.

We addressed the problem by inserting this preprocessor in the `c.Autograde.autograde_preprocessors` list immediately after `Execute` to detect and remove pre-defined warning (stderr) outputs from GradeCells. The module uses a configured regular expression (`c.IgnorePattern.pattern`) to find and delete the matching outputs before the grading process.  The default value is empty, which short-circuits the processing so has no effect (and anyway an empty value would have no effect by default).

It shouldn't be needed often, but when it's needed it will save a lot of time.

It has tests and all. We haven't used it live since we made it too late, but we definitely had a need for it.


Discussion / hints:
[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/AaltoSciComp/nbgrader/ignore-pattern?urlpath=lab) :point_left: Launch a Binder on branch _AaltoSciComp/nbgrader/ignore-pattern_

Fix patch:
diff --git a/nbgrader/converters/autograde.py b/nbgrader/converters/autograde.py
index b4ceac7ab..d3b4e8bac 100644
--- a/nbgrader/converters/autograde.py
+++ b/nbgrader/converters/autograde.py
@@ -9,7 +9,7 @@
 from .base import BaseConverter, NbGraderException
 from ..preprocessors import (
     AssignLatePenalties, ClearOutput, DeduplicateIds, OverwriteCells, SaveAutoGrades,
-    Execute, LimitOutput, OverwriteKernelspec, CheckCellMetadata)
+    Execute, LimitOutput, OverwriteKernelspec, CheckCellMetadata, IgnorePattern)
 from ..api import Gradebook, MissingEntry
 from .. import utils
 
@@ -61,6 +61,7 @@ def _output_directory(self) -> str:
     ]).tag(config=True)
     autograde_preprocessors = List([
         Execute,
+        IgnorePattern,
         ClearMetadataPreprocessor,
         LimitOutput,
         SaveAutoGrades,
diff --git a/nbgrader/preprocessors/__init__.py b/nbgrader/preprocessors/__init__.py
index 1507ee0aa..51fb5cf57 100644
--- a/nbgrader/preprocessors/__init__.py
+++ b/nbgrader/preprocessors/__init__.py
@@ -16,6 +16,7 @@
 from .clearhiddentests import ClearHiddenTests
 from .clearmarkingscheme import ClearMarkScheme
 from .overwritekernelspec import OverwriteKernelspec
+from .ignorepattern import IgnorePattern
 
 __all__ = [
     "AssignLatePenalties",
@@ -35,4 +36,5 @@
     "ClearHiddenTests",
     "ClearMarkScheme",
     "OverwriteKernelspec",
+    "IgnorePattern",
 ]
diff --git a/nbgrader/preprocessors/ignorepattern.py b/nbgrader/preprocessors/ignorepattern.py
new file mode 100644
index 000000000..546070b18
--- /dev/null
+++ b/nbgrader/preprocessors/ignorepattern.py
@@ -0,0 +1,31 @@
+from . import NbGraderPreprocessor
+
+from traitlets import Unicode, Bool
+from nbformat.notebooknode import NotebookNode
+from nbconvert.exporters.exporter import ResourcesDict
+from typing import Tuple
+import re
+
+
+class IgnorePattern(NbGraderPreprocessor):
+    """Preprocessor for removing cell outputs that match a particular pattern"""
+    
+    pattern = Unicode("", help="The regular expression to remove from stderr").tag(config=True)
+    enabled = Bool(False, help="Whether to use this preprocessor when running nbgrader").tag(config=True)
+    
+    def preprocess_cell(self,
+                        cell: NotebookNode,
+                        resources: ResourcesDict,
+                        cell_index: int
+                        ) -> Tuple[NotebookNode, ResourcesDict]:
+        
+        if self.pattern and cell.cell_type == "code":
+            new_outputs = []
+            for output in cell.outputs:
+                if output.output_type == "stream" and output.name == "stderr" \
+                    and re.search(self.pattern, output.text):
+                        continue
+                new_outputs.append(output)
+            cell.outputs = new_outputs
+                    
+        return cell, resources


Test patch:
diff --git a/nbgrader/tests/preprocessors/files/warning-pattern.ipynb b/nbgrader/tests/preprocessors/files/warning-pattern.ipynb
new file mode 100644
index 000000000..ffc91ea50
--- /dev/null
+++ b/nbgrader/tests/preprocessors/files/warning-pattern.ipynb
@@ -0,0 +1,82 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "efd8b463-ea80-4620-b01b-1382a56e7836",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[W NNPACK.cpp:64] Could not initialize NNPACK! Reason: Unsupported hardware.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "5"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "layer = torch.nn.Conv2d(1, 32, 3, stride=1, padding=1)\n",
+    "x = torch.randn(1, 1, 28, 28)\n",
+    "y = layer(x)\n",
+    "5"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2a2b6474-f64b-4c28-ac83-5d1c7bbfd92f",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_522/3731920106.py:3: UserWarning: This is a warning message\n",
+      "  warnings.warn(\"This is a warning message\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "import warnings\n",
+    "\n",
+    "warnings.warn(\"This is a warning message\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/nbgrader/tests/preprocessors/test_ignorepattern.py b/nbgrader/tests/preprocessors/test_ignorepattern.py
new file mode 100644
index 000000000..fc0a21479
--- /dev/null
+++ b/nbgrader/tests/preprocessors/test_ignorepattern.py
@@ -0,0 +1,40 @@
+import pytest
+import os
+
+from ...preprocessors import IgnorePattern
+from .base import BaseTestPreprocessor
+
+
+@pytest.fixture
+def preprocessor():
+    pp = IgnorePattern()
+    pp.pattern = r"\[.*\.cpp.*\] Could not initialize NNPACK! Reason: Unsupported hardware."
+    return pp
+
+
+class TestIgnorePattern(BaseTestPreprocessor):
+
+    def test_remove_matching_output(self, preprocessor):
+        nb = self._read_nb(os.path.join("files", "warning-pattern.ipynb"))
+        cell = nb.cells[0]
+        
+        outputs = cell.outputs
+        assert len(outputs) == 2
+
+        cell, _ = preprocessor.preprocess_cell(cell, {}, 0)
+
+        assert len(cell.outputs) == 1
+
+
+    def test_skip_nonmatching_output(self, preprocessor):
+        nb = self._read_nb(os.path.join("files", "warning-pattern.ipynb"))
+        cell = nb.cells[1]
+        
+        outputs = cell.outputs
+        assert len(outputs) == 1
+
+        cell, _ = preprocessor.preprocess_cell(cell, {}, 1)
+
+        assert len(cell.outputs) == 1
+        assert cell.outputs[0].name == "stderr"
+