Repository: pre-commit/pre-commit
Pull request #1643 (2020-10-14 17:05:22)

ENH add --negate flag to pygrep
xref this comment on StackOverflow: https://stackoverflow.com/a/64344519/9988333

>  A PR may be considered if you can demonstrate a real use case for it

A use-case comes from `PyMC3`, where there is a pre-commit hook to check that all notebooks have a watermark. Currently, this is done by defining
```
- repo: local
  hooks:
    - id: watermark
      name: Check notebooks have watermark
      types: [jupyter]
      entry: python scripts/check_watermark.py
      language: python
```
in https://github.com/pymc-devs/pymc3/blob/master/.pre-commit-config.yaml, and then
```python
"""
Check that given Jupyter notebooks all contain a final watermark cell to facilite reproducibility.
This is intended to be used as a pre-commit hook, see `.pre-commit-config.yaml`.
You can run it manually with `pre-commit run watermark --all`.
"""

import argparse
from pathlib import Path
import re

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("filenames", nargs="*")
    args = parser.parse_args()
    for file_ in args.filenames:
        assert (
            re.search(
                r"%load_ext watermark.*%watermark -n -u -v -iv -w",
                Path(file_).read_text(),
                flags=re.DOTALL,
            )
            is not None
        ), (
            f"Watermark not found in {file_} - please see the PyMC3 Jupyter Notebook Style guide:\n"
            "https://github.com/pymc-devs/pymc3/wiki/PyMC's-Jupyter-Notebook-Style"
        )
```
in https://github.com/pymc-devs/pymc3/blob/master/scripts/check_watermark.py .

----

I think the output should be a file which does _not_ contain the pattern...I'll work on this tomorrow

MarcoGorelli: Marking as ready for review as I'd like to think this is enough to tell whether it's the right direction / whether the feature would be welcome. Thanks for the 'don't put logic in tests' link!

Patch:
diff --git a/pre_commit/languages/pygrep.py b/pre_commit/languages/pygrep.py
index 40adba0..c80d679 100644
--- a/pre_commit/languages/pygrep.py
+++ b/pre_commit/languages/pygrep.py
@@ -1,6 +1,7 @@
 import argparse
 import re
 import sys
+from typing import NamedTuple
 from typing import Optional
 from typing import Pattern
 from typing import Sequence
@@ -45,6 +46,46 @@ def _process_filename_at_once(pattern: Pattern[bytes], filename: str) -> int:
     return retv
 
 
+def _process_filename_by_line_negated(
+        pattern: Pattern[bytes],
+        filename: str,
+) -> int:
+    with open(filename, 'rb') as f:
+        for line in f:
+            if pattern.search(line):
+                return 0
+        else:
+            output.write_line(filename)
+            return 1
+
+
+def _process_filename_at_once_negated(
+        pattern: Pattern[bytes],
+        filename: str,
+) -> int:
+    with open(filename, 'rb') as f:
+        contents = f.read()
+    match = pattern.search(contents)
+    if match:
+        return 0
+    else:
+        output.write_line(filename)
+        return 1
+
+
+class Choice(NamedTuple):
+    multiline: bool
+    negate: bool
+
+
+FNS = {
+    Choice(multiline=True, negate=True): _process_filename_at_once_negated,
+    Choice(multiline=True, negate=False): _process_filename_at_once,
+    Choice(multiline=False, negate=True): _process_filename_by_line_negated,
+    Choice(multiline=False, negate=False): _process_filename_by_line,
+}
+
+
 def run_hook(
         hook: Hook,
         file_args: Sequence[str],
@@ -64,6 +105,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     )
     parser.add_argument('-i', '--ignore-case', action='store_true')
     parser.add_argument('--multiline', action='store_true')
+    parser.add_argument('--negate', action='store_true')
     parser.add_argument('pattern', help='python regex pattern.')
     parser.add_argument('filenames', nargs='*')
     args = parser.parse_args(argv)
@@ -75,11 +117,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     pattern = re.compile(args.pattern.encode(), flags)
 
     retv = 0
+    process_fn = FNS[Choice(multiline=args.multiline, negate=args.negate)]
     for filename in args.filenames:
-        if args.multiline:
-            retv |= _process_filename_at_once(pattern, filename)
-        else:
-            retv |= _process_filename_by_line(pattern, filename)
+        retv |= process_fn(pattern, filename)
     return retv
 
 


Test patch:
diff --git a/tests/languages/pygrep_test.py b/tests/languages/pygrep_test.py
index 6eef56b..d8bacc4 100644
--- a/tests/languages/pygrep_test.py
+++ b/tests/languages/pygrep_test.py
@@ -8,6 +8,9 @@ def some_files(tmpdir):
     tmpdir.join('f1').write_binary(b'foo\nbar\n')
     tmpdir.join('f2').write_binary(b'[INFO] hi\n')
     tmpdir.join('f3').write_binary(b"with'quotes\n")
+    tmpdir.join('f4').write_binary(b'foo\npattern\nbar\n')
+    tmpdir.join('f5').write_binary(b'[INFO] hi\npattern\nbar')
+    tmpdir.join('f6').write_binary(b"pattern\nbarwith'foo\n")
     with tmpdir.as_cwd():
         yield
 
@@ -30,6 +33,58 @@ def test_main(cap_out, pattern, expected_retcode, expected_out):
     assert out == expected_out
 
 
+@pytest.mark.usefixtures('some_files')
+def test_negate_by_line_no_match(cap_out):
+    ret = pygrep.main(('pattern\nbar', 'f4', 'f5', 'f6', '--negate'))
+    out = cap_out.get()
+    assert ret == 1
+    assert out == 'f4\nf5\nf6\n'
+
+
+@pytest.mark.usefixtures('some_files')
+def test_negate_by_line_two_match(cap_out):
+    ret = pygrep.main(('foo', 'f4', 'f5', 'f6', '--negate'))
+    out = cap_out.get()
+    assert ret == 1
+    assert out == 'f5\n'
+
+
+@pytest.mark.usefixtures('some_files')
+def test_negate_by_line_all_match(cap_out):
+    ret = pygrep.main(('pattern', 'f4', 'f5', 'f6', '--negate'))
+    out = cap_out.get()
+    assert ret == 0
+    assert out == ''
+
+
+@pytest.mark.usefixtures('some_files')
+def test_negate_by_file_no_match(cap_out):
+    ret = pygrep.main(('baz', 'f4', 'f5', 'f6', '--negate', '--multiline'))
+    out = cap_out.get()
+    assert ret == 1
+    assert out == 'f4\nf5\nf6\n'
+
+
+@pytest.mark.usefixtures('some_files')
+def test_negate_by_file_one_match(cap_out):
+    ret = pygrep.main(
+        ('foo\npattern', 'f4', 'f5', 'f6', '--negate', '--multiline'),
+    )
+    out = cap_out.get()
+    assert ret == 1
+    assert out == 'f5\nf6\n'
+
+
+@pytest.mark.usefixtures('some_files')
+def test_negate_by_file_all_match(cap_out):
+    ret = pygrep.main(
+        ('pattern\nbar', 'f4', 'f5', 'f6', '--negate', '--multiline'),
+    )
+    out = cap_out.get()
+    assert ret == 0
+    assert out == ''
+
+
 @pytest.mark.usefixtures('some_files')
 def test_ignore_case(cap_out):
     ret = pygrep.main(('--ignore-case', 'info', 'f1', 'f2', 'f3'))