Repository: r1chardj0n3s/pip-check-reqs

Issue:
"log.debug('found requirement: %s', requirement.name)"
Since a recent update to pip, I see this:

```
Traceback (most recent call last):
283  File "/home/travis/virtualenv/python3.6.7/bin/pip-extra-reqs", line 8, in <module>
284    sys.exit(main())
285  File "/home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages/pip_check_reqs/find_extra_reqs.py", line 103, in main
286    extras = find_extra_reqs(options)
287  File "/home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages/pip_check_reqs/find_extra_reqs.py", line 52, in find_extra_reqs
288    explicit = common.find_required_modules(options)
289  File "/home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages/pip_check_reqs/common.py", line 132, in find_required_modules
290    log.debug('found requirement: %s', requirement.name)
291AttributeError: 'ParsedRequirement' object has no attribute 'name'
292Makefile:35: recipe for target 'pip-extra-reqs' failed
293make: *** [pip-extra-reqs] Error 1
294The command "make lint" exited with 2.
```

Discussion:

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b277842..8079ba9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -7,6 +7,10 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    # Run at 1:00 every day
+    - cron:  '0 1 * * *'
 
 jobs:
   build:
@@ -16,8 +20,9 @@ jobs:
     strategy:
       matrix:
         # These versions match the minimum and maximum versions of pip in
-        # requirements.txt
-        pip-version: ['==10.0.1', '<= 19']
+        # requirements.txt.
+        # An empty string here represents the latest version.
+        pip-version: ['==10.0.1', '']
         python-version: [3.6, 3.7, 3.8]
 
     steps:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index e6745ea..5290723 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -7,7 +7,8 @@ Release History
 - Remove support for Python 2.
   Please use an older version of this tool if you require that support.
 - Remove requirement for setuptools.
-- Add restriction on the pip version to install to match the pip versions which work.
+- Support newer versions of pip, including the current version, for more features (20.1.1).
+  Thanks to @Czaki for important parts of this change.
 
 2.0.1
 
diff --git a/pip_check_reqs/common.py b/pip_check_reqs/common.py
index dc01058..6b6e370 100644
--- a/pip_check_reqs/common.py
+++ b/pip_check_reqs/common.py
@@ -6,7 +6,11 @@ import os
 import re
 
 from packaging.utils import canonicalize_name
-from pip._internal.download import PipSession
+# Between different versions of pip the location of PipSession has changed.
+try:
+    from pip._internal.network.session import PipSession
+except ImportError:  # pragma: no cover
+    from pip._internal.download import PipSession
 from pip._internal.req.req_file import parse_requirements
 
 log = logging.getLogger(__name__)
@@ -121,15 +125,26 @@ def find_imported_modules(options):
     return vis.finalise()
 
 
-def find_required_modules(options):
+def find_required_modules(options, requirements_filename: str):
     explicit = set()
-    for requirement in parse_requirements('requirements.txt',
+    for requirement in parse_requirements(requirements_filename,
                                           session=PipSession()):
+        try:
+            requirement_name = requirement.name
+        # The type of "requirement" changed between pip versions.
+        # We exclude the "except" from coverage so that on any pip version we
+        # can report 100% coverage.
+        except AttributeError:  # pragma: no cover
+            from pip._internal.req.constructors import install_req_from_line
+            requirement_name = install_req_from_line(
+                requirement.requirement,
+            ).name
+
         if options.ignore_reqs(requirement):
-            log.debug('ignoring requirement: %s', requirement.name)
+            log.debug('ignoring requirement: %s', requirement_name)
         else:
-            log.debug('found requirement: %s', requirement.name)
-            explicit.add(canonicalize_name(requirement.name))
+            log.debug('found requirement: %s', requirement_name)
+            explicit.add(canonicalize_name(requirement_name))
     return explicit
 
 
@@ -149,9 +164,22 @@ def ignorer(ignore_cfg):
 
     def f(candidate, ignore_cfg=ignore_cfg):
         for ignore in ignore_cfg:
-            if fnmatch.fnmatch(candidate, ignore):
+            try:
+                from pip._internal.req.constructors import (
+                    install_req_from_line,
+                )
+                candidate_path = install_req_from_line(  # pragma: no cover
+                    candidate.requirement,
+                ).name
+            except (ImportError, AttributeError):
+                try:
+                    candidate_path = candidate.name
+                except AttributeError:
+                    candidate_path = candidate
+
+            if fnmatch.fnmatch(candidate_path, ignore):
                 return True
-            elif fnmatch.fnmatch(os.path.relpath(candidate), ignore):
+            elif fnmatch.fnmatch(os.path.relpath(candidate_path), ignore):
                 return True
         return False
 
diff --git a/pip_check_reqs/find_extra_reqs.py b/pip_check_reqs/find_extra_reqs.py
index 6b25e29..5549d89 100644
--- a/pip_check_reqs/find_extra_reqs.py
+++ b/pip_check_reqs/find_extra_reqs.py
@@ -12,7 +12,7 @@ from pip_check_reqs import common
 log = logging.getLogger(__name__)
 
 
-def find_extra_reqs(options):
+def find_extra_reqs(options, requirements_filename):
     # 1. find files used by imports in the code (as best we can without
     #    executing)
     used_modules = common.find_imported_modules(options)
@@ -50,7 +50,10 @@ def find_extra_reqs(options):
                 modname, info.filename)
 
     # 4. compare with requirements.txt
-    explicit = common.find_required_modules(options)
+    explicit = common.find_required_modules(
+        options=options,
+        requirements_filename=requirements_filename,
+    )
 
     return [name for name in explicit if name not in used]
 
@@ -121,12 +124,20 @@ def main():
 
     log.info('using pip_check_reqs-%s from %s', __version__, __file__)
 
-    extras = find_extra_reqs(options)
+    requirements_filename = 'requirements.txt'
+    extras = find_extra_reqs(
+        options=options,
+        requirements_filename=requirements_filename,
+    )
 
     if extras:
         log.warning('Extra requirements:')
     for name in extras:
-        log.warning('%s in requirements.txt' % name)
+        message = '{name} in {requirements_filename}'.format(
+            name=name,
+            requirements_filename=requirements_filename,
+        )
+        log.warning(message)
 
     if extras:
         sys.exit(1)
diff --git a/pip_check_reqs/find_missing_reqs.py b/pip_check_reqs/find_missing_reqs.py
index d6ca5bf..a20296a 100644
--- a/pip_check_reqs/find_missing_reqs.py
+++ b/pip_check_reqs/find_missing_reqs.py
@@ -6,7 +6,11 @@ import sys
 
 from packaging.utils import canonicalize_name
 from pip._internal.commands.show import search_packages_info
-from pip._internal.download import PipSession
+# Between different versions of pip the location of PipSession has changed.
+try:
+    from pip._internal.network.session import PipSession
+except ImportError:  # pragma: no cover
+    from pip._internal.download import PipSession
 from pip._internal.req.req_file import parse_requirements
 from pip._internal.utils.misc import get_installed_distributions
 
@@ -15,7 +19,7 @@ from pip_check_reqs import common
 log = logging.getLogger(__name__)
 
 
-def find_missing_reqs(options):
+def find_missing_reqs(options, requirements_filename):
     # 1. find files used by imports in the code (as best we can without
     #    executing)
     used_modules = common.find_imported_modules(options)
@@ -54,10 +58,23 @@ def find_missing_reqs(options):
 
     # 4. compare with requirements.txt
     explicit = set()
-    for requirement in parse_requirements('requirements.txt',
-                                          session=PipSession()):
-        log.debug('found requirement: %s', requirement.name)
-        explicit.add(canonicalize_name(requirement.name))
+    for requirement in parse_requirements(
+        requirements_filename,
+        session=PipSession(),
+    ):
+        try:
+            requirement_name = requirement.name
+        # The type of "requirement" changed between pip versions.
+        # We exclude the "except" from coverage so that on any pip version we
+        # can report 100% coverage.
+        except AttributeError:  # pragma: no cover
+            from pip._internal.req.constructors import install_req_from_line
+            requirement_name = install_req_from_line(
+                requirement.requirement,
+            ).name
+
+        log.debug('found requirement: %s', requirement_name)
+        explicit.add(canonicalize_name(requirement_name))
 
     return [(name, used[name]) for name in used if name not in explicit]
 
@@ -121,7 +138,11 @@ def main():
 
     log.info('using pip_check_reqs-%s from %s', __version__, __file__)
 
-    missing = find_missing_reqs(options)
+    requirements_filename = 'requirements.txt'
+    missing = find_missing_reqs(
+        options=options,
+        requirements_filename=requirements_filename,
+    )
 
     if missing:
         log.warning('Missing requirements:')
diff --git a/requirements.txt b/requirements.txt
index 1c48781..455f26e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
 packaging
-# These versions are matched in the GitHub workflows CI.
-pip >= 10.0.1, <= 19
+# Pinned pip versions are matched in the GitHub workflows CI matrix.
+pip >= 10.0.1


Test patch:
diff --git a/tests/test_common.py b/tests/test_common.py
index 8da2d07..4c58fe6 100644
--- a/tests/test_common.py
+++ b/tests/test_common.py
@@ -1,9 +1,9 @@
 from __future__ import absolute_import
 
 import ast
-import collections
 import logging
 import os.path
+from pathlib import Path
 
 import pytest
 import pretend
@@ -157,25 +157,23 @@ def test_find_imported_modules(monkeypatch, caplog, ignore_ham, ignore_hashlib,
     (['spam*'], 'eggs', False),
     (['spam'], '/spam', True),
 ])
-def test_ignorer(monkeypatch, ignore_cfg, candidate, result):
+def test_ignorer(monkeypatch, tmp_path: Path, ignore_cfg, candidate, result):
     monkeypatch.setattr(os.path, 'relpath', lambda s: s.lstrip('/'))
     ignorer = common.ignorer(ignore_cfg)
     assert ignorer(candidate) == result
 
 
-def test_find_required_modules(monkeypatch):
+def test_find_required_modules(monkeypatch, tmp_path: Path):
     class options:
-        @staticmethod
-        def ignore_reqs(req):
-            if req.name == 'barfoo':
-                return True
-            return False
+        pass
+
+    options.ignore_reqs = common.ignorer(ignore_cfg=['barfoo'])
 
-    FakeReq = collections.namedtuple('FakeReq', ['name'])
-    requirements = [FakeReq('foobar'), FakeReq('barfoo')]
-    monkeypatch.setattr(
-        common, 'parse_requirements',
-        pretend.call_recorder(lambda a, session=None: requirements))
+    fake_requirements_file = tmp_path / 'requirements.txt'
+    fake_requirements_file.write_text('foobar==1\nbarfoo==2')
 
-    reqs = common.find_required_modules(options)
+    reqs = common.find_required_modules(
+        options=options,
+        requirements_filename=str(fake_requirements_file),
+    )
     assert reqs == set(['foobar'])
diff --git a/tests/test_find_extra_reqs.py b/tests/test_find_extra_reqs.py
index 05df45a..6d0c748 100644
--- a/tests/test_find_extra_reqs.py
+++ b/tests/test_find_extra_reqs.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import
 import collections
 import logging
 import optparse
+from pathlib import Path
 
 import pytest
 import pretend
@@ -37,7 +38,7 @@ def fake_opts():
     return FakeOptParse
 
 
-def test_find_extra_reqs(monkeypatch):
+def test_find_extra_reqs(monkeypatch, tmp_path: Path):
     imported_modules = dict(spam=common.FoundModule('spam',
                                                     'site-spam/spam.py',
                                                     [('ham.py', 1)]),
@@ -64,11 +65,8 @@ def test_find_extra_reqs(monkeypatch):
     monkeypatch.setattr(find_extra_reqs, 'search_packages_info',
                         pretend.call_recorder(lambda x: packages_info))
 
-    FakeReq = collections.namedtuple('FakeReq', ['name'])
-    requirements = [FakeReq('foobar')]
-    monkeypatch.setattr(
-        common, 'parse_requirements',
-        pretend.call_recorder(lambda a, session=None: requirements))
+    fake_requirements_file = tmp_path / 'requirements.txt'
+    fake_requirements_file.write_text('foobar==1')
 
     class options:
         def ignore_reqs(x, y):
@@ -76,7 +74,10 @@ def test_find_extra_reqs(monkeypatch):
 
     options = options()
 
-    result = find_extra_reqs.find_extra_reqs(options)
+    result = find_extra_reqs.find_extra_reqs(
+        options=options,
+        requirements_filename=str(fake_requirements_file),
+    )
     assert result == ['foobar']
 
 
@@ -86,7 +87,7 @@ def test_main_failure(monkeypatch, caplog, fake_opts):
     caplog.set_level(logging.WARN)
 
     monkeypatch.setattr(find_extra_reqs, 'find_extra_reqs',
-                        lambda x: ['extra'])
+                        lambda options, requirements_filename: ['extra'])
 
     with pytest.raises(SystemExit) as excinfo:
         find_extra_reqs.main()
@@ -145,7 +146,11 @@ def test_logging_config(monkeypatch, caplog, verbose_cfg, debug_cfg, result):
 
     monkeypatch.setattr(optparse, 'OptionParser', FakeOptParse)
 
-    monkeypatch.setattr(find_extra_reqs, 'find_extra_reqs', lambda x: [])
+    monkeypatch.setattr(
+        find_extra_reqs,
+        'find_extra_reqs',
+        lambda options, requirements_filename: [],
+    )
     find_extra_reqs.main()
 
     for event in [(logging.DEBUG, 'debug'), (logging.INFO, 'info'),
diff --git a/tests/test_find_missing_reqs.py b/tests/test_find_missing_reqs.py
index eb4aceb..b1ec7f4 100644
--- a/tests/test_find_missing_reqs.py
+++ b/tests/test_find_missing_reqs.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import
 import collections
 import logging
 import optparse
+from pathlib import Path
 
 import pytest
 import pretend
@@ -36,7 +37,7 @@ def fake_opts():
     return FakeOptParse
 
 
-def test_find_missing_reqs(monkeypatch):
+def test_find_missing_reqs(monkeypatch, tmp_path: Path):
     imported_modules = dict(spam=common.FoundModule('spam',
                                                     'site-spam/spam.py',
                                                     [('ham.py', 1)]),
@@ -63,13 +64,15 @@ def test_find_missing_reqs(monkeypatch):
     monkeypatch.setattr(find_missing_reqs, 'search_packages_info',
                         pretend.call_recorder(lambda x: packages_info))
 
-    FakeReq = collections.namedtuple('FakeReq', ['name'])
-    requirements = [FakeReq('spam')]
-    monkeypatch.setattr(
-        find_missing_reqs, 'parse_requirements',
-        pretend.call_recorder(lambda a, session=None: requirements))
+    fake_requirements_file = tmp_path / 'requirements.txt'
+    fake_requirements_file.write_text('spam==1')
 
-    result = list(find_missing_reqs.find_missing_reqs(None))
+    result = list(
+        find_missing_reqs.find_missing_reqs(
+            options=None,
+            requirements_filename=str(fake_requirements_file),
+        )
+    )
     assert result == [('shrub', [imported_modules['shrub']])]
 
 
@@ -78,10 +81,25 @@ def test_main_failure(monkeypatch, caplog, fake_opts):
 
     caplog.set_level(logging.WARN)
 
+    def fake_find_missing_reqs(options, requirements_filename):
+        return [
+            (
+                'missing',
+                [
+                    common.FoundModule(
+                        'missing',
+                        'missing.py',
+                        [('location.py', 1)],
+                    )
+                ]
+            )
+        ]
+
     monkeypatch.setattr(
-        find_missing_reqs, 'find_missing_reqs', lambda x: [('missing', [
-            common.FoundModule('missing', 'missing.py', [('location.py', 1)])
-        ])])
+        find_missing_reqs,
+        'find_missing_reqs',
+        fake_find_missing_reqs,
+    )
 
     with pytest.raises(SystemExit) as excinfo:
         find_missing_reqs.main()
@@ -139,7 +157,11 @@ def test_logging_config(monkeypatch, caplog, verbose_cfg, debug_cfg, result):
 
     monkeypatch.setattr(optparse, 'OptionParser', FakeOptParse)
 
-    monkeypatch.setattr(find_missing_reqs, 'find_missing_reqs', lambda x: [])
+    monkeypatch.setattr(
+        find_missing_reqs,
+        'find_missing_reqs',
+        lambda options, requirements_filename: [],
+    )
     find_missing_reqs.main()
 
     for event in [(logging.DEBUG, 'debug'), (logging.INFO, 'info'),