Repository: pypa/pip

Issue:
Improve error message for invalid requirements files
A `pip3 install` command is failing with esoteric error message. I am running as the root user on Ubuntu.

* pip version: pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)
* Python version: Python 3.6.7
* OS: Ubuntu


I have this:

```Dockerfile
WORKDIR /apps/datavana/moodwire/mwapi
RUN mkdir -p /pth/to/downloaded/files
RUN mkdir -p /interos/temp
ARG commit

ADD https://bitbucket.org/interos/datavana/raw/"$commit"/moodwire/mwapi/requirements.txt  '/interos/temp/requirements.txt'
ADD https://bitbucket.org/interos/datavana/raw/"$commit"/moodwire/mwapi/constraints.txt '/interos/temp/constraints.txt'

# the ADD commands succeed

RUN pip3 install --cache-dir="/pth/to/downloaded/files" -r /interos/temp/requirements.txt -c /interos/temp/constraints.txt
```

I get this error:

```
Step 28/92 : RUN pip3 install --cache-dir="/pth/to/downloaded/files" -r /interos/temp/requirements.txt -c /interos/temp/constraints.txt
 ---> Running in ac48ca1a7f93
Invalid requirement: '<html>'
Traceback (most recent call last):
  File "/usr/share/python-wheels/packaging-17.1-py2.py3-none-any.whl/packaging/requirements.py", line 93, in __init__
    req = REQUIREMENT.parseString(requirement_string)
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 1632, in parseString
    raise exc
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 1622, in parseString
    loc, tokens = self._parse( instring, 0 )
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 1379, in _parseNoCache
    loc,tokens = self.parseImpl( instring, preloc, doActions )
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 3395, in parseImpl
    loc, exprtokens = e._parse( instring, loc, doActions )
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 1379, in _parseNoCache
    loc,tokens = self.parseImpl( instring, preloc, doActions )
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 3717, in parseImpl
    return self.expr._parse( instring, loc, doActions, callPreParse=False )
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 1379, in _parseNoCache
    loc,tokens = self.parseImpl( instring, preloc, doActions )
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 3378, in parseImpl
    loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False )
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 1383, in _parseNoCache
    loc,tokens = self.parseImpl( instring, preloc, doActions )
  File "/usr/share/python-wheels/pyparsing-2.2.0-py2.py3-none-any.whl/pyparsing.py", line 2689, in parseImpl
    raise ParseException(instring, loc, self.errmsg, self)
pyparsing.ParseException: Expected W:(abcd...) (at char 0), (line:1, col:1)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/pip/req/req_install.py", line 82, in __init__
    req = Requirement(req)
  File "/usr/share/python-wheels/packaging-17.1-py2.py3-none-any.whl/packaging/requirements.py", line 97, in __init__
    requirement_string[e.loc:e.loc + 8]))
pip._vendor.packaging.requirements.InvalidRequirement: Invalid requirement, parse error at "'<html>'"

The command '/bin/sh -c pip3 install --cache-dir="/pth/to/downloaded/files" -r /interos/temp/requirements.txt -c /interos/temp/constraints.txt' returned a non-zero code: 1

```

the error message is pretty bad - no matter what's going on.


Discussion:
My gut feeling is that this has nothing to do with pip, but the `ADD` commands are downloading something different from what you’re expecting. Probably an HTML page instead of the raw text file.
yeah it was auth problem with bitbucket, getting 403s and html instead of txt files. not sure if the error message could be improved much.
Note that issue #6513 is a similar issue regarding parsing requirement strings and the log message not containing enough info. However, it's still different (e.g. different code path).

Resolving patch:
diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py
--- a/src/pip/_internal/req/constructors.py
+++ b/src/pip/_internal/req/constructors.py
@@ -205,11 +205,15 @@ def install_req_from_line(
     isolated=False,  # type: bool
     options=None,  # type: Optional[Dict[str, Any]]
     wheel_cache=None,  # type: Optional[WheelCache]
-    constraint=False  # type: bool
+    constraint=False,  # type: bool
+    line_source=None,  # type: Optional[str]
 ):
     # type: (...) -> InstallRequirement
     """Creates an InstallRequirement from a name, which might be a
     requirement, directory containing 'setup.py', filename, or URL.
+
+    :param line_source: An optional string describing where the line is from,
+        for logging purposes in case of an error.
     """
     if is_url(name):
         marker_sep = '; '
@@ -289,10 +293,17 @@ def install_req_from_line(
                   not any(op in req_as_string for op in operators)):
                 add_msg = "= is not a valid operator. Did you mean == ?"
             else:
-                add_msg = ""
-            raise InstallationError(
-                "Invalid requirement: '%s'\n%s" % (req_as_string, add_msg)
+                add_msg = ''
+            if line_source is None:
+                source = ''
+            else:
+                source = ' (from {})'.format(line_source)
+            msg = (
+                'Invalid requirement: {!r}{}'.format(req_as_string, source)
             )
+            if add_msg:
+                msg += '\nHint: {}'.format(add_msg)
+            raise InstallationError(msg)
     else:
         req = None
 
diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py
--- a/src/pip/_internal/req/req_file.py
+++ b/src/pip/_internal/req/req_file.py
@@ -138,7 +138,7 @@ def process_line(
     session=None,  # type: Optional[PipSession]
     wheel_cache=None,  # type: Optional[WheelCache]
     use_pep517=None,  # type: Optional[bool]
-    constraint=False  # type: bool
+    constraint=False,  # type: bool
 ):
     # type: (...) -> Iterator[InstallRequirement]
     """Process a single requirements line; This can result in creating/yielding
@@ -187,10 +187,16 @@ def process_line(
         for dest in SUPPORTED_OPTIONS_REQ_DEST:
             if dest in opts.__dict__ and opts.__dict__[dest]:
                 req_options[dest] = opts.__dict__[dest]
+        line_source = 'line {} of {}'.format(line_number, filename)
         yield install_req_from_line(
-            args_str, line_comes_from, constraint=constraint,
+            args_str,
+            comes_from=line_comes_from,
             use_pep517=use_pep517,
-            isolated=isolated, options=req_options, wheel_cache=wheel_cache
+            isolated=isolated,
+            options=req_options,
+            wheel_cache=wheel_cache,
+            constraint=constraint,
+            line_source=line_source,
         )
 
     # yield an editable requirement


Test patch:
diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py
--- a/tests/unit/test_req.py
+++ b/tests/unit/test_req.py
@@ -530,7 +530,7 @@ def test_unidentifiable_name(self):
         with pytest.raises(InstallationError) as e:
             install_req_from_line(test_name)
         err_msg = e.value.args[0]
-        assert ("Invalid requirement: '%s'\n" % test_name) == err_msg
+        assert "Invalid requirement: '{}'".format(test_name) == err_msg
 
     def test_requirement_file(self):
         req_file_path = os.path.join(self.tempdir, 'test.txt')
diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py
--- a/tests/unit/test_req_file.py
+++ b/tests/unit/test_req_file.py
@@ -190,6 +190,26 @@ def test_only_one_req_per_line(self):
         with pytest.raises(InstallationError):
             list(process_line("req1 req2", "file", 1))
 
+    def test_error_message(self):
+        """
+        Test the error message if a parsing error occurs (all of path,
+        line number, and hint).
+        """
+        iterator = process_line(
+            'my-package=1.0',
+            filename='path/requirements.txt',
+            line_number=3
+        )
+        with pytest.raises(InstallationError) as exc:
+            list(iterator)
+
+        expected = (
+            "Invalid requirement: 'my-package=1.0' "
+            '(from line 3 of path/requirements.txt)\n'
+            'Hint: = is not a valid operator. Did you mean == ?'
+        )
+        assert str(exc.value) == expected
+
     def test_yield_line_requirement(self):
         line = 'SomeProject'
         filename = 'filename'