Repository: python-poetry/poetry

Issue:
Poetry 1.1.2 does not export dependency url for nested git dependencies
<!--
  Hi there! Thank you for discovering and submitting an issue.

  Before you submit this; let's make sure of a few things.
  Please make sure the following boxes are ticked if they are correct.
  If not, please try and fulfill these first.
-->

<!-- Checked checkbox should look like this: [x] -->
- [x] I am on the [latest](https://github.com/python-poetry/poetry/releases/latest) Poetry version.
- [x] I have searched the [issues](https://github.com/python-poetry/poetry/issues) of this repo and believe that this is not a duplicate.
- [ ] If an exception occurs when executing a command, I executed it again in debug mode (`-vvv` option).

<!--
  Once those are done, if you're able to fill in the following list with your information,
  it'd be very helpful to whoever handles the issue.
-->

- **OS version and name**: Ubuntu 16.04
- **Poetry version**: 1.1.2
- **Link of a [Gist](https://gist.github.com/) with the contents of your pyproject.toml file**: https://gist.github.com/marns93/45bdf3adce4b457a35dc481b55fc219d

## Issue
<!-- Now feel free to write your issue, but please be descriptive! Thanks again 🙌 ❤️ -->

Regarding to the PR https://github.com/python-poetry/poetry/pull/3089 the dependencies are not exported correctly.
The exception does not occur anymore, but the git url of the nested git dependency is missing in the requirements.txt.

```
$ poetry export -f requirements.txt --without-hashes
flake8==3.8.4
mccabe==0.6.1
peppercorn==0.6
pycodestyle==2.6.0
pyflakes==2.2.0
sampleproject-mm @ git+ssh://git@github.com/moneymeets/sampleproject.git@83068b995c2f94b0d51333b932b1a939084e9ef7
sampleproject==2.0.0
```

The output of the export command in the sampleproject repository is as expected.

```
poetry export -f requirements.txt --without-hashes
flake8==3.8.4
mccabe==0.6.1
peppercorn==0.6
pycodestyle==2.6.0
pyflakes==2.2.0
sampleproject @ git+ssh://git@github.com/pypa/sampleproject.git@52966defd6a61e97295b0bb82cd3474ac3e11c7a

```

You can reproduce this issue with the linked pyproject.toml.



Discussion:
@abn Could you please have a look?

Resolving patch:
diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py
--- a/poetry/packages/locker.py
+++ b/poetry/packages/locker.py
@@ -1,4 +1,3 @@
-import itertools
 import json
 import logging
 import os
@@ -6,9 +5,15 @@
 
 from copy import deepcopy
 from hashlib import sha256
-from typing import Any
+from typing import Dict
+from typing import Iterable
+from typing import Iterator
 from typing import List
 from typing import Optional
+from typing import Sequence
+from typing import Set
+from typing import Tuple
+from typing import Union
 
 from tomlkit import array
 from tomlkit import document
@@ -25,8 +30,10 @@
 from poetry.core.semver.version import Version
 from poetry.core.toml.file import TOMLFile
 from poetry.core.version.markers import parse_marker
+from poetry.packages import DependencyPackage
 from poetry.utils._compat import OrderedDict
 from poetry.utils._compat import Path
+from poetry.utils.extras import get_extra_package_names
 
 
 logger = logging.getLogger(__name__)
@@ -181,40 +188,118 @@ def locked_repository(
 
         return packages
 
-    def get_project_dependencies(
-        self, project_requires, pinned_versions=False, with_nested=False, with_dev=False
-    ):  # type: (List[Dependency], bool, bool, bool) -> Any
-        packages = self.locked_repository(with_dev).packages
+    @staticmethod
+    def __get_locked_package(
+        _dependency, packages_by_name
+    ):  # type: (Dependency, Dict[str, List[Package]]) -> Optional[Package]
+        """
+        Internal helper to identify corresponding locked package using dependency
+        version constraints.
+        """
+        for _package in packages_by_name.get(_dependency.name, []):
+            if _dependency.constraint.allows(_package.version):
+                return _package
+        return None
+
+    @classmethod
+    def __walk_dependency_level(
+        cls,
+        dependencies,
+        level,
+        pinned_versions,
+        packages_by_name,
+        project_level_dependencies,
+        nested_dependencies,
+    ):  # type: (List[Dependency], int,  bool, Dict[str, List[Package]], Set[str], Dict[Tuple[str, str], Dependency]) -> Dict[Tuple[str, str], Dependency]
+        if not dependencies:
+            return nested_dependencies
+
+        next_level_dependencies = []
+
+        for requirement in dependencies:
+            locked_package = cls.__get_locked_package(requirement, packages_by_name)
+
+            if locked_package:
+                for require in locked_package.requires:
+                    if require.marker.is_empty():
+                        require.marker = requirement.marker
+                    else:
+                        require.marker = require.marker.intersect(requirement.marker)
+
+                    require.marker = require.marker.intersect(locked_package.marker)
+                    next_level_dependencies.append(require)
+
+            if requirement.name in project_level_dependencies and level == 0:
+                # project level dependencies take precedence
+                continue
+
+            if locked_package:
+                # create dependency from locked package to retain dependency metadata
+                # if this is not done, we can end-up with incorrect nested dependencies
+                marker = requirement.marker
+                requirement = locked_package.to_dependency()
+                requirement.marker = requirement.marker.intersect(marker)
+            else:
+                # we make a copy to avoid any side-effects
+                requirement = deepcopy(requirement)
+
+            if pinned_versions:
+                requirement.set_constraint(
+                    cls.__get_locked_package(requirement, packages_by_name)
+                    .to_dependency()
+                    .constraint
+                )
 
-        # group packages entries by name, this is required because requirement might use
-        # different constraints
+            # dependencies use extra to indicate that it was activated via parent
+            # package's extras, this is not required for nested exports as we assume
+            # the resolver already selected this dependency
+            requirement.marker = requirement.marker.without_extras()
+
+            key = (requirement.name, requirement.pretty_constraint)
+            if key not in nested_dependencies:
+                nested_dependencies[key] = requirement
+            else:
+                nested_dependencies[key].marker = nested_dependencies[
+                    key
+                ].marker.intersect(requirement.marker)
+
+        return cls.__walk_dependency_level(
+            dependencies=next_level_dependencies,
+            level=level + 1,
+            pinned_versions=pinned_versions,
+            packages_by_name=packages_by_name,
+            project_level_dependencies=project_level_dependencies,
+            nested_dependencies=nested_dependencies,
+        )
+
+    @classmethod
+    def get_project_dependencies(
+        cls, project_requires, locked_packages, pinned_versions=False, with_nested=False
+    ):  # type: (List[Dependency], List[Package], bool, bool) -> Iterable[Dependency]
+        # group packages entries by name, this is required because requirement might use different constraints
         packages_by_name = {}
-        for pkg in packages:
+        for pkg in locked_packages:
             if pkg.name not in packages_by_name:
                 packages_by_name[pkg.name] = []
             packages_by_name[pkg.name].append(pkg)
 
-        def __get_locked_package(
-            _dependency,
-        ):  # type: (Dependency) -> Optional[Package]
-            """
-            Internal helper to identify corresponding locked package using dependency
-            version constraints.
-            """
-            for _package in packages_by_name.get(_dependency.name, []):
-                if _dependency.constraint.allows(_package.version):
-                    return _package
-            return None
-
         project_level_dependencies = set()
         dependencies = []
 
         for dependency in project_requires:
             dependency = deepcopy(dependency)
-            if pinned_versions:
-                locked_package = __get_locked_package(dependency)
-                if locked_package:
-                    dependency.set_constraint(locked_package.to_dependency().constraint)
+            locked_package = cls.__get_locked_package(dependency, packages_by_name)
+            if locked_package:
+                locked_dependency = locked_package.to_dependency()
+                locked_dependency.marker = dependency.marker.intersect(
+                    locked_package.marker
+                )
+
+                if not pinned_versions:
+                    locked_dependency.set_constraint(dependency.constraint)
+
+                dependency = locked_dependency
+
             project_level_dependencies.add(dependency.name)
             dependencies.append(dependency)
 
@@ -222,46 +307,74 @@ def __get_locked_package(
             # return only with project level dependencies
             return dependencies
 
-        nested_dependencies = list()
+        nested_dependencies = cls.__walk_dependency_level(
+            dependencies=dependencies,
+            level=0,
+            pinned_versions=pinned_versions,
+            packages_by_name=packages_by_name,
+            project_level_dependencies=project_level_dependencies,
+            nested_dependencies=dict(),
+        )
 
-        for pkg in packages:  # type: Package
-            for requirement in pkg.requires:  # type: Dependency
-                if requirement.name in project_level_dependencies:
-                    # project level dependencies take precedence
-                    continue
+        # Merge same dependencies using marker union
+        for requirement in dependencies:
+            key = (requirement.name, requirement.pretty_constraint)
+            if key not in nested_dependencies:
+                nested_dependencies[key] = requirement
+            else:
+                nested_dependencies[key].marker = nested_dependencies[key].marker.union(
+                    requirement.marker
+                )
 
-                # we make a copy to avoid any side-effects
-                requirement = deepcopy(requirement)
-                requirement._category = pkg.category
+        return sorted(nested_dependencies.values(), key=lambda x: x.name.lower())
 
-                if pinned_versions:
-                    requirement.set_constraint(
-                        __get_locked_package(requirement).to_dependency().constraint
-                    )
+    def get_project_dependency_packages(
+        self, project_requires, dev=False, extras=None
+    ):  # type: (List[Dependency], bool, Optional[Union[bool, Sequence[str]]]) -> Iterator[DependencyPackage]
+        repository = self.locked_repository(with_dev_reqs=dev)
 
-                # dependencies use extra to indicate that it was activated via parent
-                # package's extras
-                marker = requirement.marker.without_extras()
-                for project_requirement in project_requires:
-                    if (
-                        pkg.name == project_requirement.name
-                        and project_requirement.constraint.allows(pkg.version)
-                    ):
-                        requirement.marker = marker.intersect(
-                            project_requirement.marker
-                        )
-                        break
-                else:
-                    # this dependency was not from a project requirement
-                    requirement.marker = marker.intersect(pkg.marker)
+        # Build a set of all packages required by our selected extras
+        extra_package_names = (
+            None if (isinstance(extras, bool) and extras is True) else ()
+        )
 
-                if requirement not in nested_dependencies:
-                    nested_dependencies.append(requirement)
+        if extra_package_names is not None:
+            extra_package_names = set(
+                get_extra_package_names(
+                    repository.packages, self.lock_data.get("extras", {}), extras or (),
+                )
+            )
 
-        return sorted(
-            itertools.chain(dependencies, nested_dependencies),
-            key=lambda x: x.name.lower(),
-        )
+        # If a package is optional and we haven't opted in to it, do not select
+        selected = []
+        for dependency in project_requires:
+            try:
+                package = repository.find_packages(dependency=dependency)[0]
+            except IndexError:
+                continue
+
+            if extra_package_names is not None and (
+                package.optional and package.name not in extra_package_names
+            ):
+                # a package is locked as optional, but is not activated via extras
+                continue
+
+            selected.append(dependency)
+
+        for dependency in self.get_project_dependencies(
+            project_requires=selected,
+            locked_packages=repository.packages,
+            with_nested=True,
+        ):
+            try:
+                package = repository.find_packages(dependency=dependency)[0]
+            except IndexError:
+                continue
+
+            for extra in dependency.extras:
+                package.requires_extras.append(extra)
+
+            yield DependencyPackage(dependency=dependency, package=package)
 
     def set_lock_data(self, root, packages):  # type: (...) -> bool
         files = table()
diff --git a/poetry/utils/exporter.py b/poetry/utils/exporter.py
--- a/poetry/utils/exporter.py
+++ b/poetry/utils/exporter.py
@@ -1,3 +1,5 @@
+from typing import Optional
+from typing import Sequence
 from typing import Union
 
 from clikit.api.io import IO
@@ -5,7 +7,6 @@
 from poetry.poetry import Poetry
 from poetry.utils._compat import Path
 from poetry.utils._compat import decode
-from poetry.utils.extras import get_extra_package_names
 
 
 class Exporter(object):
@@ -30,7 +31,7 @@ def export(
         dev=False,
         extras=None,
         with_credentials=False,
-    ):  # type: (str, Path, Union[IO, str], bool, bool, bool) -> None
+    ):  # type: (str, Path, Union[IO, str], bool, bool, Optional[Union[bool, Sequence[str]]], bool) -> None
         if fmt not in self.ACCEPTED_FORMATS:
             raise ValueError("Invalid export format: {}".format(fmt))
 
@@ -51,38 +52,19 @@ def _export_requirements_txt(
         dev=False,
         extras=None,
         with_credentials=False,
-    ):  # type: (Path, Union[IO, str], bool, bool, bool) -> None
+    ):  # type: (Path, Union[IO, str], bool, bool, Optional[Union[bool, Sequence[str]]], bool) -> None
         indexes = set()
         content = ""
-        repository = self._poetry.locker.locked_repository(dev)
-
-        # Build a set of all packages required by our selected extras
-        extra_package_names = set(
-            get_extra_package_names(
-                repository.packages,
-                self._poetry.locker.lock_data.get("extras", {}),
-                extras or (),
-            )
-        )
-
         dependency_lines = set()
 
-        for dependency in self._poetry.locker.get_project_dependencies(
-            project_requires=self._poetry.package.all_requires,
-            with_nested=True,
-            with_dev=dev,
+        for dependency_package in self._poetry.locker.get_project_dependency_packages(
+            project_requires=self._poetry.package.all_requires, dev=dev, extras=extras
         ):
-            try:
-                package = repository.find_packages(dependency=dependency)[0]
-            except IndexError:
-                continue
-
-            # If a package is optional and we haven't opted in to it, continue
-            if package.optional and package.name not in extra_package_names:
-                continue
-
             line = ""
 
+            dependency = dependency_package.dependency
+            package = dependency_package.package
+
             if package.develop:
                 line += "-e "
 


Test patch:
diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py
--- a/tests/utils/test_exporter.py
+++ b/tests/utils/test_exporter.py
@@ -2,6 +2,7 @@
 
 import pytest
 
+from poetry.core.packages import dependency_from_pep_508
 from poetry.core.toml.file import TOMLFile
 from poetry.factory import Factory
 from poetry.packages import Locker as BaseLocker
@@ -59,13 +60,18 @@ def poetry(fixture_dir, locker):
     return p
 
 
-def set_package_requires(poetry):
+def set_package_requires(poetry, skip=None):
+    skip = skip or set()
     packages = poetry.locker.locked_repository(with_dev_reqs=True).packages
     poetry.package.requires = [
-        pkg.to_dependency() for pkg in packages if pkg.category == "main"
+        pkg.to_dependency()
+        for pkg in packages
+        if pkg.category == "main" and pkg.name not in skip
     ]
     poetry.package.dev_requires = [
-        pkg.to_dependency() for pkg in packages if pkg.category == "dev"
+        pkg.to_dependency()
+        for pkg in packages
+        if pkg.category == "dev" and pkg.name not in skip
     ]
 
 
@@ -170,6 +176,141 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers
     assert expected == content
 
 
+def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers(
+    tmp_dir, poetry
+):
+    poetry.locker.mock_lock_data(
+        {
+            "package": [
+                {
+                    "name": "a",
+                    "version": "1.2.3",
+                    "category": "main",
+                    "optional": False,
+                    "python-versions": "*",
+                    "marker": "python_version < '3.7'",
+                    "dependencies": {"b": ">=0.0.0", "c": ">=0.0.0"},
+                },
+                {
+                    "name": "b",
+                    "version": "4.5.6",
+                    "category": "main",
+                    "optional": False,
+                    "python-versions": "*",
+                    "marker": "platform_system == 'Windows'",
+                    "dependencies": {"d": ">=0.0.0"},
+                },
+                {
+                    "name": "c",
+                    "version": "7.8.9",
+                    "category": "main",
+                    "optional": False,
+                    "python-versions": "*",
+                    "marker": "sys_platform == 'win32'",
+                    "dependencies": {"d": ">=0.0.0"},
+                },
+                {
+                    "name": "d",
+                    "version": "0.0.1",
+                    "category": "main",
+                    "optional": False,
+                    "python-versions": "*",
+                },
+            ],
+            "metadata": {
+                "python-versions": "*",
+                "content-hash": "123456789",
+                "hashes": {"a": [], "b": [], "c": [], "d": []},
+            },
+        }
+    )
+    set_package_requires(poetry, skip={"b", "c", "d"})
+
+    exporter = Exporter(poetry)
+
+    exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt")
+
+    with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
+        content = f.read()
+
+    expected = {
+        "a": dependency_from_pep_508("a==1.2.3; python_version < '3.7'"),
+        "b": dependency_from_pep_508(
+            "b==4.5.6; platform_system == 'Windows' and python_version < '3.7'"
+        ),
+        "c": dependency_from_pep_508(
+            "c==7.8.9; sys_platform == 'win32' and python_version < '3.7'"
+        ),
+        "d": dependency_from_pep_508(
+            "d==0.0.1; python_version < '3.7' and platform_system == 'Windows' and sys_platform == 'win32'"
+        ),
+    }
+
+    for line in content.strip().split("\n"):
+        dependency = dependency_from_pep_508(line)
+        assert dependency.name in expected
+        expected_dependency = expected.pop(dependency.name)
+        assert dependency == expected_dependency
+        assert dependency.marker == expected_dependency.marker
+
+    assert expected == {}
+
+
+@pytest.mark.parametrize(
+    "dev,lines",
+    [(False, ['a==1.2.3; python_version < "3.8"']), (True, ["a==1.2.3", "b==4.5.6"])],
+)
+def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any(
+    tmp_dir, poetry, dev, lines
+):
+    poetry.locker.mock_lock_data(
+        {
+            "package": [
+                {
+                    "name": "a",
+                    "version": "1.2.3",
+                    "category": "main",
+                    "optional": False,
+                    "python-versions": "*",
+                },
+                {
+                    "name": "b",
+                    "version": "4.5.6",
+                    "category": "dev",
+                    "optional": False,
+                    "python-versions": "*",
+                    "dependencies": {"a": ">=1.2.3"},
+                },
+            ],
+            "metadata": {
+                "python-versions": "*",
+                "content-hash": "123456789",
+                "hashes": {"a": [], "b": []},
+            },
+        }
+    )
+
+    poetry.package.requires = [
+        Factory.create_dependency(
+            name="a", constraint=dict(version="^1.2.3", python="<3.8")
+        ),
+    ]
+    poetry.package.dev_requires = [
+        Factory.create_dependency(
+            name="b", constraint=dict(version="^4.5.6"), category="dev"
+        ),
+    ]
+
+    exporter = Exporter(poetry)
+
+    exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=dev)
+
+    with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
+        content = f.read()
+
+    assert content.strip() == "\n".join(lines)
+
+
 def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes(
     tmp_dir, poetry
 ):
@@ -399,8 +540,17 @@ def test_exporter_exports_requirements_txt_without_optional_packages(tmp_dir, po
     assert expected == content
 
 
-def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in(
-    tmp_dir, poetry
+@pytest.mark.parametrize(
+    "extras,lines",
+    [
+        (None, ["foo==1.2.3"]),
+        (False, ["foo==1.2.3"]),
+        (True, ["bar==4.5.6", "foo==1.2.3", "spam==0.1.0"]),
+        (["feature_bar"], ["bar==4.5.6", "foo==1.2.3", "spam==0.1.0"]),
+    ],
+)
+def test_exporter_exports_requirements_txt_with_optional_packages(
+    tmp_dir, poetry, extras, lines
 ):
     poetry.locker.mock_lock_data(
         {
@@ -445,22 +595,16 @@ def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in(
         Path(tmp_dir),
         "requirements.txt",
         dev=True,
-        extras=["feature_bar"],
+        with_hashes=False,
+        extras=extras,
     )
 
     with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
         content = f.read()
 
-    expected = """\
-bar==4.5.6 \\
-    --hash=sha256:67890
-foo==1.2.3 \\
-    --hash=sha256:12345
-spam==0.1.0 \\
-    --hash=sha256:abcde
-"""
+    expected = "\n".join(lines)
 
-    assert expected == content
+    assert content.strip() == expected
 
 
 def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry):
@@ -503,6 +647,55 @@ def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry)
     assert expected == content
 
 
+def test_exporter_can_export_requirements_txt_with_nested_packages(tmp_dir, poetry):
+    poetry.locker.mock_lock_data(
+        {
+            "package": [
+                {
+                    "name": "foo",
+                    "version": "1.2.3",
+                    "category": "main",
+                    "optional": False,
+                    "python-versions": "*",
+                    "source": {
+                        "type": "git",
+                        "url": "https://github.com/foo/foo.git",
+                        "reference": "123456",
+                    },
+                },
+                {
+                    "name": "bar",
+                    "version": "4.5.6",
+                    "category": "main",
+                    "optional": False,
+                    "python-versions": "*",
+                    "dependencies": {"foo": "rev 123456"},
+                },
+            ],
+            "metadata": {
+                "python-versions": "*",
+                "content-hash": "123456789",
+                "hashes": {"foo": [], "bar": []},
+            },
+        }
+    )
+    set_package_requires(poetry, skip={"foo"})
+
+    exporter = Exporter(poetry)
+
+    exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt")
+
+    with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
+        content = f.read()
+
+    expected = """\
+bar==4.5.6
+foo @ git+https://github.com/foo/foo.git@123456
+"""
+
+    assert expected == content
+
+
 def test_exporter_can_export_requirements_txt_with_git_packages_and_markers(
     tmp_dir, poetry
 ):