Repository: pypa/pip

Issue:
install --download doesn't download dependencies
Using the --download (-d) option only downloads the explicitly requested package, but does not download any of the usual dependencies. For example, running `$pip install fabric` will install pycrypto, paramiko, and fabric, but `$pip install -d . fabric` will only download fabric itself.

Pip should download the same packages when using the --download flag as would have been installed without the flag.

Note that there is also an explicit --no-deps option. The --download mode should respect this flag if it is provided.



Discussion:
Maybe you could give the changeset included in #416 a look

_I posted the following in #417, but this issue may be a better place to discuss.  So I repost it.  Sorry for the noise._

@carljm I am trying to change the behavior of `--download` to download dependencies.  However, I meet some test case design problem. To detect what the dependencies are, I have to unpacked code in the build dir and run_egg_info() on it.  Then, should I delete the unpacked source code in the clean stage?

According to [pip/commands/install.py](https://github.com/pypa/pip/blob/1da56757fa7ea2a8a4745563ce1edf7ff8573471/pip/commands/install.py#L249):

```
    if not options.no_install:
        requirement_set.cleanup_files(bundle=self.bundle)
```

`--download` option will forbid the cleanup stage.  So [this test case](https://github.com/pypa/pip/blob/1da56757fa7ea2a8a4745563ce1edf7ff8573471/tests/test_cleanup.py#L114) will fail because the build directory has source code unpacked there (a INITools directory was created there).

If I changed install.py to force cleanup the source code, [another test case](https://github.com/pypa/pip/blob/1da56757fa7ea2a8a4745563ce1edf7ff8573471/tests/test_basic.py#L148) would fail, because it thinks the source dir should exist.

What behavior do you prefer for `--download`?  Remove the unpacked source code anyway, or keep it?  Or depend on if `--editable` exists in options?


Resolving patch:
diff --git a/pip/commands/install.py b/pip/commands/install.py
--- a/pip/commands/install.py
+++ b/pip/commands/install.py
@@ -246,7 +246,7 @@ def run(self, options, args):
             requirement_set.create_bundle(self.bundle_filename)
             logger.notify('Created bundle in %s' % self.bundle_filename)
         # Clean up
-        if not options.no_install:
+        if not options.no_install or options.download_dir:
             requirement_set.cleanup_files(bundle=self.bundle)
         return requirement_set
 
diff --git a/pip/download.py b/pip/download.py
--- a/pip/download.py
+++ b/pip/download.py
@@ -407,7 +407,7 @@ def _copy_file(filename, location, content_type, link):
         logger.notify('Saved %s' % display_path(download_location))
 
 
-def unpack_http_url(link, location, download_cache, only_download):
+def unpack_http_url(link, location, download_cache, download_dir=None):
     temp_dir = tempfile.mkdtemp('-unpack', 'pip-')
     target_url = link.url.split('#', 1)[0]
     target_file = None
@@ -451,10 +451,9 @@ def unpack_http_url(link, location, download_cache, only_download):
         download_hash = _download_url(resp, link, temp_location)
     if link.md5_hash:
         _check_md5(download_hash, link)
-    if only_download:
-        _copy_file(temp_location, location, content_type, link)
-    else:
-        unpack_file(temp_location, location, content_type, link)
+    if download_dir:
+        _copy_file(temp_location, download_dir, content_type, link)
+    unpack_file(temp_location, location, content_type, link)
     if target_file and target_file != temp_location:
         cache_download(target_file, temp_location, content_type)
     if target_file is None:
diff --git a/pip/req.py b/pip/req.py
--- a/pip/req.py
+++ b/pip/req.py
@@ -971,6 +971,7 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
                     location = req_to_install.build_location(self.build_dir, not self.is_download)
                     ## FIXME: is the existance of the checkout good enough to use it?  I don't think so.
                     unpack = True
+                    url = None
                     if not os.path.exists(os.path.join(location, 'setup.py')):
                         ## FIXME: this won't upgrade when there's an existing package unpacked in `location`
                         if req_to_install.url is None:
@@ -993,7 +994,6 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
                             unpack = False
                     if unpack:
                         is_bundle = req_to_install.is_bundle
-                        url = None
                         if is_bundle:
                             req_to_install.move_bundle_files(self.build_dir, self.src_dir)
                             for subreq in req_to_install.bundle_requirements():
@@ -1001,8 +1001,8 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
                                 self.add_requirement(subreq)
                         elif self.is_download:
                             req_to_install.source_dir = location
+                            req_to_install.run_egg_info()
                             if url and url.scheme in vcs.all_schemes:
-                                req_to_install.run_egg_info()
                                 req_to_install.archive(self.download_dir)
                         else:
                             req_to_install.source_dir = location
@@ -1025,7 +1025,7 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
                                 req_to_install.satisfied_by = None
                             else:
                                 install = False
-                if not is_bundle and not self.is_download:
+                if not is_bundle:
                     ## FIXME: shouldn't be globally added:
                     finder.add_dependency_links(req_to_install.dependency_links)
                     if (req_to_install.extras):
@@ -1047,8 +1047,11 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
                             self.add_requirement(subreq)
                     if req_to_install.name not in self.requirements:
                         self.requirements[req_to_install.name] = req_to_install
+                    if self.is_download:
+                        self.reqs_to_cleanup.append(req_to_install)
                 else:
                     self.reqs_to_cleanup.append(req_to_install)
+
                 if install:
                     self.successfully_downloaded.append(req_to_install)
                     if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')):
@@ -1093,15 +1096,20 @@ def copy_to_build_dir(self, req_to_install):
 
     def unpack_url(self, link, location, only_download=False):
         if only_download:
-            location = self.download_dir
+            loc = self.download_dir
+        else:
+            loc = location
         if is_vcs_url(link):
-            return unpack_vcs_link(link, location, only_download)
+            return unpack_vcs_link(link, loc, only_download)
         elif is_file_url(link):
-            return unpack_file_url(link, location)
+            return unpack_file_url(link, loc)
         else:
             if self.download_cache:
                 self.download_cache = os.path.expanduser(self.download_cache)
-            return unpack_http_url(link, location, self.download_cache, only_download)
+            retval = unpack_http_url(link, location, self.download_cache, self.download_dir)
+            if only_download:
+                _write_delete_marker_message(os.path.join(location, PIP_DELETE_MARKER_FILENAME))
+            return retval
 
     def install(self, install_options, global_options=()):
         """Install everything in this set (after having downloaded and unpacked the packages)"""


Test patch:
diff --git a/tests/test_download.py b/tests/test_download.py
--- a/tests/test_download.py
+++ b/tests/test_download.py
@@ -26,3 +26,16 @@ def test_single_download_from_requirements_file():
     result = run_pip('install', '-r', env.scratch_path/ 'test-req.txt', '-d', '.', expect_error=True)
     assert Path('scratch')/ 'INITools-0.1.tar.gz' in result.files_created
     assert env.site_packages/ 'initools' not in result.files_created
+
+
+def test_download_should_download_dependencies():
+    """
+    It should download dependencies (in the scratch path)
+    """
+
+    env = reset_env()
+    result = run_pip('install', 'Paste[openid]==1.7.5.1', '-d', '.', expect_error=True)
+    assert Path('scratch')/ 'Paste-1.7.5.1.tar.gz' in result.files_created
+    openid_tarball_prefix = str(Path('scratch')/ 'python-openid-')
+    assert any(path.startswith(openid_tarball_prefix) for path in result.files_created)
+    assert env.site_packages/ 'openid' not in result.files_created