Repository: pre-commit/pre-commit

Issue:
Investigate / implement shallow cloning
Now that github supports it, might be worth the speed improvements

It would have to be smart enough to fall back to slow clone when the option isn't available (either the client is too old or the server is too old)


Discussion:
Here's where we currently clone code:

https://github.com/pre-commit/pre-commit/blob/aa4bc9d241d805d67efa29f040b29fe3baba5523/pre_commit/store.py#L142-L149

Here's the rough sketch of the commands to execute a shallow clone:

```bash
git init directory
git -C directory remote add origin $remote
git -C directory fetch origin $revision --depth=1
git -C directory checkout $revision
git -C directory submodule update --init --recursive --depth=1
```
Here's the stackoverflow post describing this: https://stackoverflow.com/a/30701724/812183

Resolving patch:
diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py
--- a/pre_commit/languages/node.py
+++ b/pre_commit/languages/node.py
@@ -62,11 +62,13 @@ def install_environment(prefix, version, additional_dependencies):
             cmd.extend(['-n', version])
         cmd_output(*cmd)
 
-        dep = 'git+file:///{}'.format(prefix.prefix_dir)
         with in_env(prefix, version):
+            # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449
+            # install as if we installed from git
+            helpers.run_setup_cmd(prefix, ('npm', 'install'))
             helpers.run_setup_cmd(
                 prefix,
-                ('npm', 'install', '-g', dep) + additional_dependencies,
+                ('npm', 'install', '-g', '.') + additional_dependencies,
             )
 
 
diff --git a/pre_commit/store.py b/pre_commit/store.py
--- a/pre_commit/store.py
+++ b/pre_commit/store.py
@@ -10,6 +10,7 @@
 import pre_commit.constants as C
 from pre_commit import file_lock
 from pre_commit import git
+from pre_commit.util import CalledProcessError
 from pre_commit.util import clean_path_on_failure
 from pre_commit.util import cmd_output
 from pre_commit.util import resource_text
@@ -134,19 +135,43 @@ def _get_result():
                 )
         return directory
 
+    def _complete_clone(self, ref, git_cmd):
+        """Perform a complete clone of a repository and its submodules """
+
+        git_cmd('fetch', 'origin')
+        git_cmd('checkout', ref)
+        git_cmd('submodule', 'update', '--init', '--recursive')
+
+    def _shallow_clone(self, ref, git_cmd):  # pragma: windows no cover
+        """Perform a shallow clone of a repository and its submodules """
+
+        git_config = 'protocol.version=2'
+        git_cmd('-c', git_config, 'fetch', 'origin', ref, '--depth=1')
+        git_cmd('checkout', ref)
+        git_cmd(
+            '-c', git_config, 'submodule', 'update', '--init',
+            '--recursive', '--depth=1',
+        )
+
     def clone(self, repo, ref, deps=()):
         """Clone the given url and checkout the specific ref."""
+
+        if os.path.isdir(repo):
+            repo = os.path.abspath(repo)
+
         def clone_strategy(directory):
             env = git.no_git_env()
 
-            cmd = ('git', 'clone', '--no-checkout', repo, directory)
-            cmd_output(*cmd, env=env)
-
             def _git_cmd(*args):
-                return cmd_output('git', *args, cwd=directory, env=env)
+                cmd_output('git', *args, cwd=directory, env=env)
+
+            _git_cmd('init', '.')
+            _git_cmd('remote', 'add', 'origin', repo)
 
-            _git_cmd('reset', ref, '--hard')
-            _git_cmd('submodule', 'update', '--init', '--recursive')
+            try:
+                self._shallow_clone(ref, _git_cmd)
+            except CalledProcessError:
+                self._complete_clone(ref, _git_cmd)
 
         return self._new_repo(repo, ref, deps, clone_strategy)
 


Test patch:
diff --git a/tests/store_test.py b/tests/store_test.py
--- a/tests/store_test.py
+++ b/tests/store_test.py
@@ -12,6 +12,7 @@
 from pre_commit import git
 from pre_commit.store import _get_default_directory
 from pre_commit.store import Store
+from pre_commit.util import CalledProcessError
 from testing.fixtures import git_dir
 from testing.util import cwd
 from testing.util import git_commit
@@ -111,6 +112,41 @@ def test_clone_when_repo_already_exists(store):
     assert store.clone('fake_repo', 'fake_ref') == 'fake_path'
 
 
+def test_clone_shallow_failure_fallback_to_complete(
+    store, tempdir_factory,
+    log_info_mock,
+):
+    path = git_dir(tempdir_factory)
+    with cwd(path):
+        git_commit()
+        rev = git.head_rev(path)
+        git_commit()
+
+    # Force shallow clone failure
+    def fake_shallow_clone(self, *args, **kwargs):
+        raise CalledProcessError(None, None, None)
+    store._shallow_clone = fake_shallow_clone
+
+    ret = store.clone(path, rev)
+
+    # Should have printed some stuff
+    assert log_info_mock.call_args_list[0][0][0].startswith(
+        'Initializing environment for ',
+    )
+
+    # Should return a directory inside of the store
+    assert os.path.exists(ret)
+    assert ret.startswith(store.directory)
+    # Directory should start with `repo`
+    _, dirname = os.path.split(ret)
+    assert dirname.startswith('repo')
+    # Should be checked out to the rev we specified
+    assert git.head_rev(ret) == rev
+
+    # Assert there's an entry in the sqlite db for this
+    assert store.select_all_repos() == [(path, rev, ret)]
+
+
 def test_create_when_directory_exists_but_not_db(store):
     # In versions <= 0.3.5, there was no sqlite db causing a need for
     # backward compatibility