Repository: huggingface/huggingface_hub
Pull request #17 (2021-02-20 19:03:45)

Repository helper class: continuation of #7
continuation of #7 


Patch:
diff --git a/src/huggingface_hub/commands/lfs.py b/src/huggingface_hub/commands/lfs.py
index 866c787b..07ad27a7 100644
--- a/src/huggingface_hub/commands/lfs.py
+++ b/src/huggingface_hub/commands/lfs.py
@@ -27,14 +27,12 @@ from typing import Dict, List, Optional
 
 import requests
 from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
 
 
 logger = logging.getLogger(__name__)
 
 
-LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload"
-
-
 class LfsCommands(BaseHuggingfaceCLICommand):
     """
     Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload
diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py
index e35ce2ca..bf24a924 100644
--- a/src/huggingface_hub/hf_api.py
+++ b/src/huggingface_hub/hf_api.py
@@ -158,6 +158,9 @@ class HfApi:
             exist_ok: Do not raise an error if repo already exists
 
             lfsmultipartthresh: Optional: internal param for testing purposes.
+
+        Returns:
+            URL to the newly created repo.
         """
         path = "{}/api/repos/create".format(self.endpoint)
 
diff --git a/src/huggingface_hub/lfs.py b/src/huggingface_hub/lfs.py
new file mode 100644
index 00000000..6a615158
--- /dev/null
+++ b/src/huggingface_hub/lfs.py
@@ -0,0 +1,17 @@
+# coding=utf-8
+# Copyright 2019-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload"
diff --git a/src/huggingface_hub/repository.py b/src/huggingface_hub/repository.py
new file mode 100644
index 00000000..6ba94204
--- /dev/null
+++ b/src/huggingface_hub/repository.py
@@ -0,0 +1,236 @@
+import logging
+import os
+import subprocess
+from typing import List, Optional, Union
+
+from .hf_api import HfFolder
+from .lfs import LFS_MULTIPART_UPLOAD_COMMAND
+
+
+logger = logging.getLogger(__name__)
+
+
+class Repository:
+    """
+    Helper class to wrap the git and git-lfs commands.
+
+    The aim is to facilitate interacting with huggingface.co hosted model or dataset repos,
+    though not a lot here (if any) is actually specific to huggingface.co.
+    """
+
+    def __init__(
+        self,
+        local_dir: str,
+        clone_from: Optional[str] = None,
+        use_auth_token: Union[bool, str, None] = None,
+        git_user: Optional[str] = None,
+        git_email: Optional[str] = None,
+    ):
+        """
+        Instantiate a local clone of a git repo.
+
+        If specifying a `clone_from`:
+        will clone an existing remote repository
+        that was previously created using ``HfApi().create_repo(token=huggingface_token, name=repo_name)``.
+        ``Repository`` uses the local git credentials by default, but if required, the ``huggingface_token``
+        as well as the git ``user`` and the ``email`` can be specified.
+        ``Repository`` will then override them.
+        If `clone_from` is used, and the repository is being instantiated into a non-empty directory,
+        e.g. a directory with your trained model files, it will automatically merge them.
+
+        Args:
+            local_dir (``str``):
+                path (e.g. ``'my_trained_model/'``) to the local directory, where the ``Repository`` will be either initalized.
+            clone_from (``str``, optional):
+                repository url (e.g. ``'https://huggingface.co/philschmid/playground-tests'``).
+            use_auth_token (``str`` or ``bool``, `optional`, defaults ``None``):
+                huggingface_token can be extract from ``HfApi().login(username, password)`` and is used to authenticate against the hub.
+            git_user (``str``, `optional`, defaults ``None``):
+                will override the ``git config user.name`` for committing and pushing files to the hub.
+            git_email (``str``, `optional`, defaults ``None``):
+                will override the ``git config user.email`` for committing and pushing files to the hub.
+        """
+
+        os.makedirs(local_dir, exist_ok=True)
+        self.local_dir = local_dir
+
+        self.check_git_versions()
+
+        if clone_from is not None:
+            self.clone_from(repo_url=clone_from, use_auth_token=use_auth_token)
+        else:
+            try:
+                remotes = subprocess.check_output(
+                    ["git", "remote", "-v"],
+                    encoding="utf-8",
+                    cwd=self.local_dir,
+                )
+                logger.debug("[Repository] has remotes")
+                logger.debug(remotes)
+            except subprocess.CalledProcessError:
+                logger.error(
+                    "If not specifying `clone_from`, you need to pass Repository a valid git clone."
+                )
+                raise ValueError(
+                    "If not specifying `clone_from`, you need to pass Repository a valid git clone."
+                )
+
+        # overrides .git config if user and email is provided.
+        if git_user is not None or git_email is not None:
+            self.git_config_username_and_email(git_user, git_email)
+
+    def check_git_versions(self):
+        """
+        print git and git-lfs versions, raises if they aren't installed.
+        """
+        try:
+            git_version = subprocess.check_output(
+                ["git", "--version"], encoding="utf-8"
+            ).strip()
+        except FileNotFoundError:
+            raise EnvironmentError(
+                "Looks like you do not have git installed, please install."
+            )
+
+        try:
+            lfs_version = subprocess.check_output(
+                ["git-lfs", "--version"],
+                encoding="utf-8",
+            ).strip()
+        except FileNotFoundError:
+            raise EnvironmentError(
+                "Looks like you do not have git-lfs installed, please install."
+                " You can install from https://git-lfs.github.com/."
+                " Then run `git lfs install` (you only have to do this once)."
+            )
+        logger.info(git_version + "\n" + lfs_version)
+
+    def clone_from(self, repo_url: str, use_auth_token: Union[bool, str, None] = None):
+        """
+        Clone from a remote.
+        """
+        if isinstance(use_auth_token, str):
+            huggingface_token = use_auth_token
+        elif use_auth_token:
+            huggingface_token = HfFolder.get_token()
+        else:
+            huggingface_token = None
+
+        if (
+            huggingface_token is not None
+            and "huggingface.co" in repo_url
+            and "@" not in repo_url
+        ):
+            # adds huggingface_token to repo url if it is provided.
+            # do not leak user token if it's not a repo on hf.co
+            repo_url = repo_url.replace(
+                "https://", f"https://user:{huggingface_token}@"
+            )
+
+        subprocess.run("git lfs install".split(), check=True)
+
+        # checks if repository is initialized in a empty repository or in one with files
+        if len(os.listdir(self.local_dir)) == 0:
+            subprocess.run(
+                ["git", "clone", repo_url, "."], check=True, cwd=self.local_dir
+            )
+        else:
+            logger.warning(
+                "[Repository] local_dir is not empty, so let's try to pull the remote over a non-empty folder."
+            )
+            subprocess.run("git init".split(), check=True, cwd=self.local_dir)
+            subprocess.run(
+                ["git", "remote", "add", "origin", repo_url],
+                check=True,
+                cwd=self.local_dir,
+            )
+            subprocess.run("git fetch".split(), check=True, cwd=self.local_dir)
+            subprocess.run(
+                "git reset origin/main".split(), check=True, cwd=self.local_dir
+            )
+            # TODO(check if we really want the --force flag)
+            subprocess.run(
+                "git checkout origin/main -ft".split(), check=True, cwd=self.local_dir
+            )
+
+    def git_config_username_and_email(
+        self, git_user: Optional[str] = None, git_email: Optional[str] = None
+    ):
+        """
+        sets git user name and email (only in the current repo)
+        """
+        if git_user is not None:
+            subprocess.run(
+                f"git config user.name {git_user}".split(),
+                check=True,
+                cwd=self.local_dir,
+            )
+        if git_email is not None:
+            subprocess.run(
+                f"git config user.email {git_email}".split(),
+                check=True,
+                cwd=self.local_dir,
+            )
+
+    def lfs_track(self, patterns: List[str]):
+        """
+        Tell git-lfs to track those files.
+        """
+        for pattern in patterns:
+            subprocess.run(
+                ["git", "lfs", "track", pattern], check=True, cwd=self.local_dir
+            )
+
+    def lfs_enable_largefiles(self):
+        """
+        HF-specific. This enables upload support of files >5GB.
+        """
+        subprocess.run(
+            "git config lfs.customtransfer.multipart.path huggingface-cli".split(),
+            check=True,
+            cwd=self.local_dir,
+        )
+        subprocess.run(
+            f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
+            check=True,
+            cwd=self.local_dir,
+        )
+
+    def git_pull(self, rebase: Optional[bool] = False):
+        """
+        git pull
+        """
+        args = "git pull".split()
+        if rebase:
+            args.append("--rebase")
+        subprocess.run(args, check=True, cwd=self.local_dir)
+
+    def git_add(self, pattern="."):
+        """
+        git add
+        """
+        subprocess.run("git add .".split(), check=True, cwd=self.local_dir)
+
+    def git_commit(self, commit_message="commit files to HF hub"):
+        """
+        git commit
+        """
+        subprocess.run(
+            ["git", "commit", "-m", commit_message], check=True, cwd=self.local_dir
+        )
+
+    def git_push(self):
+        """
+        git push
+        """
+        subprocess.run("git push".split(), check=True, cwd=self.local_dir)
+
+    def push_to_hub(self, commit_message="commit files to HF hub"):
+        """
+        Helper to add, commit, and pushe file to remote repository on the HuggingFace Hub.
+        Args:
+            commit_message: commit message.
+        """
+        self.git_add()
+        self.git_commit(commit_message)
+        self.git_push()


Test patch:
diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py
index 58dce425..3c4b4004 100644
--- a/tests/test_hf_api.py
+++ b/tests/test_hf_api.py
@@ -23,15 +23,10 @@ from huggingface_hub.constants import REPO_TYPE_DATASET
 from huggingface_hub.hf_api import HfApi, HfFolder, ModelInfo, RepoObj
 from requests.exceptions import HTTPError
 
+from .testing_constants import ENDPOINT_STAGING, ENDPOINT_STAGING_BASIC_AUTH, PASS, USER
 from .testing_utils import require_git_lfs
 
 
-USER = "__DUMMY_TRANSFORMERS_USER__"
-PASS = "__DUMMY_TRANSFORMERS_PASS__"
-
-ENDPOINT_STAGING = "https://moon-staging.huggingface.co"
-ENDPOINT_STAGING_BASIC_AUTH = f"https://{USER}:{PASS}@moon-staging.huggingface.co"
-
 REPO_NAME = "my-model-{}".format(int(time.time() * 10e3))
 REPO_NAME_LARGE_FILE = "my-model-largefiles-{}".format(int(time.time() * 10e3))
 DATASET_REPO_NAME = "my-dataset-{}".format(int(time.time() * 10e3))
diff --git a/tests/test_repository.py b/tests/test_repository.py
new file mode 100644
index 00000000..37e70f24
--- /dev/null
+++ b/tests/test_repository.py
@@ -0,0 +1,116 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+import unittest
+
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.repository import Repository
+
+from .testing_constants import ENDPOINT_STAGING, PASS, USER
+
+
+REPO_NAME = "repo-{}".format(int(time.time() * 10e3))
+
+
+WORKING_REPO_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "fixtures/working_repo_2"
+)
+
+
+class RepositoryCommonTest(unittest.TestCase):
+    _api = HfApi(endpoint=ENDPOINT_STAGING)
+
+
+class RepositoryTest(RepositoryCommonTest):
+    @classmethod
+    def setUpClass(cls):
+        """
+        Share this valid token in all tests below.
+        """
+        cls._token = cls._api.login(username=USER, password=PASS)
+
+    def setUp(self):
+        try:
+            shutil.rmtree(WORKING_REPO_DIR)
+        except FileNotFoundError:
+            pass
+
+        self._repo_url = self._api.create_repo(token=self._token, name=REPO_NAME)
+
+    def tearDown(self):
+        self._api.delete_repo(token=self._token, name=REPO_NAME)
+
+    def test_init_from_existing_local_clone(self):
+        subprocess.run(
+            ["git", "clone", self._repo_url, WORKING_REPO_DIR],
+            check=True,
+        )
+
+        repo = Repository(WORKING_REPO_DIR)
+        repo.lfs_track(["*.pdf"])
+        repo.lfs_enable_largefiles()
+        repo.git_pull()
+
+    def test_init_failure(self):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with self.assertRaises(ValueError):
+                _ = Repository(tmpdirname)
+
+    def test_init_clone_in_empty_folder(self):
+        repo = Repository(WORKING_REPO_DIR, clone_from=self._repo_url)
+        repo.lfs_track(["*.pdf"])
+        repo.lfs_enable_largefiles()
+        repo.git_pull()
+
+    def test_init_clone_in_nonempty_folder(self):
+        # Create dummy files
+        # one is lfs-tracked, the other is not.
+        os.makedirs(WORKING_REPO_DIR, exist_ok=True)
+        with open(os.path.join(WORKING_REPO_DIR, "dummy.txt"), "w") as f:
+            f.write("hello")
+        with open(os.path.join(WORKING_REPO_DIR, "model.bin"), "w") as f:
+            f.write("hello")
+        repo = Repository(WORKING_REPO_DIR, clone_from=self._repo_url)
+        repo.lfs_track(["*.pdf"])
+        repo.lfs_enable_largefiles()
+        repo.git_pull()
+
+    def test_add_commit_push(self):
+        repo = Repository(
+            WORKING_REPO_DIR,
+            clone_from=self._repo_url,
+            use_auth_token=self._token,
+            git_user="ci",
+            git_email="ci@dummy.com",
+        )
+
+        # Create dummy files
+        # one is lfs-tracked, the other is not.
+        with open(os.path.join(WORKING_REPO_DIR, "dummy.txt"), "w") as f:
+            f.write("hello")
+        with open(os.path.join(WORKING_REPO_DIR, "model.bin"), "w") as f:
+            f.write("hello")
+
+        repo.git_add()
+        repo.git_commit()
+        try:
+            repo.git_push()
+        except subprocess.CalledProcessError as exc:
+            print(exc.stderr)
+            raise exc
diff --git a/tests/testing_constants.py b/tests/testing_constants.py
new file mode 100644
index 00000000..1410997f
--- /dev/null
+++ b/tests/testing_constants.py
@@ -0,0 +1,5 @@
+USER = "__DUMMY_TRANSFORMERS_USER__"
+PASS = "__DUMMY_TRANSFORMERS_PASS__"
+
+ENDPOINT_STAGING = "https://moon-staging.huggingface.co"
+ENDPOINT_STAGING_BASIC_AUTH = f"https://{USER}:{PASS}@moon-staging.huggingface.co"