Upload 6 files

.gitignore

@@ -0,0 +1,4 @@
+.git/
+.venv/
+vent/
+_pycache_/

README.md

@@ -1,13 +1,13 @@
 ---
-title: HF TO WISEMODEL
-emoji: 🌖
-colorFrom: indigo
-colorTo: indigo
+title: Wisemodel To Hf
+emoji: 😻
+colorFrom: yellow
+colorTo: green
 sdk: gradio
-sdk_version: 4.27.0
+sdk_version: 4.22.0
 app_file: app.py
 pinned: false
-license: mit
+license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,232 @@
+import gradio as gr
+import os
+from git_commd import GitCommandWrapper
+from typing import List, Optional
+import logging
+
+HF_TOKEN = os.environ["HF_TOKEN"] if "HF_TOKEN" in os.environ else ""
+WiseModel_TOKEN = os.environ["WM_TOKEN"] if "WM_TOKEN" in os.environ else ""
+
+
+def get_cache_dir():
+    from random_word import RandomWords
+
+    r = RandomWords()
+    return r.get_random_word()
+
+
+def check_disk():
+    import os
+
+    return os.system("df -h /")
+
+
+def pull_from_wisemodel(
+    token: str, url: str, repo_name: str, cache_dir, branch: Optional[str] = None
+):
+
+    print("pull_from_wisemodel start")
+    print(cache_dir)
+    os.makedirs(cache_dir, exist_ok=True)
+    # os.system("cd "+cache_dir)
+    gitCmd = GitCommandWrapper()
+    isLfs = gitCmd.is_lfs_installed()
+    if not isLfs:
+        gitCmd.git_lfs_install()
+    gitCmd.clone(cache_dir, token, url, repo_name, branch)
+    print("pull_from_wisemodel end")
+    return f"Pulled {branch} to temp folder {cache_dir}: {url}"
+
+
+def push_to_wiseModel(
+    token: str,
+    url: str,
+    repo_name: str,
+    cache_dir: str,
+    hf_repo_id: str,
+    branch: Optional[str] = None,
+):
+
+    print("push_to_wiseModel start")
+
+    gitCmd = GitCommandWrapper()
+    gitCmd.clone(cache_dir, token, url, repo_name, branch)
+
+    repo_dir = f"./{cache_dir}/{repo_name}"
+    os.makedirs(repo_dir, exist_ok=True)
+    source_dir = f"./{cache_dir}/hf/{hf_repo_id}"
+    move_file(source_dir, repo_dir, excludes=[".git", ".gitattributes"])
+    gitCmd.add(repo_dir, all_files=True)
+    gitCmd.commit(repo_dir, message="commit from hf to wisemodel")
+    gitCmd.push(repo_dir, token, url, branch, branch)
+    print("push_to_wiseModel end")
+    return f"Pushed {branch} to {url}"
+
+
+def move_file(source: str, destination: str, excludes: list[str] = []):
+    import shutil
+
+    try:
+        # move all files in the source directory to the destination directory
+        # list all files in the source directory
+        files = os.listdir(source)
+        for file in files:
+            if file in excludes:
+                continue
+            logging.info(f"Moving {file} to {destination}")
+            # move each file to destination Directory
+            shutil.move(f"{source}/{file}", destination)
+    except Exception as e:
+        logging.exception(e)
+        return f"Error moving files from {source} to {destination}, {e.args[0]}"
+    return "file moved"
+
+
+def remove_file(cache_dir, repo_name):
+    import os
+
+    try:
+        os.remove(f"{cache_dir}/{repo_name}")
+    except:
+        return ""
+    return "README.md file removed"
+
+
+def push_to_hf(cache_dir, WiseModel_repo_name, hf_repo_id):
+    from huggingface_hub import HfApi
+
+    if not HF_TOKEN:
+        raise gr.Error("Please enter your HF_TOKEN")
+    print("push_to_hf start")
+    api = HfApi(token=HF_TOKEN)  # Token is not persisted on the machine.
+    output = api.upload_folder(
+        folder_path=f"{cache_dir}/{WiseModel_repo_name}",
+        repo_id=hf_repo_id,
+        repo_type="model",
+    )
+    print("push_to_hf end")
+    return f"Pushed to {hf_repo_id}"
+
+
+def pull_from_hf(cache_dir, hf_repo_id):
+    from huggingface_hub import HfApi
+
+    if not HF_TOKEN:
+        raise gr.Error("Please enter your HF_TOKEN")
+    print("pull_from_hf start")
+    api = HfApi(token=HF_TOKEN)  # Token is not persisted on the machine.
+    output = api.snapshot_download(
+        repo_id=hf_repo_id, repo_type="model", local_dir=cache_dir + "/hf/" + hf_repo_id
+    )
+    print(f"pull_from_hf end, output: {output}")
+    return f"Pulled from {hf_repo_id}"
+
+
+def handle(wisemodel_link, hf_repo_id):
+    cache_dir = get_cache_dir()
+    wiseModel_repo_url = (
+        wisemodel_link.replace(".git", "")
+        .replace("git", "")
+        .replace("clone", "")
+        .replace(" ", "")
+    )
+    wiseModel_repo_info = (
+        wisemodel_link.replace(".git", "")
+        .replace("git", "")
+        .replace("clone", "")
+        .replace(" ", "")
+        .split("/")
+    )
+    print(wiseModel_repo_info)
+    wisemodel_repo_name = wiseModel_repo_info[-1]
+    stages = [
+        (check_disk, (), {}),
+        # # Run all the sanity checks on README.md
+        # (pull_from_wisemodel, (WiseModel_TOKEN,wiseModel_repo_url,wisemodel_repo_name, cache_dir,"main"), {}),
+        # (remove_file, (wisemodel_repo_name, cache_dir), {}),
+        # (check_disk, (), {}),
+        # (push_to_hf, (cache_dir, wisemodel_repo_name, hf_repo_id), {}),
+        # (check_disk, (), {}),
+        (pull_from_hf, (cache_dir, hf_repo_id), {}),
+        (remove_file, (hf_repo_id, cache_dir), {}),
+        (check_disk, (), {}),
+        (
+            push_to_wiseModel,
+            (
+                WiseModel_TOKEN,
+                wiseModel_repo_url,
+                wisemodel_repo_name,
+                cache_dir,
+                hf_repo_id,
+                "main",
+            ),
+            {},
+        ),
+        (check_disk, (), {}),
+    ]
+
+    results = []
+    errors = []
+    for func, args, kwargs in stages:
+        try:
+            results.append(str(func(*args, **kwargs)))
+        except Exception as e:
+            logging.exception(e)
+            errors.append(str(e))
+        if errors:
+            break
+
+    return "\n\n".join(results), "\n\n".join(errors)
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(
+        """
+
+
+# HF-to-wisemodel/从Huggingface上拉取模型、数据集等到wisemodel
+
+# 这是一个示例Space,实际使用请参考下面说明，先Duplicate一个私有的space
+
+- 这个space可以把已经发布在Huggingface上的模型拉取到wisemodel上。
+- This space uploads model from Huggingface to wisemodel.
+
+- **请确认您是repo的拥有者或者有权限操作！**
+- **Please make sure that you're the owner of the repo or have permission from the owner to do so!**
+
+
+# 如何使用这个空间？
+# How to use this Space?
+
+- 点击右上角settings后面的“…”按钮，选择“Duplicate this Space”创建一个私有的space，同时输入wisemodel的token(必填)，以及Huggingface的token（必填），确保有相应repo写入的权限。
+- Duplicate this Space and providing WiseModel token (mandatory) and your read/write HF token (mandatory).
+
+- 在wiseModel上创建一个空的repo，这一步需要手动完成，Space不会为您创建一个空的repo。
+- Create your target model repo on WiseModel. This step is not automated.
+
+- 在刚刚自己创建的私有space里填写相应的信息，wisemodel的git clone链接，以及Huggingface的repo名称。
+- In your own private Space, fill in information below.
+
+- 点击submit按钮，然后可以通过logs按钮查看进度。
+- Click submit then watch for output in container log for progress.
+
+- Create README.md file (since the metadata is not compatible with HF)
+    """
+    )
+
+    wisemodel_link = gr.Textbox(
+        label="Copy the git download link from the model detail page of wisemodel（从wisemodel上获取该模型的完整git clone链接） "
+    )
+    hf_repo_id = gr.Textbox(
+        label="Source HF Model Repo ID (case sensitive). \nPlease make sure that this model has already been created"
+    )
+    with gr.Row():
+        button = gr.Button("Submit", variant="primary")
+        clear = gr.Button("Clear")
+    error = gr.Textbox(label="Error")
+    output = gr.Textbox(label="Output")
+
+    button.click(handle, [wisemodel_link, hf_repo_id], [output, error])
+
+if __name__ == "__main__":
+    demo.launch(debug=True)

git_commd.py

@@ -0,0 +1,238 @@
+import os
+import subprocess
+from typing import List, Optional
+
+from logger import get_logger
+logger = get_logger()
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton,
+                                        cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+class GitError(Exception):
+    pass
+class GitCommandWrapper(metaclass=Singleton):
+    """Some git operation wrapper
+    """
+    default_git_path = 'git'  # The default git command line
+    git_path='git'
+    MODEL_BRANCH="main"
+    def __init__(self, path: str = None):
+        self.git_path = path or self.default_git_path
+
+    def _run_git_command(self, *args) -> subprocess.CompletedProcess:
+        """Run git command, if command return 0, return subprocess.response
+             otherwise raise GitError, message is stdout and stderr.
+
+        Args:
+            args: List of command args.
+
+        Raises:
+            GitError: Exception with stdout and stderr.
+
+        Returns:
+            subprocess.CompletedProcess: the command response
+        """
+        logger.debug(' '.join(args))
+        git_env = os.environ.copy()
+        git_env['GIT_TERMINAL_PROMPT'] = '0'
+        response = subprocess.run(
+            [self.git_path, *args],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=git_env,
+        )  # compatible for python3.6
+        try:
+            response.check_returncode()
+            return response
+        except subprocess.CalledProcessError as error:
+            logger.error('There are error run git command.')
+            raise GitError(
+                'stdout: %s, stderr: %s' %
+                (response.stdout.decode('utf8'), error.stderr.decode('utf8')))
+
+    def config_auth_token(self, repo_dir, auth_token):
+        url = self.get_repo_remote_url(repo_dir)
+        if '//oauth2' not in url:
+            auth_url = self._add_token(auth_token, url)
+            cmd_args = '-C %s remote set-url origin %s' % (repo_dir, auth_url)
+            cmd_args = cmd_args.split(' ')
+            rsp = self._run_git_command(*cmd_args)
+            logger.debug(rsp.stdout.decode('utf8'))
+
+    def _add_token(self, token: str, url: str):
+        if token:
+            if '//oauth2' not in url:
+                url = url.replace('//', '//oauth2:%s@' % token)
+        return url
+
+    def remove_token_from_url(self, url: str):
+        if url and '//oauth2' in url:
+            start_index = url.find('oauth2')
+            end_index = url.find('@')
+            url = url[:start_index] + url[end_index + 1:]
+        return url
+
+    def is_lfs_installed(self):
+        cmd = ['lfs', 'env']
+        try:
+            self._run_git_command(*cmd)
+            return True
+        except GitError:
+            return False
+
+    def git_lfs_install(self, repo_dir):
+        cmd = ['-C', repo_dir, 'lfs', 'install']
+        try:
+            self._run_git_command(*cmd)
+            return True
+        except GitError:
+            return False
+
+    def clone(self,
+              repo_base_dir: str,
+              token: str,
+              url: str,
+              repo_name: str,
+              branch: Optional[str] = None):
+        """ git clone command wrapper.
+        For public project, token can None, private repo, there must token.
+
+        Args:
+            repo_base_dir (str): The local base dir, the repository will be clone to local_dir/repo_name
+            token (str): The git token, must be provided for private project.
+            url (str): The remote url
+            repo_name (str): The local repository path name.
+            branch (str, optional): _description_. Defaults to None.
+
+        Returns:
+            The popen response.
+        """
+        url = self._add_token(token, url)
+        if branch:
+            clone_args = '-C %s clone %s %s --branch %s' % (repo_base_dir, url,
+                                                            repo_name, branch)
+        else:
+            clone_args = '-C %s clone %s' % (repo_base_dir, url)
+        logger.debug(clone_args)
+        clone_args = clone_args.split(' ')
+        response = self._run_git_command(*clone_args)
+        logger.debug(response.stdout.decode('utf8'))
+        return response
+
+
+
+    def add(self,
+            repo_dir: str,
+            files: List[str] = list(),
+            all_files: bool = False):
+        if all_files:
+            add_args = '-C %s add -A' % repo_dir
+        elif len(files) > 0:
+            files_str = ' '.join(files)
+            add_args = '-C %s add %s' % (repo_dir, files_str)
+        add_args = add_args.split(' ')
+        rsp = self._run_git_command(*add_args)
+        logger.debug(rsp.stdout.decode('utf8'))
+        return rsp
+
+    def commit(self, repo_dir: str, message: str):
+        """Run git commit command
+
+        Args:
+            repo_dir (str): the repository directory.
+            message (str): commit message.
+
+        Returns:
+            The command popen response.
+        """
+        commit_args = ['-C', '%s' % repo_dir, 'commit', '-m', "'%s'" % message]
+        rsp = self._run_git_command(*commit_args)
+        logger.info(rsp.stdout.decode('utf8'))
+        return rsp
+
+    def checkout(self, repo_dir: str, revision: str):
+        cmds = ['-C', '%s' % repo_dir, 'checkout', '%s' % revision]
+        return self._run_git_command(*cmds)
+
+    def new_branch(self, repo_dir: str, revision: str):
+        cmds = ['-C', '%s' % repo_dir, 'checkout', '-b', revision]
+        return self._run_git_command(*cmds)
+
+    def get_remote_branches(self, repo_dir: str):
+        cmds = ['-C', '%s' % repo_dir, 'branch', '-r']
+        rsp = self._run_git_command(*cmds)
+        info = [
+            line.strip()
+            for line in rsp.stdout.decode('utf8').strip().split(os.linesep)
+        ]
+        if len(info) == 1:
+            return ['/'.join(info[0].split('/')[1:])]
+        else:
+            return ['/'.join(line.split('/')[1:]) for line in info[1:]]
+
+    def pull(self,
+             repo_dir: str,
+             remote: str = 'origin',
+             branch: str = 'main'):
+        cmds = ['-C', repo_dir, 'pull', remote, branch]
+        return self._run_git_command(*cmds)
+
+    def push(self,
+             repo_dir: str,
+             token: str,
+             url: str,
+             local_branch: str,
+             remote_branch: str,
+             force: bool = False):
+        url = self._add_token(token, url)
+
+        push_args = '-C %s push %s %s:%s' % (repo_dir, url, local_branch,
+                                             remote_branch)
+        if force:
+            push_args += ' -f'
+        push_args = push_args.split(' ')
+        rsp = self._run_git_command(*push_args)
+        logger.debug(rsp.stdout.decode('utf8'))
+        return rsp
+
+    def get_repo_remote_url(self, repo_dir: str):
+        cmd_args = '-C %s config --get remote.origin.url' % repo_dir
+        cmd_args = cmd_args.split(' ')
+        rsp = self._run_git_command(*cmd_args)
+        url = rsp.stdout.decode('utf8')
+        return url.strip()
+
+    def list_lfs_files(self, repo_dir: str):
+        cmd_args = '-C %s lfs ls-files' % repo_dir
+        cmd_args = cmd_args.split(' ')
+        rsp = self._run_git_command(*cmd_args)
+        out = rsp.stdout.decode('utf8').strip()
+        files = []
+        for line in out.split(os.linesep):
+            files.append(line.split(' ')[-1])
+
+        return files
+
+    def tag(self,
+            repo_dir: str,
+            tag_name: str,
+            message: str,
+            ref: str = MODEL_BRANCH):
+        cmd_args = [
+            '-C', repo_dir, 'tag', tag_name, '-m',
+            '"%s"' % message, ref
+        ]
+        rsp = self._run_git_command(*cmd_args)
+        logger.debug(rsp.stdout.decode('utf8'))
+        return rsp
+
+    def push_tag(self, repo_dir: str, tag_name):
+        cmd_args = ['-C', repo_dir, 'push', 'origin', tag_name]
+        rsp = self._run_git_command(*cmd_args)
+        logger.debug(rsp.stdout.decode('utf8'))
+        return rsp

logger.py

@@ -0,0 +1,89 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import importlib
+import logging
+from typing import Optional
+
+init_loggers = {}
+
+formatter = logging.Formatter(
+    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+
+def get_logger(log_file: Optional[str] = None,
+               log_level: int = logging.INFO,
+               file_mode: str = 'w'):
+    """ Get logging logger
+
+    Args:
+        log_file: Log filename, if specified, file handler will be added to
+            logger
+        log_level: Logging level.
+        file_mode: Specifies the mode to open the file, if filename is
+            specified (if filemode is unspecified, it defaults to 'w').
+    """
+
+    logger_name = __name__.split('.')[0]
+    logger = logging.getLogger(logger_name)
+    logger.propagate = False
+    if logger_name in init_loggers:
+        add_file_handler_if_needed(logger, log_file, file_mode, log_level)
+        return logger
+
+    # handle duplicate logs to the console
+    # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler <stderr> (NOTSET)
+    # to the root logger. As logger.propagate is True by default, this root
+    # level handler causes logging messages from rank>0 processes to
+    # unexpectedly show up on the console, creating much unwanted clutter.
+    # To fix this issue, we set the root logger's StreamHandler, if any, to log
+    # at the ERROR level.
+    torch_dist = False
+    is_worker0 = True
+    if importlib.util.find_spec('torch') is not None:
+        from modelscope.utils.torch_utils import is_dist, is_master
+        torch_dist = is_dist()
+        is_worker0 = is_master()
+
+    if torch_dist:
+        for handler in logger.root.handlers:
+            if type(handler) is logging.StreamHandler:
+                handler.setLevel(logging.ERROR)
+
+    stream_handler = logging.StreamHandler()
+    handlers = [stream_handler]
+
+    if is_worker0 and log_file is not None:
+        file_handler = logging.FileHandler(log_file, file_mode)
+        handlers.append(file_handler)
+
+    for handler in handlers:
+        handler.setFormatter(formatter)
+        handler.setLevel(log_level)
+        logger.addHandler(handler)
+
+    if is_worker0:
+        logger.setLevel(log_level)
+    else:
+        logger.setLevel(logging.ERROR)
+
+    init_loggers[logger_name] = True
+
+    return logger
+
+
+def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
+    for handler in logger.handlers:
+        if isinstance(handler, logging.FileHandler):
+            return
+
+    if importlib.util.find_spec('torch') is not None:
+        from modelscope.utils.torch_utils import is_master
+        is_worker0 = is_master()
+    else:
+        is_worker0 = True
+
+    if is_worker0 and log_file is not None:
+        file_handler = logging.FileHandler(log_file, file_mode)
+        file_handler.setFormatter(formatter)
+        file_handler.setLevel(log_level)
+        logger.addHandler(file_handler)

requirements.txt

@@ -0,0 +1,3 @@
+gradio[auth]
+huggingface_hub
+random-word