Spaces:
Runtime error
Runtime error
| import csv | |
| import os | |
| import shutil | |
| from datetime import datetime | |
| from pathlib import Path | |
| from tempfile import TemporaryDirectory | |
| from typing import Optional | |
| import gradio as gr | |
| from huggingface_hub import HfApi, ModelCard, Repository, scan_cache_dir | |
| from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError | |
| from convert import convert | |
| # Repo with files totalling more than 24GB are not converted. Avoid to have a memory issue. | |
| try: | |
| MAX_REPO_SIZE = int(os.environ.get("MAX_REPO_SIZE")) | |
| except: | |
| MAX_REPO_SIZE = 24 * 1000 * 1000 * 1000 | |
| # Used to log Space usage | |
| # Taken from https://huggingface.co/spaces/onnx/export | |
| DATASET_REPO_ID = "Wauplin/bloom.cpp-converters" | |
| DATASET_LOCAL_DIR = "usage_data" | |
| DATASET_LOCAL_FILE = Path(DATASET_LOCAL_DIR) / "data.csv" | |
| HF_TOKEN = os.environ.get("HF_TOKEN") | |
| repo: Optional[Repository] = None | |
| if HF_TOKEN: | |
| repo = Repository( | |
| local_dir=DATASET_LOCAL_DIR, | |
| clone_from=DATASET_REPO_ID, | |
| repo_type="dataset", | |
| token=HF_TOKEN, | |
| ) | |
| class Generator: | |
| # Taken from https://stackoverflow.com/a/34073559 | |
| # Allows to log process in Gradio | |
| def __init__(self, gen): | |
| self.gen = gen | |
| def __iter__(self): | |
| self.value = yield from self.gen | |
| def run( | |
| token: str, model_id: str, precision: str, quantization: bool, destination: str | |
| ): | |
| _log_usage( | |
| status="start", | |
| model_id=model_id, | |
| precision=precision, | |
| quantization=quantization, | |
| destination=destination, | |
| pr_url=None, | |
| ) | |
| _all_logs = [] | |
| def _log(msg: str): | |
| print(msg) # for container logs | |
| _all_logs.append(msg) | |
| return "\n\n".join(_all_logs) # for Gradio output | |
| if token == "" or model_id == "": | |
| yield _log("### Invalid input 🐞\n\nPlease fill a token and model_id.") | |
| _log_usage( | |
| status="invalid input", | |
| model_id=model_id, | |
| precision=precision, | |
| quantization=quantization, | |
| destination=destination, | |
| pr_url=None, | |
| ) | |
| return | |
| if destination == "": | |
| _log("Destination not provided. Will default to the initial repo.") | |
| destination = model_id | |
| api = HfApi(token=token) | |
| try: | |
| # TODO: make a PR to bloomz.cpp to be able to pass a token | |
| model_info = api.model_info(repo_id=model_id, files_metadata=True, token=False) | |
| _log(f"Model {model_id} exists.") | |
| except RepositoryNotFoundError: | |
| yield _log( | |
| f"\n### Error 😢😢😢\n\nRepository {model_id} not found. Only public models are convertible at the moment." | |
| ) | |
| _log_usage( | |
| status="model not found", | |
| model_id=model_id, | |
| precision=precision, | |
| quantization=quantization, | |
| destination=destination, | |
| pr_url=None, | |
| ) | |
| return | |
| try: | |
| total_size = sum( | |
| file.size | |
| for file in model_info.siblings | |
| if file.rfilename.endswith(".pt") or file.rfilename.endswith(".bin") | |
| ) | |
| if total_size > MAX_REPO_SIZE: | |
| yield _log( | |
| f"### Unprocessable 😢😢😢\n\nModel {model_id} is too big and cannot be processed in this Space. This Space needs to be able to load the model in memory before converting it. To avoid a memory issue, we do not process models bigger than {MAX_REPO_SIZE}b.\n\nYou have 2 options:\n- [Duplicate this Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter?duplicate=true) and assign a bigger machine. You will need to set 'MAX_REPO_SIZE' as a secret to overwrite the default value. Once you are done, remove the upgraded hardware and/or delete the Space.\n- Manually convert the weights by following [this guide](https://github.com/NouamaneTazi/bloomz.cpp#usage)." | |
| ) | |
| _log_usage( | |
| status="unprocessable", | |
| model_id=model_id, | |
| precision=precision, | |
| quantization=quantization, | |
| destination=destination, | |
| pr_url=None, | |
| ) | |
| return | |
| with TemporaryDirectory() as cache_folder: | |
| convert_progress = Generator( | |
| convert( | |
| cache_folder=Path(cache_folder), | |
| model_id=model_id, | |
| precision=precision, | |
| quantization=quantization, | |
| ) | |
| ) | |
| for msg in convert_progress: | |
| yield _log(msg) | |
| model_path = convert_progress.value | |
| yield _log(f"Model converted: {model_path}") | |
| destination_url = api.create_repo(repo_id=destination, exist_ok=True) | |
| destination = destination_url.repo_id | |
| yield _log(f"Destination model: {destination_url}") | |
| pr = api.create_pull_request( | |
| repo_id=destination_url.repo_id, | |
| title=f"Add {model_path.name} from bloomz.cpp converter.", | |
| description="This PR has been created using the [bloomz.cpp converter Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter). It adds weights compatible with the [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp#usage) project.", | |
| ) | |
| pr_url = f"https://huggingface.co/{destination}/discussions/{pr.num}" | |
| yield _log(f"Created PR: {pr_url} (empty)") | |
| yield _log(f"Uploading model to PR") | |
| api.upload_file( | |
| repo_id=destination, | |
| path_or_fileobj=model_path, | |
| path_in_repo=model_path.name, | |
| revision=pr.git_reference, | |
| ) | |
| yield _log(f"Model uploaded to PR") | |
| yield _log(f"Modifying model card in PR (add `bloom` and `ggml` tags)") | |
| try: | |
| card = ModelCard.load(repo_id_or_path=destination) | |
| except EntryNotFoundError: # new repo => no model card yet | |
| card = ModelCard( | |
| "This model contains a model based on the Bloom architecture with weights compatible with [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp). This model card has been automatically generated [by the bloomz.cpp converter Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter) and must be completed." | |
| ) | |
| if card.data.tags is None: | |
| card.data.tags = [] | |
| tags = card.data.tags | |
| if "ggml" not in tags: | |
| tags.append("ggml") | |
| if "bloom" not in tags: | |
| tags.append("bloom") | |
| card.push_to_hub( | |
| repo_id=destination, token=token, revision=pr.git_reference | |
| ) | |
| yield _log(f"Model card modified in PR.") | |
| api.change_discussion_status( | |
| repo_id=destination, | |
| discussion_num=pr.num, | |
| new_status="open", | |
| comment="PR is now complete and ready to be reviewed.", | |
| ) | |
| yield _log(f"[PR]({pr_url}) is complete and ready to be reviewed.") | |
| yield _log( | |
| f"### Success 🔥\n\nYay! This model was successfully converted! Make sure to let the repo owner know about it and review your PR. You might need to complete the PR manually, especially to add information in the model card." | |
| ) | |
| _log_usage( | |
| status="success", | |
| model_id=model_id, | |
| precision=precision, | |
| quantization=quantization, | |
| destination=destination, | |
| pr_url=pr_url, | |
| ) | |
| shutil.rmtree(model_path.parent) | |
| _delete_cache() | |
| return | |
| except Exception as e: | |
| _log_usage( | |
| status="error", | |
| model_id=model_id, | |
| precision=precision, | |
| quantization=quantization, | |
| destination=destination, | |
| pr_url=None, | |
| ) | |
| yield _log(f"### Error 😢😢😢\n\n{e}") | |
| _delete_cache() | |
| return | |
| def _delete_cache(): | |
| """Delete cache dir between each run to avoid filling up the Space disk.""" | |
| scan = scan_cache_dir() | |
| scan.delete_revisions( | |
| *[rev.commit_hash for repo in scan.repos for rev in repo.revisions] | |
| ) | |
| def _log_usage(**kwargs): | |
| # save in a private dataset | |
| # Taken from https://huggingface.co/spaces/onnx/export | |
| if repo is not None: | |
| repo.git_pull(rebase=True) | |
| with DATASET_LOCAL_FILE.open("a") as csv_file: | |
| writer = csv.DictWriter(csv_file, fieldnames=["time"] + list(kwargs.keys())) | |
| writer.writerow({"time": str(datetime.now()), **kwargs}) | |
| commit_url = repo.push_to_hub() | |
| print("[dataset]", commit_url) | |
| TITLE = """ | |
| <h1 style="font-weight: 900; font-size: 32px; margin-bottom: 10px; margin-top: 10px; text-align: center;"> | |
| Make any BLOOM-like model compatible with bloomz.cpp | |
| </h1> | |
| """ | |
| DESCRIPTION = """ | |
| This Space allows you to automatically export any Bloom-like model hosted on the 🤗 Hub to be compatible with [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp). Converted weights are either exported to a repo you own (or that we create for you) or to the original repo by opening a PR on the target model. Once exported, the model can run with bloomz.cpp. Check out [this guide](https://github.com/NouamaneTazi/bloomz.cpp#usage) to see how! | |
| Don't know which Bloom model are available on the 🤗 Hub? Find a complete list at https://huggingface.co/models?other=bloom. | |
| To use this Space, please follow these steps: | |
| 1. Paste your HF token. You can create one in your [settings page](https://huggingface.co/settings/tokens). The token requires a write-access token to create a PR and upload the weights. | |
| 1. Input a model id from the Hub. This model must be public. | |
| 1. Choose which precision you want to use (default to FP16). | |
| 1. (optional) Opt-in for 4-bit quantization. | |
| 1. (optional) By default a PR to the initial repo will be created. You can choose a different destination repo if you want. The destination repo will be created if it doesn't exist. | |
| 1. Click "Convert!" | |
| That's it! You'll get feedback if it works or not, and if it worked, you'll get the URL of the opened PR 🔥 | |
| If you encounter any issues please let us know [by opening a Discussion](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter/discussions/new). | |
| """ | |
| with gr.Blocks() as demo: | |
| gr.HTML(TITLE) | |
| with gr.Row(): | |
| with gr.Column(scale=50): | |
| gr.Markdown(DESCRIPTION) | |
| with gr.Column(scale=50): | |
| input_token = gr.Text( | |
| max_lines=1, label="Hugging Face token", type="password" | |
| ) | |
| input_model = gr.Text( | |
| max_lines=1, label="Model id (e.g.: bigscience/bloomz-7b1)" | |
| ) | |
| input_precision = gr.Radio( | |
| choices=["FP16", "FP32"], label="Precision", value="FP16" | |
| ) | |
| input_quantization = gr.Checkbox(value=False, label="4-bits quantization") | |
| input_destination = gr.Text( | |
| max_lines=1, | |
| label="Destination (e.g.: bloomz-7b1.cpp) - optional", | |
| ) | |
| btn = gr.Button("Convert!") | |
| output = gr.Markdown(label="Output") | |
| btn.click( | |
| fn=run, | |
| inputs=[ | |
| input_token, | |
| input_model, | |
| input_precision, | |
| input_quantization, | |
| input_destination, | |
| ], | |
| outputs=output, | |
| ) | |
| demo.queue().launch() | |