File size: 11,586 Bytes
63f0f28
8ec55c7
526c343
63f0f28
1fc2558
713b2b5
63f0f28
1fc2558
 
63f0f28
8ec55c7
1fc2558
 
 
8ec55c7
 
 
 
 
 
63f0f28
 
 
 
a05357f
63f0f28
 
 
 
 
 
 
 
 
 
 
8ec55c7
 
 
 
 
 
 
 
 
 
1fc2558
713b2b5
 
8ec55c7
63f0f28
 
 
 
 
 
 
 
8ec55c7
 
 
 
 
 
 
1fc2558
042ed31
63f0f28
 
 
 
 
 
 
 
8ec55c7
713b2b5
8ec55c7
713b2b5
1fc2558
 
 
 
8ec55c7
 
1fc2558
8ec55c7
042ed31
8ec55c7
63f0f28
 
 
 
 
 
 
 
8ec55c7
 
7db56b8
 
 
 
 
8ec55c7
7db56b8
 
042ed31
7db56b8
63f0f28
 
 
 
 
 
 
 
7db56b8
1fc2558
713b2b5
8ec55c7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93b7cb2
8ec55c7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
713b2b5
8ec55c7
1fc2558
8ec55c7
713b2b5
8ec55c7
 
 
713b2b5
8ec55c7
713b2b5
8ec55c7
042ed31
8ec55c7
63f0f28
 
 
 
 
 
 
 
526c343
8c68c2f
8ec55c7
1fc2558
63f0f28
 
 
 
 
 
 
 
042ed31
8c68c2f
8ec55c7
1fc2558
 
8c68c2f
 
 
 
 
 
 
 
63f0f28
 
 
 
 
 
 
 
 
 
 
 
8ec55c7
 
 
 
 
 
1fc2558
8ec55c7
 
 
 
 
 
 
 
 
 
 
 
713b2b5
 
8ec55c7
1fc2558
 
8ec55c7
 
 
 
 
 
 
 
 
8c68c2f
 
 
8ec55c7
 
 
 
 
 
 
 
 
8c4ee1f
8ec55c7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
import csv
import os
import shutil
from datetime import datetime
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Optional

import gradio as gr
from huggingface_hub import HfApi, ModelCard, Repository, scan_cache_dir
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError

from convert import convert

# Repo with files totalling more than 24GB are not converted. Avoid to have a memory issue.
try:
    MAX_REPO_SIZE = int(os.environ.get("MAX_REPO_SIZE"))
except:
    MAX_REPO_SIZE = 24 * 1000 * 1000 * 1000

# Used to log Space usage
# Taken from https://huggingface.co/spaces/onnx/export
DATASET_REPO_ID = "Wauplin/bloom.cpp-converters"
DATASET_LOCAL_DIR = "usage_data"
DATASET_LOCAL_FILE = Path(DATASET_LOCAL_DIR) / "data.csv"
HF_TOKEN = os.environ.get("HF_TOKEN")

repo: Optional[Repository] = None
if HF_TOKEN:
    repo = Repository(
        local_dir=DATASET_LOCAL_DIR,
        clone_from=DATASET_REPO_ID,
        repo_type="dataset",
        token=HF_TOKEN,
    )


class Generator:
    # Taken from https://stackoverflow.com/a/34073559
    # Allows to log process in Gradio
    def __init__(self, gen):
        self.gen = gen

    def __iter__(self):
        self.value = yield from self.gen


def run(
    token: str, model_id: str, precision: str, quantization: bool, destination: str
):
    _log_usage(
        status="start",
        model_id=model_id,
        precision=precision,
        quantization=quantization,
        destination=destination,
        pr_url=None,
    )
    _all_logs = []

    def _log(msg: str):
        print(msg)  # for container logs
        _all_logs.append(msg)
        return "\n\n".join(_all_logs)  # for Gradio output

    if token == "" or model_id == "":
        yield _log("### Invalid input 🐞\n\nPlease fill a token and model_id.")
        _log_usage(
            status="invalid input",
            model_id=model_id,
            precision=precision,
            quantization=quantization,
            destination=destination,
            pr_url=None,
        )
        return
    if destination == "":
        _log("Destination not provided. Will default to the initial repo.")
        destination = model_id

    api = HfApi(token=token)
    try:
        # TODO: make a PR to bloomz.cpp to be able to pass a token
        model_info = api.model_info(repo_id=model_id, files_metadata=True, token=False)
        _log(f"Model {model_id} exists.")
    except RepositoryNotFoundError:
        yield _log(
            f"\n### Error 😒😒😒\n\nRepository {model_id} not found. Only public models are convertible at the moment."
        )
        _log_usage(
            status="model not found",
            model_id=model_id,
            precision=precision,
            quantization=quantization,
            destination=destination,
            pr_url=None,
        )
        return

    try:
        total_size = sum(
            file.size
            for file in model_info.siblings
            if file.rfilename.endswith(".pt") or file.rfilename.endswith(".bin")
        )
        if total_size > MAX_REPO_SIZE:
            yield _log(
                f"### Unprocessable 😒😒😒\n\nModel {model_id} is too big and cannot be processed in this Space. This Space needs to be able to load the model in memory before converting it. To avoid a memory issue, we do not process models bigger than {MAX_REPO_SIZE}b.\n\nYou have 2 options:\n- [Duplicate this Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter?duplicate=true) and assign a bigger machine. You will need to set 'MAX_REPO_SIZE' as a secret to overwrite the default value. Once you are done, remove the upgraded hardware and/or delete the Space.\n- Manually convert the weights by following [this guide](https://github.com/NouamaneTazi/bloomz.cpp#usage)."
            )
            _log_usage(
                status="unprocessable",
                model_id=model_id,
                precision=precision,
                quantization=quantization,
                destination=destination,
                pr_url=None,
            )
            return

        with TemporaryDirectory() as cache_folder:
            convert_progress = Generator(
                convert(
                    cache_folder=Path(cache_folder),
                    model_id=model_id,
                    precision=precision,
                    quantization=quantization,
                )
            )
            for msg in convert_progress:
                yield _log(msg)
            model_path = convert_progress.value
            yield _log(f"Model converted: {model_path}")

            destination_url = api.create_repo(repo_id=destination, exist_ok=True)
            destination = destination_url.repo_id
            yield _log(f"Destination model: {destination_url}")
            pr = api.create_pull_request(
                repo_id=destination_url.repo_id,
                title=f"Add {model_path.name} from bloomz.cpp converter.",
                description="This PR has been created using the [bloomz.cpp converter Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter). It adds weights compatible with the [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp#usage) project.",
            )
            pr_url = f"https://huggingface.co/{destination}/discussions/{pr.num}"
            yield _log(f"Created PR: {pr_url} (empty)")

            yield _log(f"Uploading model to PR")
            api.upload_file(
                repo_id=destination,
                path_or_fileobj=model_path,
                path_in_repo=model_path.name,
                revision=pr.git_reference,
            )
            yield _log(f"Model uploaded to PR")

            yield _log(f"Modifying model card in PR (add `bloom` and `ggml` tags)")
            try:
                card = ModelCard.load(repo_id_or_path=destination)
            except EntryNotFoundError:  # new repo => no model card yet
                card = ModelCard(
                    "This model contains a model based on the Bloom architecture with weights compatible with [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp). This model card has been automatically generated [by the bloomz.cpp converter Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter) and must be completed."
                )
            if card.data.tags is None:
                card.data.tags = []
            tags = card.data.tags
            if "ggml" not in tags:
                tags.append("ggml")
            if "bloom" not in tags:
                tags.append("bloom")
            card.push_to_hub(
                repo_id=destination, token=token, revision=pr.git_reference
            )
            yield _log(f"Model card modified in PR.")

            api.change_discussion_status(
                repo_id=destination,
                discussion_num=pr.num,
                new_status="open",
                comment="PR is now complete and ready to be reviewed.",
            )
            yield _log(f"[PR]({pr_url}) is complete and ready to be reviewed.")

            yield _log(
                f"### Success πŸ”₯\n\nYay! This model was successfully converted! Make sure to let the repo owner know about it and review your PR. You might need to complete the PR manually, especially to add information in the model card."
            )
            _log_usage(
                status="success",
                model_id=model_id,
                precision=precision,
                quantization=quantization,
                destination=destination,
                pr_url=pr_url,
            )
            shutil.rmtree(model_path.parent)
            _delete_cache()
            return
    except Exception as e:
        _log_usage(
            status="error",
            model_id=model_id,
            precision=precision,
            quantization=quantization,
            destination=destination,
            pr_url=None,
        )
        yield _log(f"### Error 😒😒😒\n\n{e}")
        _delete_cache()
        return


def _delete_cache():
    """Delete cache dir between each run to avoid filling up the Space disk."""
    scan = scan_cache_dir()
    scan.delete_revisions(
        *[rev.commit_hash for repo in scan.repos for rev in repo.revisions]
    )


def _log_usage(**kwargs):
    # save in a private dataset
    # Taken from https://huggingface.co/spaces/onnx/export
    if repo is not None:
        repo.git_pull(rebase=True)
        with DATASET_LOCAL_FILE.open("a") as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames=["time"] + list(kwargs.keys()))
            writer.writerow({"time": str(datetime.now()), **kwargs})
        commit_url = repo.push_to_hub()
        print("[dataset]", commit_url)


TITLE = """
<h1 style="font-weight: 900; font-size: 32px; margin-bottom: 10px; margin-top: 10px; text-align: center;">
    Make any BLOOM-like model compatible with bloomz.cpp
</h1>
"""

DESCRIPTION = """
This Space allows you to automatically export any Bloom-like model hosted on the πŸ€— Hub to be compatible with [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp). Converted weights are either exported to a repo you own (or that we create for you) or to the original repo by opening a PR on the target model. Once exported, the model can run with bloomz.cpp. Check out [this guide](https://github.com/NouamaneTazi/bloomz.cpp#usage) to see how!

Don't know which Bloom model are available on the πŸ€— Hub? Find a complete list at https://huggingface.co/models?other=bloom.

To use this Space, please follow these steps:

1. Paste your HF token. You can create one in your [settings page](https://huggingface.co/settings/tokens). The token requires a write-access token to create a PR and upload the weights.
1. Input a model id from the Hub. This model must be public.
1. Choose which precision you want to use (default to FP16).
1. (optional) Opt-in for 4-bit quantization.
1. (optional) By default a PR to the initial repo will be created. You can choose a different destination repo if you want. The destination repo will be created if it doesn't exist.
1. Click "Convert!"

That's it! You'll get feedback if it works or not, and if it worked, you'll get the URL of the opened PR πŸ”₯
If you encounter any issues please let us know [by opening a Discussion](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter/discussions/new).
"""


with gr.Blocks() as demo:
    gr.HTML(TITLE)

    with gr.Row():
        with gr.Column(scale=50):
            gr.Markdown(DESCRIPTION)

        with gr.Column(scale=50):
            input_token = gr.Text(
                max_lines=1, label="Hugging Face token", type="password"
            )
            input_model = gr.Text(
                max_lines=1, label="Model id (e.g.: bigscience/bloomz-7b1)"
            )
            input_precision = gr.Radio(
                choices=["FP16", "FP32"], label="Precision", value="FP16"
            )
            input_quantization = gr.Checkbox(value=False, label="4-bits quantization")
            input_destination = gr.Text(
                max_lines=1,
                label="Destination (e.g.: bloomz-7b1.cpp) - optional",
            )
            btn = gr.Button("Convert!")

    output = gr.Markdown(label="Output")

    btn.click(
        fn=run,
        inputs=[
            input_token,
            input_model,
            input_precision,
            input_quantization,
            input_destination,
        ],
        outputs=output,
    )


demo.queue().launch()