testbot commited on
Commit
8ec55c7
β€’
1 Parent(s): a4ab7ed

Requested changes

Browse files
Files changed (2) hide show
  1. app.py +179 -51
  2. convert.py +6 -1
app.py CHANGED
@@ -1,99 +1,227 @@
 
1
  from pathlib import Path
2
  from tempfile import TemporaryDirectory
3
 
4
  import gradio as gr
5
- from huggingface_hub import CommitOperationAdd, HfApi
6
- from huggingface_hub.utils import RepositoryNotFoundError
7
 
8
  from convert import convert
9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
 
11
  def run(
12
  token: str, model_id: str, precision: str, quantization: bool, destination: str
13
- ) -> str:
 
 
 
 
 
 
 
14
  if token == "" or model_id == "":
15
- return """
 
16
  ### Invalid input 🐞
17
 
18
  Please fill a token and model_id.
19
  """
 
 
20
  if destination == "":
 
21
  destination = model_id
22
 
23
  api = HfApi(token=token)
24
  try:
25
  # TODO: make a PR to bloomz.cpp to be able to pass a token
26
- api.model_info(
27
- repo_id=model_id, token=False
28
- ) # only public repos are accessible
29
  except RepositoryNotFoundError:
30
- return f"""
 
31
  ### Error 😒😒😒
32
 
33
  Repository {model_id} not found. Only public models are convertible at the moment.
34
  """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
 
36
  try:
37
  with TemporaryDirectory() as cache_folder:
38
- model_path = convert(
39
- cache_folder=Path(cache_folder),
40
- model_id=model_id,
41
- precision=precision,
42
- quantization=quantization,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  )
44
- print("[model_path]", model_path)
45
 
46
- commit_info = api.create_commit(
47
  repo_id=destination,
48
- operations=[
49
- CommitOperationAdd(
50
- path_or_fileobj=model_path, path_in_repo=model_path.name
51
- )
52
- ],
53
- create_pr=True,
54
- commit_message=f"Add {model_path.name} from bloomz.cpp converter.",
55
  )
 
56
 
57
- return f"""
 
58
  ### Success πŸ”₯
59
- Yay! This model was successfully converted and a PR was open using your token, here:
60
-
61
- # [{commit_info.pr_url}]({commit_info.pr_url})
62
  """
 
 
63
  except Exception as e:
64
- return f"""
 
65
  ### Error 😒😒😒
66
 
67
  {e}
68
  """
 
 
69
 
70
 
 
 
 
 
 
 
71
  DESCRIPTION = """
72
- The steps are the following:
73
- - Paste your HF token. You can create one in your [settings page](https://huggingface.co/settings/tokens).
74
- - Input a model id from the Hub. This model must be public.
75
- - Choose which precision you want to use (default to FP16).
76
- - (optional) Opt-in for 4-bit quantization.
77
- - (optional) By default a PR to the initial repo will be created. You can choose a different destination repo if you want. The destination repo must exist.
78
- - Click Submit:
 
 
 
 
 
79
 
80
  That's it! You'll get feedback if it works or not, and if it worked, you'll get the URL of the opened PR πŸ”₯
 
81
  """
82
 
83
- demo = gr.Interface(
84
- title="Convert any BLOOM-like model to be compatible with bloomz.cpp",
85
- description=DESCRIPTION,
86
- allow_flagging="never",
87
- article="Check out the [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp) repo on GitHub",
88
- inputs=[
89
- gr.Text(max_lines=1, label="your hf_token"),
90
- gr.Text(max_lines=1, label="model_id (e.g.: bigscience/bloomz-7b1)"),
91
- gr.Radio(choices=["FP16", "FP32"], label="Precision", value="FP16"),
92
- gr.Checkbox(value=False, label="4-bits quantization"),
93
- gr.Text(max_lines=1, label="destination (e.g.: my-username/bloomz-7b1.cpp)"),
94
- ],
95
- outputs=[gr.Markdown(label="output")],
96
- fn=run,
97
- ).queue()
98
-
99
- demo.launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
  from pathlib import Path
3
  from tempfile import TemporaryDirectory
4
 
5
  import gradio as gr
6
+ from huggingface_hub import HfApi, ModelCard
7
+ from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
8
 
9
  from convert import convert
10
 
11
+ # Repo with files totalling more than 24GB are not converted. Avoid to have a memory issue.
12
+ try:
13
+ MAX_REPO_SIZE = int(os.environ.get("MAX_REPO_SIZE"))
14
+ except:
15
+ MAX_REPO_SIZE = 24 * 1000 * 1000 * 1000
16
+
17
+
18
+ class Generator:
19
+ # Taken from https://stackoverflow.com/a/34073559
20
+ # Allows to log process in Gradio
21
+ def __init__(self, gen):
22
+ self.gen = gen
23
+
24
+ def __iter__(self):
25
+ self.value = yield from self.gen
26
+
27
 
28
  def run(
29
  token: str, model_id: str, precision: str, quantization: bool, destination: str
30
+ ):
31
+ _all_logs = []
32
+
33
+ def _log(msg: str):
34
+ print(msg) # for container logs
35
+ _all_logs.append(msg)
36
+ return "\n\n".join(_all_logs) # for Gradio output
37
+
38
  if token == "" or model_id == "":
39
+ yield _log(
40
+ """
41
  ### Invalid input 🐞
42
 
43
  Please fill a token and model_id.
44
  """
45
+ )
46
+ return
47
  if destination == "":
48
+ _log("Destination not provided. Will default to the initial repo.")
49
  destination = model_id
50
 
51
  api = HfApi(token=token)
52
  try:
53
  # TODO: make a PR to bloomz.cpp to be able to pass a token
54
+ model_info = api.model_info(repo_id=model_id, files_metadata=True, token=False)
55
+ _log(f"Model {model_id} exists.")
 
56
  except RepositoryNotFoundError:
57
+ yield _log(
58
+ f"""
59
  ### Error 😒😒😒
60
 
61
  Repository {model_id} not found. Only public models are convertible at the moment.
62
  """
63
+ )
64
+ return
65
+
66
+ total_size = sum(
67
+ file.size
68
+ for file in model_info.siblings
69
+ if file.rfilename.endswith(".pt") or file.rfilename.endswith(".bin")
70
+ )
71
+ if total_size > MAX_REPO_SIZE:
72
+ yield _log(
73
+ f"""
74
+ ### Unprocessable 😒😒😒
75
+
76
+ Model {model_id} is too big and cannot be processed in this Space. This Space needs to be able to load the model in memory before converting it. To avoid a memory issue, we do not process models bigger than {MAX_REPO_SIZE}b.
77
+
78
+ You have 2 options:
79
+ - [Duplicate this Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter?duplicate=true) and assign a bigger machine. You will need to set 'MAX_REPO_SIZE' as a secret to overwrite the default value. Once you are done, remove the upgraded hardware and/or delete the Space.
80
+ - Manually convert the weights by following [this guide](https://github.com/NouamaneTazi/bloomz.cpp#usage).
81
+ """
82
+ )
83
+ return
84
 
85
  try:
86
  with TemporaryDirectory() as cache_folder:
87
+ convert_progress = Generator(
88
+ convert(
89
+ cache_folder=Path(cache_folder),
90
+ model_id=model_id,
91
+ precision=precision,
92
+ quantization=quantization,
93
+ )
94
+ )
95
+ for msg in convert_progress:
96
+ yield _log(msg)
97
+ model_path = convert_progress.value
98
+ yield _log(f"Model converted: {model_path}")
99
+
100
+ destination_url = api.create_repo(repo_id=destination, exist_ok=True)
101
+ destination = destination_url.repo_id
102
+ yield _log(f"Destination model: {destination_url}")
103
+ pr = api.create_pull_request(
104
+ repo_id=destination_url.repo_id,
105
+ title=f"[WIP] Add {model_path.name} from bloomz.cpp converter.",
106
+ description="This PR has been created using the [bloomz.cpp converter Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter). It adds weights compatible with the [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp#usage) project.",
107
+ )
108
+ pr_url = f"https://huggingface.co/{destination}/discussions/{pr.num}"
109
+ yield _log(f"Created PR: {pr_url} (empty)")
110
+
111
+ yield _log(f"Uploading model to PR")
112
+ api.upload_file(
113
+ repo_id=destination,
114
+ path_or_fileobj=model_path,
115
+ path_in_repo=model_path.name,
116
+ revision=pr.git_reference,
117
+ )
118
+ yield _log(f"Model uploaded to PR")
119
+
120
+ yield _log(f"Modifying model card in PR (add `bloom` and `ggml` tags)")
121
+ try:
122
+ card = ModelCard.load(repo_id_or_path=destination)
123
+ except EntryNotFoundError: # new repo => no model card yet
124
+ card = ModelCard(
125
+ "This model contains a model based on the Bloom architecture with weights compatible with [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp). This model card has been automatically generated [by the bloomz.cpp converter Space](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter) and must be completed."
126
+ )
127
+ if card.data.tags is None:
128
+ card.data.tags = []
129
+ tags = card.data.tags
130
+ if "ggml" not in tags:
131
+ tags.append("ggml")
132
+ if "bloom" not in tags:
133
+ tags.append("bloom")
134
+ card.push_to_hub(
135
+ repo_id=destination, token=token, revision=pr.git_reference
136
  )
137
+ yield _log(f"Model card modified in PR.")
138
 
139
+ api.change_discussion_status(
140
  repo_id=destination,
141
+ discussion_num=pr.num,
142
+ new_status="open",
143
+ comment="PR is now complete and ready to be reviewed.",
 
 
 
 
144
  )
145
+ yield _log(f"[PR]({pr_url}) is complete and ready to be reviewed.")
146
 
147
+ yield _log(
148
+ f"""
149
  ### Success πŸ”₯
150
+ Yay! This model was successfully converted! Make sure to let the repo owner know about it and review your PR. You might need to complete the PR manually, especially to add information in the model card.
 
 
151
  """
152
+ )
153
+ return
154
  except Exception as e:
155
+ yield _log(
156
+ f"""
157
  ### Error 😒😒😒
158
 
159
  {e}
160
  """
161
+ )
162
+ return
163
 
164
 
165
+ TITLE = """
166
+ <h1 style="font-weight: 900; font-size: 32px; margin-bottom: 10px; margin-top: 10px; text-align: center;">
167
+ Make any BLOOM-like model compatible with bloomz.cpp
168
+ </h1>
169
+ """
170
+
171
  DESCRIPTION = """
172
+ This Space allows you to automatically export any Bloom-like model hosted on the πŸ€— Hub to be compatible with [bloomz.cpp](https://github.com/NouamaneTazi/bloomz.cpp). Converted weights are either exported to a repo you own (or that we create for you) or to the original repo by opening a PR on the target model. Once exported, the model can run with bloomz.cpp. Check out [this guide](https://github.com/NouamaneTazi/bloomz.cpp#usage) to see how!
173
+
174
+ Don't know which Bloom model are available on the πŸ€— Hub? Find a complete list at https://huggingface.co/models?other=bloom.
175
+
176
+ To use this Space, please follow these steps:
177
+
178
+ 1. Paste your HF token. You can create one in your [settings page](https://huggingface.co/settings/tokens). The token requires a write-access token to create a PR and upload the weights.
179
+ 1. Input a model id from the Hub. This model must be public.
180
+ 1. Choose which precision you want to use (default to FP16).
181
+ 1. (optional) Opt-in for 4-bit quantization.
182
+ 1. (optional) By default a PR to the initial repo will be created. You can choose a different destination repo if you want. The destination repo will be created if it doesn't exist.
183
+ 1. Click "Convert!"
184
 
185
  That's it! You'll get feedback if it works or not, and if it worked, you'll get the URL of the opened PR πŸ”₯
186
+ If you encounter any issues please let us know [by opening a Discussion](https://huggingface.co/spaces/Wauplin/bloomz.cpp-converter/discussions/new).
187
  """
188
 
189
+
190
+ with gr.Blocks() as demo:
191
+ gr.HTML(TITLE)
192
+
193
+ with gr.Row():
194
+ with gr.Column(scale=50):
195
+ gr.Markdown(DESCRIPTION)
196
+
197
+ with gr.Column(scale=50):
198
+ input_token = gr.Text(max_lines=1, label="Hugging Face token")
199
+ input_model = gr.Text(
200
+ max_lines=1, label="Model id (e.g.: bigscience/bloomz-7b1)"
201
+ )
202
+ input_precision = gr.Radio(
203
+ choices=["FP16", "FP32"], label="Precision", value="FP16"
204
+ )
205
+ input_quantization = gr.Checkbox(value=False, label="4-bits quantization")
206
+ input_destination = gr.Text(
207
+ max_lines=1,
208
+ label="Destination (e.g.: my-username/bloomz-7b1.cpp) - optional",
209
+ )
210
+ btn = gr.Button("Convert!")
211
+
212
+ output = gr.Markdown(label="Output")
213
+
214
+ btn.click(
215
+ fn=run,
216
+ inputs=[
217
+ input_token,
218
+ input_model,
219
+ input_precision,
220
+ input_quantization,
221
+ input_destination,
222
+ ],
223
+ outputs=output,
224
+ )
225
+
226
+
227
+ demo.queue().launch()
convert.py CHANGED
@@ -1,12 +1,13 @@
1
  from pathlib import Path
2
  from subprocess import run
 
3
 
4
  BLOOMZ_FOLDER = Path(__file__).parent / "bloomz.cpp"
5
 
6
 
7
  def convert(
8
  cache_folder: Path, model_id: str, precision: str, quantization: bool
9
- ) -> Path:
10
  # Conversion
11
  cmd = [
12
  "python",
@@ -16,6 +17,7 @@ def convert(
16
  ]
17
  if precision == "FP32":
18
  cmd.append("--use-fp32")
 
19
  run(cmd, check=True)
20
 
21
  # Model file should exist
@@ -23,6 +25,7 @@ def convert(
23
  _, model_name = model_id.split("/")
24
  model_path = cache_folder / f"ggml-model-{model_name}-{f_suffix}.bin"
25
  assert model_path.is_file()
 
26
 
27
  # Quantization
28
  if quantization:
@@ -35,9 +38,11 @@ def convert(
35
  str(q_model_path),
36
  "2",
37
  ]
 
38
  run(cmd, check=True)
39
  assert q_model_path.is_file()
40
  model_path = q_model_path
 
41
 
42
  # Return
43
  return model_path
 
1
  from pathlib import Path
2
  from subprocess import run
3
+ from typing import Generator
4
 
5
  BLOOMZ_FOLDER = Path(__file__).parent / "bloomz.cpp"
6
 
7
 
8
  def convert(
9
  cache_folder: Path, model_id: str, precision: str, quantization: bool
10
+ ) -> Generator[str, Path, None]:
11
  # Conversion
12
  cmd = [
13
  "python",
 
17
  ]
18
  if precision == "FP32":
19
  cmd.append("--use-fp32")
20
+ yield f"Running command: `{' '.join(cmd)}`"
21
  run(cmd, check=True)
22
 
23
  # Model file should exist
 
25
  _, model_name = model_id.split("/")
26
  model_path = cache_folder / f"ggml-model-{model_name}-{f_suffix}.bin"
27
  assert model_path.is_file()
28
+ yield f"Model successfully converted to ggml: {model_path}"
29
 
30
  # Quantization
31
  if quantization:
 
38
  str(q_model_path),
39
  "2",
40
  ]
41
+ yield f"Running command: `{' '.join(cmd)}`"
42
  run(cmd, check=True)
43
  assert q_model_path.is_file()
44
  model_path = q_model_path
45
+ yield f"Model successfully quantized: {model_path}"
46
 
47
  # Return
48
  return model_path