patrickvonplaten julien-c HF staff commited on
Commit
1609923
0 Parent(s):

Duplicate from safetensors/convert

Browse files

Co-authored-by: Julien Chaumond <julien-c@users.noreply.huggingface.co>

Files changed (7) hide show
  1. .gitattributes +33 -0
  2. .gitignore +1 -0
  3. .vscode/settings.json +4 -0
  4. README.md +17 -0
  5. app.py +94 -0
  6. convert.py +285 -0
  7. requirements.txt +6 -0
.gitattributes ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ftz filter=lfs diff=lfs merge=lfs -text
6
+ *.gz filter=lfs diff=lfs merge=lfs -text
7
+ *.h5 filter=lfs diff=lfs merge=lfs -text
8
+ *.joblib filter=lfs diff=lfs merge=lfs -text
9
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
10
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
11
+ *.model filter=lfs diff=lfs merge=lfs -text
12
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
13
+ *.npy filter=lfs diff=lfs merge=lfs -text
14
+ *.npz filter=lfs diff=lfs merge=lfs -text
15
+ *.onnx filter=lfs diff=lfs merge=lfs -text
16
+ *.ot filter=lfs diff=lfs merge=lfs -text
17
+ *.parquet filter=lfs diff=lfs merge=lfs -text
18
+ *.pb filter=lfs diff=lfs merge=lfs -text
19
+ *.pickle filter=lfs diff=lfs merge=lfs -text
20
+ *.pkl filter=lfs diff=lfs merge=lfs -text
21
+ *.pt filter=lfs diff=lfs merge=lfs -text
22
+ *.pth filter=lfs diff=lfs merge=lfs -text
23
+ *.rar filter=lfs diff=lfs merge=lfs -text
24
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
25
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
26
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
27
+ *.tflite filter=lfs diff=lfs merge=lfs -text
28
+ *.tgz filter=lfs diff=lfs merge=lfs -text
29
+ *.wasm filter=lfs diff=lfs merge=lfs -text
30
+ *.xz filter=lfs diff=lfs merge=lfs -text
31
+ *.zip filter=lfs diff=lfs merge=lfs -text
32
+ *.zst filter=lfs diff=lfs merge=lfs -text
33
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ .env/
.vscode/settings.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "editor.formatOnSave": true,
3
+ "python.formatting.provider": "black"
4
+ }
README.md ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Convert to Safetensors
3
+ emoji: 🐶
4
+ colorFrom: yellow
5
+ colorTo: red
6
+ sdk: gradio
7
+ sdk_version: 3.8.1
8
+ app_file: app.py
9
+ pinned: true
10
+ license: apache-2.0
11
+ models: []
12
+ datasets:
13
+ - safetensors/conversions
14
+ duplicated_from: safetensors/convert
15
+ ---
16
+
17
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import csv
2
+ from datetime import datetime
3
+ import os
4
+ from typing import Optional
5
+ import gradio as gr
6
+
7
+ from convert import convert
8
+ from huggingface_hub import HfApi, Repository
9
+
10
+
11
+ DATASET_REPO_URL = "https://huggingface.co/datasets/safetensors/conversions"
12
+ DATA_FILENAME = "data.csv"
13
+ DATA_FILE = os.path.join("data", DATA_FILENAME)
14
+
15
+ HF_TOKEN = os.environ.get("HF_TOKEN")
16
+
17
+ repo: Optional[Repository] = None
18
+ if HF_TOKEN:
19
+ repo = Repository(local_dir="data", clone_from=DATASET_REPO_URL, token=HF_TOKEN)
20
+
21
+
22
+ def run(token: str, model_id: str) -> str:
23
+ if token == "" or model_id == "":
24
+ return """
25
+ ### Invalid input 🐞
26
+
27
+ Please fill a token and model_id.
28
+ """
29
+ try:
30
+ api = HfApi(token=token)
31
+ is_private = api.model_info(repo_id=model_id).private
32
+ print("is_private", is_private)
33
+
34
+ commit_info = convert(api=api, model_id=model_id)
35
+ print("[commit_info]", commit_info)
36
+
37
+ # save in a (public) dataset:
38
+ if repo is not None and not is_private:
39
+ repo.git_pull(rebase=True)
40
+ print("pulled")
41
+ with open(DATA_FILE, "a") as csvfile:
42
+ writer = csv.DictWriter(
43
+ csvfile, fieldnames=["model_id", "pr_url", "time"]
44
+ )
45
+ writer.writerow(
46
+ {
47
+ "model_id": model_id,
48
+ "pr_url": commit_info.pr_url,
49
+ "time": str(datetime.now()),
50
+ }
51
+ )
52
+ commit_url = repo.push_to_hub()
53
+ print("[dataset]", commit_url)
54
+
55
+ return f"""
56
+ ### Success 🔥
57
+
58
+ Yay! This model was successfully converted and a PR was open using your token, here:
59
+
60
+ [{commit_info.pr_url}]({commit_info.pr_url})
61
+ """
62
+ except Exception as e:
63
+ return f"""
64
+ ### Error 😢😢😢
65
+
66
+ {e}
67
+ """
68
+
69
+
70
+ DESCRIPTION = """
71
+ The steps are the following:
72
+
73
+ - Paste a read-access token from hf.co/settings/tokens. Read access is enough given that we will open a PR against the source repo.
74
+ - Input a model id from the Hub
75
+ - Click "Submit"
76
+ - That's it! You'll get feedback if it works or not, and if it worked, you'll get the URL of the opened PR 🔥
77
+
78
+ ⚠️ For now only `pytorch_model.bin` files are supported but we'll extend in the future.
79
+ """
80
+
81
+ demo = gr.Interface(
82
+ title="Convert any model to Safetensors and open a PR",
83
+ description=DESCRIPTION,
84
+ allow_flagging="never",
85
+ article="Check out the [Safetensors repo on GitHub](https://github.com/huggingface/safetensors)",
86
+ inputs=[
87
+ gr.Text(max_lines=1, label="your_hf_token"),
88
+ gr.Text(max_lines=1, label="model_id"),
89
+ ],
90
+ outputs=[gr.Markdown(label="output")],
91
+ fn=run,
92
+ )
93
+
94
+ demo.launch()
convert.py ADDED
@@ -0,0 +1,285 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import os
4
+ import shutil
5
+ from collections import defaultdict
6
+ from inspect import signature
7
+ from tempfile import TemporaryDirectory
8
+ from typing import Dict, List, Optional, Set
9
+
10
+ import torch
11
+
12
+ from huggingface_hub import CommitInfo, CommitOperationAdd, Discussion, HfApi, hf_hub_download
13
+ from huggingface_hub.file_download import repo_folder_name
14
+ from safetensors.torch import load_file, save_file
15
+ from transformers import AutoConfig
16
+ from transformers.pipelines.base import infer_framework_load_model
17
+
18
+
19
+ class AlreadyExists(Exception):
20
+ pass
21
+
22
+
23
+ def shared_pointers(tensors):
24
+ ptrs = defaultdict(list)
25
+ for k, v in tensors.items():
26
+ ptrs[v.data_ptr()].append(k)
27
+ failing = []
28
+ for ptr, names in ptrs.items():
29
+ if len(names) > 1:
30
+ failing.append(names)
31
+ return failing
32
+
33
+
34
+ def check_file_size(sf_filename: str, pt_filename: str):
35
+ sf_size = os.stat(sf_filename).st_size
36
+ pt_size = os.stat(pt_filename).st_size
37
+
38
+ if (sf_size - pt_size) / pt_size > 0.01:
39
+ raise RuntimeError(
40
+ f"""The file size different is more than 1%:
41
+ - {sf_filename}: {sf_size}
42
+ - {pt_filename}: {pt_size}
43
+ """
44
+ )
45
+
46
+
47
+ def rename(pt_filename: str) -> str:
48
+ filename, ext = os.path.splitext(pt_filename)
49
+ local = f"{filename}.safetensors"
50
+ local = local.replace("pytorch_model", "model")
51
+ return local
52
+
53
+
54
+ def convert_multi(model_id: str, folder: str) -> List["CommitOperationAdd"]:
55
+ filename = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin.index.json")
56
+ with open(filename, "r") as f:
57
+ data = json.load(f)
58
+
59
+ filenames = set(data["weight_map"].values())
60
+ local_filenames = []
61
+ for filename in filenames:
62
+ pt_filename = hf_hub_download(repo_id=model_id, filename=filename)
63
+
64
+ sf_filename = rename(pt_filename)
65
+ sf_filename = os.path.join(folder, sf_filename)
66
+ convert_file(pt_filename, sf_filename)
67
+ local_filenames.append(sf_filename)
68
+
69
+ index = os.path.join(folder, "model.safetensors.index.json")
70
+ with open(index, "w") as f:
71
+ newdata = {k: v for k, v in data.items()}
72
+ newmap = {k: rename(v) for k, v in data["weight_map"].items()}
73
+ newdata["weight_map"] = newmap
74
+ json.dump(newdata, f, indent=4)
75
+ local_filenames.append(index)
76
+
77
+ operations = [
78
+ CommitOperationAdd(path_in_repo=local.split("/")[-1], path_or_fileobj=local) for local in local_filenames
79
+ ]
80
+
81
+ return operations
82
+
83
+
84
+ def convert_single(model_id: str, folder: str) -> List["CommitOperationAdd"]:
85
+ pt_filename = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin")
86
+
87
+ sf_name = "model.safetensors"
88
+ sf_filename = os.path.join(folder, sf_name)
89
+ convert_file(pt_filename, sf_filename)
90
+ operations = [CommitOperationAdd(path_in_repo=sf_name, path_or_fileobj=sf_filename)]
91
+ return operations
92
+
93
+
94
+ def convert_file(
95
+ pt_filename: str,
96
+ sf_filename: str,
97
+ ):
98
+ loaded = torch.load(pt_filename, map_location="cpu")
99
+ if "state_dict" in loaded:
100
+ loaded = loaded["state_dict"]
101
+ shared = shared_pointers(loaded)
102
+ for shared_weights in shared:
103
+ for name in shared_weights[1:]:
104
+ loaded.pop(name)
105
+
106
+ # For tensors to be contiguous
107
+ loaded = {k: v.contiguous() for k, v in loaded.items()}
108
+
109
+ dirname = os.path.dirname(sf_filename)
110
+ os.makedirs(dirname, exist_ok=True)
111
+ save_file(loaded, sf_filename, metadata={"format": "pt"})
112
+ check_file_size(sf_filename, pt_filename)
113
+ reloaded = load_file(sf_filename)
114
+ for k in loaded:
115
+ pt_tensor = loaded[k]
116
+ sf_tensor = reloaded[k]
117
+ if not torch.equal(pt_tensor, sf_tensor):
118
+ raise RuntimeError(f"The output tensors do not match for key {k}")
119
+
120
+
121
+ def create_diff(pt_infos: Dict[str, List[str]], sf_infos: Dict[str, List[str]]) -> str:
122
+ errors = []
123
+ for key in ["missing_keys", "mismatched_keys", "unexpected_keys"]:
124
+ pt_set = set(pt_infos[key])
125
+ sf_set = set(sf_infos[key])
126
+
127
+ pt_only = pt_set - sf_set
128
+ sf_only = sf_set - pt_set
129
+
130
+ if pt_only:
131
+ errors.append(f"{key} : PT warnings contain {pt_only} which are not present in SF warnings")
132
+ if sf_only:
133
+ errors.append(f"{key} : SF warnings contain {sf_only} which are not present in PT warnings")
134
+ return "\n".join(errors)
135
+
136
+
137
+ def check_final_model(model_id: str, folder: str):
138
+ config = hf_hub_download(repo_id=model_id, filename="config.json")
139
+ shutil.copy(config, os.path.join(folder, "config.json"))
140
+ config = AutoConfig.from_pretrained(folder)
141
+
142
+ _, (pt_model, pt_infos) = infer_framework_load_model(model_id, config, output_loading_info=True)
143
+ _, (sf_model, sf_infos) = infer_framework_load_model(folder, config, output_loading_info=True)
144
+
145
+ if pt_infos != sf_infos:
146
+ error_string = create_diff(pt_infos, sf_infos)
147
+ raise ValueError(f"Different infos when reloading the model: {error_string}")
148
+
149
+ pt_params = pt_model.state_dict()
150
+ sf_params = sf_model.state_dict()
151
+
152
+ pt_shared = shared_pointers(pt_params)
153
+ sf_shared = shared_pointers(sf_params)
154
+ if pt_shared != sf_shared:
155
+ raise RuntimeError("The reconstructed model is wrong, shared tensors are different {shared_pt} != {shared_tf}")
156
+
157
+ sig = signature(pt_model.forward)
158
+ input_ids = torch.arange(10).unsqueeze(0)
159
+ pixel_values = torch.randn(1, 3, 224, 224)
160
+ input_values = torch.arange(1000).float().unsqueeze(0)
161
+ kwargs = {}
162
+ if "input_ids" in sig.parameters:
163
+ kwargs["input_ids"] = input_ids
164
+ if "decoder_input_ids" in sig.parameters:
165
+ kwargs["decoder_input_ids"] = input_ids
166
+ if "pixel_values" in sig.parameters:
167
+ kwargs["pixel_values"] = pixel_values
168
+ if "input_values" in sig.parameters:
169
+ kwargs["input_values"] = input_values
170
+ if "bbox" in sig.parameters:
171
+ kwargs["bbox"] = torch.zeros((1, 10, 4)).long()
172
+ if "image" in sig.parameters:
173
+ kwargs["image"] = pixel_values
174
+
175
+ if torch.cuda.is_available():
176
+ pt_model = pt_model.cuda()
177
+ sf_model = sf_model.cuda()
178
+ kwargs = {k: v.cuda() for k, v in kwargs.items()}
179
+
180
+ pt_logits = pt_model(**kwargs)[0]
181
+ sf_logits = sf_model(**kwargs)[0]
182
+
183
+ torch.testing.assert_close(sf_logits, pt_logits)
184
+ print(f"Model {model_id} is ok !")
185
+
186
+
187
+ def previous_pr(api: "HfApi", model_id: str, pr_title: str) -> Optional["Discussion"]:
188
+ try:
189
+ discussions = api.get_repo_discussions(repo_id=model_id)
190
+ except Exception:
191
+ return None
192
+ for discussion in discussions:
193
+ if discussion.status == "open" and discussion.is_pull_request and discussion.title == pr_title:
194
+ return discussion
195
+
196
+
197
+ def convert_generic(model_id: str, folder: str, filenames: Set[str]) -> List["CommitOperationAdd"]:
198
+ operations = []
199
+
200
+ extensions = set([".bin", ".ckpt"])
201
+ for filename in filenames:
202
+ prefix, ext = os.path.splitext(filename)
203
+ if ext in extensions:
204
+ pt_filename = hf_hub_download(model_id, filename=filename)
205
+ dirname, raw_filename = os.path.split(filename)
206
+ if raw_filename == "pytorch_model.bin":
207
+ # XXX: This is a special case to handle `transformers` and the
208
+ # `transformers` part of the model which is actually loaded by `transformers`.
209
+ sf_in_repo = os.path.join(dirname, "model.safetensors")
210
+ else:
211
+ sf_in_repo = f"{prefix}.safetensors"
212
+ sf_filename = os.path.join(folder, sf_in_repo)
213
+ convert_file(pt_filename, sf_filename)
214
+ operations.append(CommitOperationAdd(path_in_repo=sf_in_repo, path_or_fileobj=sf_filename))
215
+ return operations
216
+
217
+
218
+ def convert(api: "HfApi", model_id: str, force: bool = False) -> Optional["CommitInfo"]:
219
+ pr_title = "Adding `safetensors` variant of this model"
220
+ info = api.model_info(model_id)
221
+ filenames = set(s.rfilename for s in info.siblings)
222
+
223
+ with TemporaryDirectory() as d:
224
+ folder = os.path.join(d, repo_folder_name(repo_id=model_id, repo_type="models"))
225
+ os.makedirs(folder)
226
+ new_pr = None
227
+ try:
228
+ operations = None
229
+ pr = previous_pr(api, model_id, pr_title)
230
+
231
+ library_name = getattr(info, "library_name", None)
232
+ if any(filename.endswith(".safetensors") for filename in filenames) and not force:
233
+ raise AlreadyExists(f"Model {model_id} is already converted, skipping..")
234
+ elif pr is not None and not force:
235
+ url = f"https://huggingface.co/{model_id}/discussions/{pr.num}"
236
+ new_pr = pr
237
+ raise AlreadyExists(f"Model {model_id} already has an open PR check out {url}")
238
+ elif library_name == "transformers":
239
+ if "pytorch_model.bin" in filenames:
240
+ operations = convert_single(model_id, folder)
241
+ elif "pytorch_model.bin.index.json" in filenames:
242
+ operations = convert_multi(model_id, folder)
243
+ else:
244
+ raise RuntimeError(f"Model {model_id} doesn't seem to be a valid pytorch model. Cannot convert")
245
+ check_final_model(model_id, folder)
246
+ else:
247
+ operations = convert_generic(model_id, folder, filenames)
248
+
249
+ if operations:
250
+ new_pr = api.create_commit(
251
+ repo_id=model_id,
252
+ operations=operations,
253
+ commit_message=pr_title,
254
+ create_pr=True,
255
+ )
256
+ print(f"Pr created at {new_pr.pr_url}")
257
+ else:
258
+ print("No files to convert")
259
+ finally:
260
+ shutil.rmtree(folder)
261
+ return new_pr
262
+
263
+
264
+ if __name__ == "__main__":
265
+ DESCRIPTION = """
266
+ Simple utility tool to convert automatically some weights on the hub to `safetensors` format.
267
+ It is PyTorch exclusive for now.
268
+ It works by downloading the weights (PT), converting them locally, and uploading them back
269
+ as a PR on the hub.
270
+ """
271
+ parser = argparse.ArgumentParser(description=DESCRIPTION)
272
+ parser.add_argument(
273
+ "model_id",
274
+ type=str,
275
+ help="The name of the model on the hub to convert. E.g. `gpt2` or `facebook/wav2vec2-base-960h`",
276
+ )
277
+ parser.add_argument(
278
+ "--force",
279
+ action="store_true",
280
+ help="Create the PR even if it already exists of if the model was already converted.",
281
+ )
282
+ args = parser.parse_args()
283
+ model_id = args.model_id
284
+ api = HfApi()
285
+ convert(api, model_id, force=args.force)
requirements.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ huggingface_hub
2
+ setuptools_rust
3
+ safetensors
4
+ torch==1.13.1
5
+ transformers
6
+ pytorch_lightning