Commit
•
ac886c4
1
Parent(s):
a921998
Forgot to double check. (#24)
Browse files- Forgot to double check. (62a0b6efcdc10bcb713dfb1cef91faba3d533291)
Co-authored-by: Snacke Solid <SolidSnacke@users.noreply.huggingface.co>
- gguf-imat-llama-3.py +22 -17
- gguf-imat.py +22 -17
gguf-imat-llama-3.py
CHANGED
@@ -70,6 +70,7 @@ def download_cudart_if_necessary(latest_release_tag):
|
|
70 |
def download_model_repo():
|
71 |
base_dir = os.path.dirname(os.path.abspath(__file__))
|
72 |
models_dir = os.path.join(base_dir, "models")
|
|
|
73 |
if not os.path.exists(models_dir):
|
74 |
os.makedirs(models_dir)
|
75 |
|
@@ -77,27 +78,27 @@ def download_model_repo():
|
|
77 |
model_name = model_id.split("/")[-1]
|
78 |
model_dir = os.path.join(models_dir, model_name)
|
79 |
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
|
86 |
-
|
87 |
-
convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
|
88 |
|
89 |
-
|
90 |
-
|
|
|
|
|
|
|
91 |
|
92 |
-
|
93 |
|
94 |
-
|
95 |
-
|
96 |
-
print("Model repository downloaded successfully.")
|
97 |
|
98 |
-
|
|
|
|
|
99 |
|
100 |
-
|
101 |
|
102 |
def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
|
103 |
convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
|
@@ -115,7 +116,11 @@ def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir,
|
|
115 |
print(f"Original model directory '{model_dir}' deleted.")
|
116 |
else:
|
117 |
print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
|
|
|
|
|
|
|
118 |
|
|
|
119 |
imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
|
120 |
imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
|
121 |
imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
|
@@ -161,4 +166,4 @@ def main():
|
|
161 |
print("Finished preparing resources.")
|
162 |
|
163 |
if __name__ == "__main__":
|
164 |
-
main()
|
|
|
70 |
def download_model_repo():
|
71 |
base_dir = os.path.dirname(os.path.abspath(__file__))
|
72 |
models_dir = os.path.join(base_dir, "models")
|
73 |
+
|
74 |
if not os.path.exists(models_dir):
|
75 |
os.makedirs(models_dir)
|
76 |
|
|
|
78 |
model_name = model_id.split("/")[-1]
|
79 |
model_dir = os.path.join(models_dir, model_name)
|
80 |
|
81 |
+
gguf_dir = os.path.join(base_dir, "models", f"{model_name}-GGUF")
|
82 |
+
gguf_model_path = os.path.join(gguf_dir, f"{model_name}-F16.gguf")
|
83 |
+
imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
|
84 |
+
delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
|
|
|
|
|
|
|
|
|
85 |
|
86 |
+
if os.path.exists(gguf_model_path):
|
87 |
+
create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
|
88 |
+
else:
|
89 |
+
if os.path.exists(model_dir):
|
90 |
+
print("Model repository already exists. Using existing repository.")
|
91 |
|
92 |
+
convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
|
93 |
|
94 |
+
else:
|
95 |
+
revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
|
|
|
96 |
|
97 |
+
print("Downloading model repository...")
|
98 |
+
snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
|
99 |
+
print("Model repository downloaded successfully.")
|
100 |
|
101 |
+
convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
|
102 |
|
103 |
def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
|
104 |
convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
|
|
|
116 |
print(f"Original model directory '{model_dir}' deleted.")
|
117 |
else:
|
118 |
print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
|
119 |
+
|
120 |
+
|
121 |
+
create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
|
122 |
|
123 |
+
def create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name):
|
124 |
imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
|
125 |
imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
|
126 |
imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
|
|
|
166 |
print("Finished preparing resources.")
|
167 |
|
168 |
if __name__ == "__main__":
|
169 |
+
main()
|
gguf-imat.py
CHANGED
@@ -70,6 +70,7 @@ def download_cudart_if_necessary(latest_release_tag):
|
|
70 |
def download_model_repo():
|
71 |
base_dir = os.path.dirname(os.path.abspath(__file__))
|
72 |
models_dir = os.path.join(base_dir, "models")
|
|
|
73 |
if not os.path.exists(models_dir):
|
74 |
os.makedirs(models_dir)
|
75 |
|
@@ -77,27 +78,27 @@ def download_model_repo():
|
|
77 |
model_name = model_id.split("/")[-1]
|
78 |
model_dir = os.path.join(models_dir, model_name)
|
79 |
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
|
86 |
-
|
87 |
-
convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
|
88 |
|
89 |
-
|
90 |
-
|
|
|
|
|
|
|
91 |
|
92 |
-
|
93 |
|
94 |
-
|
95 |
-
|
96 |
-
print("Model repository downloaded successfully.")
|
97 |
|
98 |
-
|
|
|
|
|
99 |
|
100 |
-
|
101 |
|
102 |
def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
|
103 |
convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
|
@@ -115,13 +116,17 @@ def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir,
|
|
115 |
print(f"Original model directory '{model_dir}' deleted.")
|
116 |
else:
|
117 |
print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
|
|
|
|
|
|
|
118 |
|
|
|
119 |
imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
|
120 |
imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
|
121 |
imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
|
122 |
if not os.path.exists(imatrix_output_dst):
|
123 |
try:
|
124 |
-
subprocess.run([imatrix_exe, "-m", gguf_model_path, "-f", os.path.join(base_dir, "imatrix", imatrix_file_name), "-ngl", "
|
125 |
shutil.move(imatrix_output_src, imatrix_output_dst)
|
126 |
print("imatrix.dat moved successfully.")
|
127 |
except Exception as e:
|
|
|
70 |
def download_model_repo():
|
71 |
base_dir = os.path.dirname(os.path.abspath(__file__))
|
72 |
models_dir = os.path.join(base_dir, "models")
|
73 |
+
|
74 |
if not os.path.exists(models_dir):
|
75 |
os.makedirs(models_dir)
|
76 |
|
|
|
78 |
model_name = model_id.split("/")[-1]
|
79 |
model_dir = os.path.join(models_dir, model_name)
|
80 |
|
81 |
+
gguf_dir = os.path.join(base_dir, "models", f"{model_name}-GGUF")
|
82 |
+
gguf_model_path = os.path.join(gguf_dir, f"{model_name}-F16.gguf")
|
83 |
+
imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
|
84 |
+
delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
|
|
|
|
|
|
|
|
|
85 |
|
86 |
+
if os.path.exists(gguf_model_path):
|
87 |
+
create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
|
88 |
+
else:
|
89 |
+
if os.path.exists(model_dir):
|
90 |
+
print("Model repository already exists. Using existing repository.")
|
91 |
|
92 |
+
convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
|
93 |
|
94 |
+
else:
|
95 |
+
revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
|
|
|
96 |
|
97 |
+
print("Downloading model repository...")
|
98 |
+
snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
|
99 |
+
print("Model repository downloaded successfully.")
|
100 |
|
101 |
+
convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
|
102 |
|
103 |
def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
|
104 |
convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
|
|
|
116 |
print(f"Original model directory '{model_dir}' deleted.")
|
117 |
else:
|
118 |
print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
|
119 |
+
|
120 |
+
|
121 |
+
create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
|
122 |
|
123 |
+
def create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name):
|
124 |
imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
|
125 |
imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
|
126 |
imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
|
127 |
if not os.path.exists(imatrix_output_dst):
|
128 |
try:
|
129 |
+
subprocess.run([imatrix_exe, "-m", gguf_model_path, "-f", os.path.join(base_dir, "imatrix", imatrix_file_name), "-ngl", "7"], cwd=gguf_dir)
|
130 |
shutil.move(imatrix_output_src, imatrix_output_dst)
|
131 |
print("imatrix.dat moved successfully.")
|
132 |
except Exception as e:
|