Commit
•
050948f
1
Parent(s):
0af877a
conversion with imatrix
Browse files
conv.sh
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
|
| 3 |
# convert PyTorch model
|
| 4 |
# LLaMA 3
|
| 5 |
-
convert --vocab-type bpe --outtype f16 --outfile conv.bin .
|
| 6 |
|
| 7 |
# LLaMA 2
|
| 8 |
# convert --outtype f16 --outfile conv.bin .
|
|
@@ -10,8 +10,12 @@ convert --vocab-type bpe --outtype f16 --outfile conv.bin .
|
|
| 10 |
# define TheBloke quants and create them
|
| 11 |
declare -a quants=(Q2_K Q3_K_S Q3_K_M Q3_K_L Q4_0 Q4_K_S Q4_K_M Q5_0 Q5_K_S Q5_K_M Q6_K Q8_0)
|
| 12 |
currpath=${PWD##*/}
|
|
|
|
| 13 |
|
| 14 |
for quant in "${quants[@]}";
|
| 15 |
do
|
| 16 |
-
|
| 17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
# convert PyTorch model
|
| 4 |
# LLaMA 3
|
| 5 |
+
# convert --vocab-type bpe --outtype f16 --outfile conv.bin .
|
| 6 |
|
| 7 |
# LLaMA 2
|
| 8 |
# convert --outtype f16 --outfile conv.bin .
|
|
|
|
| 10 |
# define TheBloke quants and create them
|
| 11 |
declare -a quants=(Q2_K Q3_K_S Q3_K_M Q3_K_L Q4_0 Q4_K_S Q4_K_M Q5_0 Q5_K_S Q5_K_M Q6_K Q8_0)
|
| 12 |
currpath=${PWD##*/}
|
| 13 |
+
basefile=$(ls -Art *.F16.gguf)
|
| 14 |
|
| 15 |
for quant in "${quants[@]}";
|
| 16 |
do
|
| 17 |
+
# basic GGUF
|
| 18 |
+
quantize $basefile ${currpath::-5}.$quant.gguf $quant
|
| 19 |
+
# imatrix GGUF
|
| 20 |
+
# quantize --imatrix model.imatrix $basefile ${currpath::-5}.$quant.gguf $quant
|
| 21 |
+
done
|