Imatrix

Dockerfile

@@ -60,4 +60,5 @@ ENV PYTHONPATH=${HOME}/app \
     PATH=/usr/local/nvidia/bin:${PATH}
 
 
-ENTRYPOINT ["/bin/bash", "-c", "cd llama.cpp && LLAMA_CUDA=1 make -j quantize gguf-split imatrix && cd .. && /bin/sh start.sh"]
+ENTRYPOINT /bin/sh start.sh
+

app.py

@@ -132,7 +132,7 @@ def process_model(model_id, q_method, use_imatrix, imatrix_q_method, private_rep
             if train_data_file:
                 train_data_path = train_data_file.name
             else:
-                train_data_path = "imatrix_calibration.txt"
+                train_data_path = "groups_merged.txt" #fallback calibration dataset
 
             print(f"Training data file path: {train_data_path}")
 

imatrix_calibration.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:52062b7643edddbbc83435331ed1bc6ffc3eb463fae9df3551df52fb5638f0e8
-size 201119

start.sh

@@ -1 +1,5 @@
+cd llama.cpp
+make -j quantize gguf-split imatrix
+
+cd ..
 python app.py