Improved quantization using Openvino 2024.5.0rc1

README.md

@@ -7,22 +7,20 @@ tags:
 
 This is an INT4 quantized version of the `Phi-3-mini-128k-instruct` model. The Python packages used in creating this model are as follows:
 ```
-openvino==2024.4.0
+openvino==2024.5.0rc1
 optimum==1.23.3
 optimum-intel==1.20.1
 nncf==2.13.0
 torch==2.5.1
-transformers==4.46.1
+transformers==4.46.2
 ```
 This quantized model is created using the following command:
 ```
-optimum-cli export openvino -m "microsoft/Phi-3-mini-128k-instruct" --task text-generation-with-past --weight-format int4 --group-size 128 --ratio 0.8 --trust-remote-code ./Phi-3-mini-128k-instruct-ov-int4
+optimum-cli export openvino --model "microsoft/Phi-3-mini-128k-instruct" --weight-format int4 --group-size 128 --sym --ratio 1 --all-layers ./Phi-3-mini-128k-instruct-ov-int4
 ```
 For more details, run the following command from your Python environment: `optimum-cli export openvino --help`
 
 INFO:nncf:Statistics of the bitwidth distribution:
 |   Num bits (N) | % all parameters (layers)   | % ratio-defining parameters (layers)   |
 |----------------|-----------------------------|----------------------------------------|
-|              8 | 24% (23 / 130)              | 20% (21 / 128)                         |
-|              4 | 76% (107 / 130)             | 80% (107 / 128)                        |
-
+|              4 | 100% (130 / 130)            | 100% (130 / 130)                       |

config.json

@@ -133,7 +133,7 @@
   "sliding_window": 262144,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.46.1",
+  "transformers_version": "4.46.2",
   "use_cache": true,
   "vocab_size": 32064
 }

generation_config.json

@@ -7,5 +7,5 @@
     32007
   ],
   "pad_token_id": 32000,
-  "transformers_version": "4.46.1"
+  "transformers_version": "4.46.2"
 }

openvino_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ce9a3d5a7f07a4cb34ddae2c2d81c2b6f3cbac5c5cd51e1ba376bcf8983195e
-size 2432074068
+oid sha256:edb9df8e7e86999f0acf6b6990ee50ef4fcbd6c4df577ad602f39ca1d797a06f
+size 1970940388