Abhinav Kulkarni
commited on
Commit
•
a5a104b
1
Parent(s):
2c8a105
Updated README
Browse files
README.md
CHANGED
|
@@ -34,9 +34,9 @@ For Docker users, the `nvcr.io/nvidia/pytorch:23.06-py3` image is runtime v12.1
|
|
| 34 |
## How to Use
|
| 35 |
|
| 36 |
```bash
|
| 37 |
-
git clone https://github.com/
|
| 38 |
&& cd llm-awq \
|
| 39 |
-
&& git checkout
|
| 40 |
&& pip install -e . \
|
| 41 |
&& cd awq/kernels \
|
| 42 |
&& python setup.py install
|
|
@@ -49,7 +49,7 @@ from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
|
|
| 49 |
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
| 50 |
from huggingface_hub import snapshot_download
|
| 51 |
|
| 52 |
-
model_name = "psmathur
|
| 53 |
|
| 54 |
# Config
|
| 55 |
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
|
|
@@ -64,7 +64,7 @@ q_config = {
|
|
| 64 |
"q_group_size": 128,
|
| 65 |
}
|
| 66 |
|
| 67 |
-
load_quant = snapshot_download(
|
| 68 |
|
| 69 |
with init_empty_weights():
|
| 70 |
model = AutoModelForCausalLM.from_config(config=config,
|
|
|
|
| 34 |
## How to Use
|
| 35 |
|
| 36 |
```bash
|
| 37 |
+
git clone https://github.com/abhinavkulkarni/llm-awq \
|
| 38 |
&& cd llm-awq \
|
| 39 |
+
&& git checkout e977c5a570c5048b67a45b1eb823b81de02d0d60 \
|
| 40 |
&& pip install -e . \
|
| 41 |
&& cd awq/kernels \
|
| 42 |
&& python setup.py install
|
|
|
|
| 49 |
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
| 50 |
from huggingface_hub import snapshot_download
|
| 51 |
|
| 52 |
+
model_name = "abhinavkulkarni/psmathur-orca_mini_v2_13b-w4-g128-awq"
|
| 53 |
|
| 54 |
# Config
|
| 55 |
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
|
|
|
|
| 64 |
"q_group_size": 128,
|
| 65 |
}
|
| 66 |
|
| 67 |
+
load_quant = snapshot_download(model_name)
|
| 68 |
|
| 69 |
with init_empty_weights():
|
| 70 |
model = AutoModelForCausalLM.from_config(config=config,
|