Abhinav Kulkarni commited on
Commit
2dae9a9
1 Parent(s): 01ab93b

Updated README

Browse files
Files changed (1) hide show
  1. README.md +5 -5
README.md CHANGED
@@ -34,9 +34,9 @@ For Docker users, the `nvcr.io/nvidia/pytorch:23.06-py3` image is runtime v12.1
34
  ## How to Use
35
 
36
  ```bash
37
- git clone https://github.com/mit-han-lab/llm-awq \
38
  && cd llm-awq \
39
- && git checkout 71d8e68df78de6c0c817b029a568c064bf22132d \
40
  && pip install -e . \
41
  && cd awq/kernels \
42
  && python setup.py install
@@ -47,9 +47,9 @@ import torch
47
  from awq.quantize.quantizer import real_quantize_model_weight
48
  from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
49
  from accelerate import init_empty_weights, load_checkpoint_and_dispatch
50
- from huggingface_hub import hf_hub_download
51
 
52
- model_name = "psmathur/orca_mini_v2_7b"
53
 
54
  # Config
55
  config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
@@ -64,7 +64,7 @@ q_config = {
64
  "q_group_size": 128,
65
  }
66
 
67
- load_quant = hf_hub_download('abhinavkulkarni/psmathur-orca_mini_v2_7b-w4-g128-awq', 'pytorch_model.bin')
68
 
69
  with init_empty_weights():
70
  model = AutoModelForCausalLM.from_config(config=config,
 
34
  ## How to Use
35
 
36
  ```bash
37
+ git clone https://github.com/abhinavkulkarni/llm-awq \
38
  && cd llm-awq \
39
+ && git checkout e977c5a570c5048b67a45b1eb823b81de02d0d60 \
40
  && pip install -e . \
41
  && cd awq/kernels \
42
  && python setup.py install
 
47
  from awq.quantize.quantizer import real_quantize_model_weight
48
  from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
49
  from accelerate import init_empty_weights, load_checkpoint_and_dispatch
50
+ from huggingface_hub import snapshot_download
51
 
52
+ model_name = "abhinavkulkarni/psmathur-orca_mini_v2_7b-w4-g128-awq"
53
 
54
  # Config
55
  config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
 
64
  "q_group_size": 128,
65
  }
66
 
67
+ load_quant = snapshot_download(model_name)
68
 
69
  with init_empty_weights():
70
  model = AutoModelForCausalLM.from_config(config=config,