app.py

from auto_round import AutoRoundConfig ##must import for autoround format
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

quantized_model_dir = "OPEA/DeepSeek-V3-int4-sym-gptq-inc"
quantization_config = AutoRoundConfig(
    backend="cpu"
)
model = AutoModelForCausalLM.from_pretrained(
    quantized_model_dir,
    torch_dtype=torch.bfloat16,
    trust_remote_code=True,
    device_map="cpu",
    revision="8fe0735",##use autoround format, the only difference is config.json
    quantization_config = quantization_config, ##cpu only machine don't need to set this value
    
)

tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir,trust_remote_code=True)
prompt = "There is a girl who likes adventure,"
messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": prompt}
]
text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True
)
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)

generated_ids = model.generate(
    model_inputs.input_ids,
    max_new_tokens=200,  ##change this to align with the official usage
    do_sample=False  ##change this to align with the official usage
)
generated_ids = [
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
]

response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(response)

prompt = "9.11和9.8哪个数字大"  

##INT4
"""要比较 **9.11** 和 **9.8** 的大小，可以按照以下步骤进行：

1. **比较整数部分**：
   - 两个数的整数部分都是 **9**，所以整数部分相同。

2. **比较小数部分**：
   - **9.11** 的小数部分是 **0.11**
   - **9.8** 的小数部分是 **0.8**（即 **0.80**）

3. **分析小数部分**：
   - **0.80** 大于 **0.11**

因此，**9.8** 大于 **9.11**。

最终答案：\boxed{9.8}

"""

prompt = "strawberry中有几个r?"
##INT4
"""
### 第一步：理解问题

首先，我需要明确问题的含义。问题是：“strawberry中有几个r？”。这里的“strawberry”是一个英文单词，意思是“草莓”。问题问的是这个单 词中有多少个字母“r”。

### 第二步：分解单词

为了找出“strawberry”中有多少个“r”，我需要将这个单词分解成单个字母。让我们逐个字母来看：

- s
- t
- r
- a
- w
- b
- e
- r
- r
- y

### 第三步：识别字母“r”

现在，我需要找出这些字母中哪些是“r”。让我们逐一检查：

1. s - 不是r
2. t - 不是r
3. r - 是r
4. a - 不是r
5. w - 不是r
6. b - 不是r
7. e - 不是r
8. r - 是r
"""

prompt = "How many r in strawberry."
##INT4 
"""The word "strawberry" contains **3 "r"s.
"""

prompt = "There is a girl who likes adventure,"
##INT4:
"""That's wonderful! A girl who loves adventure is likely curious, brave, and eager to explore the world around her. Here are some ideas to fuel her adventurous spirit:

### **Outdoor Adventures**

- **Hiking:** Explore local trails, national parks, or mountains.
- **Camping:** Spend a night under the stars and connect with nature.
- **Rock Climbing:** Challenge herself with bouldering or climbing walls.
- **Kayaking/Canoeing:** Paddle through rivers, lakes, or even the ocean.
- **Zip-lining:** Soar through the treetops for an adrenaline rush.

### **Travel Adventures**

- **Road Trips:** Plan a journey to new cities or scenic destinations.
- **Backpacking:** Travel light and explore different cultures and landscapes.
- **Volunteer Abroad:** Combine adventure with helping others in a new country.

### **Creative Adventures**

- **Photography:** Capture the beauty
"""

prompt = "Please give a brief introduction of DeepSeek company."
##INT4:
"""DeepSeek Artificial Intelligence Co., Ltd. (referred to as "DeepSeek" or "深度求索") , founded in 2023, is a Chinese company dedicated to making AGI a reality"""