Create app.py

app.py

@@ -0,0 +1,131 @@
+from auto_round import AutoRoundConfig ##must import for autoround format
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+quantized_model_dir = "OPEA/DeepSeek-V3-int4-sym-gptq-inc"
+quantization_config = AutoRoundConfig(
+    backend="cpu"
+)
+model = AutoModelForCausalLM.from_pretrained(
+    quantized_model_dir,
+    torch_dtype=torch.bfloat16,
+    trust_remote_code=True,
+    device_map="cpu",
+    revision="8fe0735",##use autoround format, the only difference is config.json
+    quantization_config = quantization_config, ##cpu only machine don't need to set this value
+    
+)
+
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir,trust_remote_code=True)
+prompt = "There is a girl who likes adventure,"
+messages = [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {"role": "user", "content": prompt}
+]
+text = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True
+)
+model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
+
+generated_ids = model.generate(
+    model_inputs.input_ids,
+    max_new_tokens=200,  ##change this to align with the official usage
+    do_sample=False  ##change this to align with the official usage
+)
+generated_ids = [
+output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+]
+
+response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+print(response)
+
+prompt = "9.11和9.8哪个数字大"  
+
+##INT4
+"""要比较 **9.11** 和 **9.8** 的大小，可以按照以下步骤进行：
+
+1. **比较整数部分**：
+   - 两个数的整数部分都是 **9**，所以整数部分相同。
+
+2. **比较小数部分**：
+   - **9.11** 的小数部分是 **0.11**
+   - **9.8** 的小数部分是 **0.8**（即 **0.80**）
+
+3. **分析小数部分**：
+   - **0.80** 大于 **0.11**
+
+因此，**9.8** 大于 **9.11**。
+
+最终答案：\boxed{9.8}
+
+"""
+
+prompt = "strawberry中有几个r?"
+##INT4
+"""
+### 第一步：理解问题
+
+首先，我需要明确问题的含义。问题是：“strawberry中有几个r？”。这里的“strawberry”是一个英文单词，意思是“草莓”。问题问的是这个单 词中有多少个字母“r”。
+
+### 第二步：分解单词
+
+为了找出“strawberry”中有多少个“r”，我需要将这个单词分解成单个字母。让我们逐个字母来看：
+
+- s
+- t
+- r
+- a
+- w
+- b
+- e
+- r
+- r
+- y
+
+### 第三步：识别字母“r”
+
+现在，我需要找出这些字母中哪些是“r”。让我们逐一检查：
+
+1. s - 不是r
+2. t - 不是r
+3. r - 是r
+4. a - 不是r
+5. w - 不是r
+6. b - 不是r
+7. e - 不是r
+8. r - 是r
+"""
+
+prompt = "How many r in strawberry."
+##INT4 
+"""The word "strawberry" contains **3 "r"s.
+"""
+
+prompt = "There is a girl who likes adventure,"
+##INT4:
+"""That's wonderful! A girl who loves adventure is likely curious, brave, and eager to explore the world around her. Here are some ideas to fuel her adventurous spirit:
+
+### **Outdoor Adventures**
+
+- **Hiking:** Explore local trails, national parks, or mountains.
+- **Camping:** Spend a night under the stars and connect with nature.
+- **Rock Climbing:** Challenge herself with bouldering or climbing walls.
+- **Kayaking/Canoeing:** Paddle through rivers, lakes, or even the ocean.
+- **Zip-lining:** Soar through the treetops for an adrenaline rush.
+
+### **Travel Adventures**
+
+- **Road Trips:** Plan a journey to new cities or scenic destinations.
+- **Backpacking:** Travel light and explore different cultures and landscapes.
+- **Volunteer Abroad:** Combine adventure with helping others in a new country.
+
+### **Creative Adventures**
+
+- **Photography:** Capture the beauty
+"""
+
+prompt = "Please give a brief introduction of DeepSeek company."
+##INT4:
+"""DeepSeek Artificial Intelligence Co., Ltd. (referred to as "DeepSeek" or "深度求索") , founded in 2023, is a Chinese company dedicated to making AGI a reality"""