Songwufeng/Qwen3_Chat_CatGirl_8B

Model Overview

Qwen3_Chat_CatGirl_8B This model is a specialized version of the Qwen3 base model, fine-tuned for the Chinese context and enhanced with cat-girl elements. It retains Qwen3's inherent capabilities in Chinese understanding, command adherence, and coherent text generation. Furthermore, it incorporates the defining traits of the popular "cat-girl" subculture, such as a soft and cute tone, feline-inspired onomatopoeia (like "meow~" and "miw"), playful and gentle expressions, and a "cat-like interaction logic" tailored for the Chinese context (like coquettish responses and affectionate nicknames). This model is ideal for casual conversations, role-playing, and creative content generation in Chinese settings.

安装依赖项

• 安装相关库

pip install --upgrade torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121

pip install --upgrade transformers datasets accelerate safetensors

pip install --upgrade bitsandbytes

pip install --upgrade sentencepiece

pip install --upgrade sentence-transformers

推理代码

from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig, snapshot_download
import torch
import threading

model_dir = snapshot_download('Songwufeng/Qwen3-YueYue-Neko-8B-Dataset')
MAX_NEW_TOKENS = 1024
USE_4BIT_QUANT = True

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

tokenizer = AutoTokenizer.from_pretrained(
    model_dir,
    trust_remote_code=True,
    padding_side="right",
    eos_token="<|endoftext|>",
    pad_token="<|endoftext|>",
    use_fast=True
)

bnb_config = BitsAndBytesConfig(
    load_in_4bit=USE_4BIT_QUANT,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
    llm_int8_skip_modules=["lm_head"]
)

model = AutoModelForCausalLM.from_pretrained(
    model_dir,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True,
    low_cpu_mem_usage=True,
    torch_dtype=torch.float16
)

def stream_inference(messages):
    prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
    inputs = tokenizer([prompt], return_tensors="pt", truncation=True, max_length=2048).to(model.device)
    streamer = TextIteratorStreamer(tokenizer, skip_special_tokens=True, timeout=10.0)

    generate_kwargs = {
        "input_ids": inputs["input_ids"],
        "attention_mask": inputs["attention_mask"],
        "streamer": streamer,
        "max_new_tokens": MAX_NEW_TOKENS,
        "temperature": 0.8,
        "top_p": 0.95,
        "repetition_penalty": 1.2,
        "no_repeat_ngram_size": 3,
        "do_sample": True,
        "pad_token_id": tokenizer.pad_token_id,
        "eos_token_id": tokenizer.eos_token_id
    }

    print(f"\n[推理结果] 悦悦AI：", end="")
    thread = threading.Thread(target=model.generate, kwargs=generate_kwargs)
    thread.start()

    for new_text in streamer:
        print(new_text, end="", flush=True)

    thread.join()
    print(f"\n\n[推理结束] 生成完成（实际长度未超过{MAX_NEW_TOKENS} token）")

if __name__ == "__main__":
    chat_messages = [
        {"role": "system", "content": "你是悦悦AI助手，回答专业且易懂，语气友好，按问题需求详细讲解。"},
        {"role": "user", "content": "能给我讲讲拉格朗日定理吗？"}
    ]
    stream_inference(chat_messages)