models/cpp_qwen2.py

"""
## convert to gguf

python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/

## predict
./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -p "I believe the meaning of life is" -n 128
./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -f prompt.txt -n 128
./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -p "You are a helpful assistant" -cnv


## timing


**重庆GPU服务器，cache为空 **
llama_print_timings:        load time =    1711.48 ms
llama_print_timings:      sample time =      73.89 ms /    41 runs   (    1.80 ms per token,   554.84 tokens per second)
llama_print_timings: prompt eval time =    2621.25 ms /     5 tokens (  524.25 ms per token,     1.91 tokens per second)   # 0.2-0.5秒/token
llama_print_timings:        eval time =    1430.91 ms /    40 runs   (   35.77 ms per token,    27.95 tokens per second)
llama_print_timings:       total time =    4848.09 ms /    45 tokens

llama_print_timings:        load time =    1939.72 ms
llama_print_timings:      sample time =     286.69 ms /   170 runs   (    1.69 ms per token,   592.99 tokens per second)
llama_print_timings: prompt eval time =       0.00 ms /     0 tokens (    -nan ms per token,     -nan tokens per second)  # warmup后，加速明显。
llama_print_timings:        eval time =    5737.50 ms /   170 runs   (   33.75 ms per token,    29.63 tokens per second)
llama_print_timings:       total time =    8219.82 ms /   170 tokens


**hf-space，cache为空 （关闭GGML_BLAS） ** -----------
llama_print_timings:        load time =   28230.06 ms
llama_print_timings:      sample time =     147.58 ms /     8 runs   (   18.45 ms per token,    54.21 tokens per second)   # 18ms/token
llama_print_timings: prompt eval time =   28864.82 ms /     5 tokens ( 5772.96 ms per token,     0.17 tokens per second)   # 5.7s/token
llama_print_timings:        eval time =    1557.94 ms /     7 runs   (  222.56 ms per token,     4.49 tokens per second)
llama_print_timings:       total time =   30753.48 ms /    12 tokens


**hf-space，cache为空 （开启GGML_BLAS）** -----------
llama_print_timings:        load time =   27347.29 ms
llama_print_timings:      sample time =      82.53 ms /    26 runs   (    3.17 ms per token,   315.05 tokens per second)   # 3ms/token
llama_print_timings: prompt eval time =   28855.64 ms /     9 tokens ( 3206.18 ms per token,     0.31 tokens per second)   # 3s/token
llama_print_timings:        eval time =    9810.01 ms /    25 runs   (  392.40 ms per token,     2.55 tokens per second)
llama_print_timings:       total time =   39073.77 ms /    34 tokens

llama_print_timings:        load time =   27347.29 ms
llama_print_timings:      sample time =     272.12 ms /    96 runs   (    2.83 ms per token,   352.79 tokens per second)   # 2.8ms/token
llama_print_timings: prompt eval time =       0.00 ms /     0 tokens (    -nan ms per token,     -nan tokens per second)
llama_print_timings:        eval time =   19974.85 ms /    96 runs   (  208.07 ms per token,     4.81 tokens per second)
llama_print_timings:       total time =   22517.08 ms /    96 tokens


## TODO:

- 解决warmup慢的问题
- 支持cache，并提前对所有预设system进行cache。

## reference

- https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/local.py
- https://github.com/awinml/llama-cpp-python-bindings
- https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/llms/llamacpp.py
- https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/server.py
- https://github.com/abetlen/llama-cpp-python/blob/main/llama_cpp/server/model.py
- https://github.com/abetlen/llama-cpp-python/blob/main/llama_cpp/server/app.py
"""

import json
import copy
import os

from models.base_model import Simulator
import llama_cpp
from transformers import AutoTokenizer
from utils.logging_util import logger
import config


class Qwen2Simulator(Simulator):

    def __init__(self):
        local_path = "/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct-GGUF/qwen2-0_5b-instruct-fp16.gguf"
        if os.path.exists(local_path):
            self.hf_tokenizer = AutoTokenizer.from_pretrained(
                "/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/")
            self.llm = llama_cpp.Llama(  # n_ctx, n_threads
                model_path=local_path,
                # 默认的tokenizer有bug，tokenize后的id不同
                tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
                n_ctx=config.MAX_SEQUENCE_LENGTH,  #
                # n_threads=None, # 默认会根据cpu数来设置 n_threads
                # use_mlock=True,
                verbose=True,
            )
        else:
            self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
            self.llm = llama_cpp.Llama.from_pretrained(
                repo_id="Qwen/Qwen2-0.5B-Instruct-GGUF",
                tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
                filename="*fp16.gguf",
                n_ctx=config.MAX_SEQUENCE_LENGTH,
                # use_mlock=True,
                verbose=True,
            )
        logger.info(f"llm has been initialized: {self.llm}, "
                    f"n_threads={self.llm.n_threads}, n_ctx={self.llm.n_ctx}, "
                    f"env[CACHE]={os.environ.get('CACHE', None)}")

        self.stop_words = [
            "<|im_end|>",
            "<|im_start|>",
            "<|endoftext|>",
        ]
        self.stop_tokens = self.tokenize("".join(self.stop_words))
        self.generation_kwargs = dict(
            temperature=config.DEFAULT_TEMPERATURE,
            top_p=config.DEFAULT_TOP_P,
            top_k=config.DEFAULT_TOP_K,
            max_tokens=config.DEFAULT_MAX_NEW_TOKENS,
            repeat_penalty=1.1,
            # qwen2-0.5b-chat 有时内容生成结束没有<|im_end|>，直接跟 <|im_start|>
            stop=self.stop_words,
        )

        self.user_start_tokens = self.tokenize("<|im_start|>user\n")
        self.assistant_start_tokens = self.tokenize("<|im_start|>assistant\n")
        # self.llm.generate  .set_cache   .last_n_tokens_size  .reset  .ctx ._ctx

        # cache = llama_cpp.LlamaDiskCache(capacity_bytes=cache_size)
        cache = llama_cpp.LlamaRAMCache(capacity_bytes=2 << 30) # 2G
        self.llm.set_cache(cache)

    def tokenize(self, text):
        return self.llm.tokenize(text.encode("utf-8"))

    def detokenize(self, tokens):
        return self.llm.detokenize(tokens).decode("utf-8")

    def strip_stoptokens(self, tokens):
        while tokens and tokens[0] in self.stop_tokens:
            logger.info(f"head-striping {tokens[0]} {self.detokenize([tokens[0]])}")
            tokens.pop(0)
        while tokens and tokens[-1] in self.stop_tokens:
            logger.info(f"tail-striping {tokens[-1]} {self.detokenize([tokens[-1]])}")
            tokens.pop()
        return tokens

    def generate(self, history, stream=True):
        """
        额外前向：remains 5 to forward "<|im_end|>\n<|im_start|>assistant\n"

        :param history:
        :param stream:
        :return:
        """
        if history[-1]['role'] in ["user"]:
            start_tokens = self.assistant_start_tokens
            suffix_tokens = self.user_start_tokens
        elif history[-1]['role'] in ["assistant", "system"]:
            start_tokens = self.user_start_tokens
            suffix_tokens = self.assistant_start_tokens

        input_ids = []
        for message in history:
            if "tokens" not in message:  # tokens
                message["tokens"] = self.tokenize(message["content"])
            input_ids += self.tokenize(f"<|im_start|>{message['role']}\n") \
                         + message["tokens"] \
                         + self.tokenize("<|im_end|>\n")
        input_ids += start_tokens
        if stream:
            return self._stream_generate(input_ids, suffix_tokens)
        else:
            return self._generate(input_ids)

    def _stream_generate(self, input_ids, suffix_tokens=None):
        logger.info(f"generation_kwargs {self.generation_kwargs}")
        output = self.llm.create_completion(
            input_ids,
            stream=True,
            **self.generation_kwargs
        )
        # TODO: 检测finish reason，如果是length，则shift，并继续生成。
        # TODO: 返回 token_id,
        for out in output:
            stream = copy.deepcopy(out)
            if stream["choices"][0]["finish_reason"] is None:
                yield stream["choices"][0]["completion_text"], stream["choices"][0]["completion_tokens"]
            else:
                logger.info(
                    f'finish_reason {stream["choices"][0]["finish_reason"]} with text: {stream["choices"][0]["text"]}')

        #
        self.post_cache(suffix_tokens)

    def pre_cache_system(self, system_list):
        """ warmup for system prompt
        :param system_list:
        :return:
        """
        logger.info(f"cache size {self.llm.cache.cache_size}")
        for system_prompt in system_list:
            logger.info(f"pre caching '{system_prompt}'")
            input_ids = self.tokenize(f"<|im_start|>system\n{system_prompt}<|im_end|>\n<|im_start|>user\n")
            output = self.llm.create_completion(
                input_ids,
                stream=False,
                max_tokens=1,
                top_k=1
            )
            logger.info(f"cache size {self.llm.cache.cache_size}")

        # disable cache after
        llama_cpp.LlamaRAMCache.__setitem__ = lambda *args: None
        llama_cpp.Llama.save_state = lambda *args: None

    def post_cache(self, suffix_tokens):
        """ warmup for next turn generation
        :param suffix_tokens:
        :return:
        """
        if suffix_tokens:
            logger.info(f"before warmup: n_tokens = {self.llm.n_tokens}")
            self.llm.eval([151645, 198] + suffix_tokens)  # <|im_end|>\n
            logger.info(f"after warmup: n_tokens = {self.llm.n_tokens}")


bot = Qwen2Simulator()

if __name__ == "__main__":

    messages = [{"role": "system", "content": "你是一个导游。"}]
    generated_tokens = None
    print("######## requesting", messages)
    for generated_text, generated_tokens in bot.generate(messages, stream=True):
        print(generated_text, generated_tokens)

    for i in range(3):
        generated_tokens = bot.strip_stoptokens(generated_tokens)
        messages.append(
            {"role": "user" if i % 2 == 0 else "assistant", "content": generated_text, "tokens": generated_tokens})
        print("######## requesting", messages)
        for generated_text, generated_tokens in bot.generate(messages, stream=True):
            pass
            # print(generated_text, all_tokens)