Spaces:
Sleeping
Sleeping
| from typing import Iterator | |
| model_id = 'xuqinyang/baichuan-13b-chat-ggml-int4' | |
| from huggingface_hub import snapshot_download,hf_hub_download | |
| #旧 | |
| #snapshot_download(model_id, local_dir="./",revision="7f71a8abefa7b2eede3f74ce0564abe5fbe6874a") | |
| snapshot_download(model_id, local_dir="./",revision="b2414a0ceee68fe09c99ace44446cfc9a1c52b08") | |
| hf_hub_download(repo_id="baichuan-inc/Baichuan-13B-Chat",local_dir="./", filename="tokenizer.model") | |
| from llama_cpp import Llama | |
| llm = Llama(model_path="./ggml-model-q4_0.bin", n_ctx=4096,seed=-1) | |
| def run(message: str, | |
| chat_history: list[tuple[str, str]], | |
| system_prompt: str, | |
| max_new_tokens: int = 1024, | |
| temperature: float = 0.3, | |
| top_p: float = 0.85, | |
| top_k: int = 5) -> Iterator[str]: | |
| history = [] | |
| print(chat_history) | |
| result="" | |
| for i in chat_history: | |
| history.append({"role": "user", "content": i[0]}) | |
| history.append({"role": "assistant", "content": i[1]}) | |
| print(history) | |
| history.append({"role": "user", "content": message}) | |
| for response in llm.create_chat_completion(history,stop=["</s>"],stream=True,max_tokens=-1,temperature=temperature,top_k=top_k,top_p=top_p,repeat_penalty=1.1): | |
| if "content" in response["choices"][0]["delta"]: | |
| result = result + response["choices"][0]["delta"]["content"] | |
| yield result |