I am trying to run the Llama model locally:

import os
import time
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
import gradio as gr
from threading import Thread

# Assuming you have the model downloaded locally in a directory named "local_llama_model"
MODEL_PATH = r"C:\Users\shermanhonsf\Desktop\Python\llama_develop\model\Llama3.2-1B-Instruct-HF-Model"

TITLE = """
# Llama 3.2 1B
"""

PLACEHOLDER = """
Hi, I'm Llama. Ask me anything.
"""

CSS = """
.duplicate-button {
    margin: auto !important;
    color: white !important;
    background: black !important;
    border-radius: 100vh !important;
}
h3 {
    text-align: center;
}
"""

# Set the device to "cuda" if you have a compatible GPU, otherwise "cpu"
device = "cuda" if torch.cuda.is_available() else "cpu"

# Configure quantization if needed (optional)
quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4"
)

# Load the tokenizer and model from the local directory
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_PATH,
    torch_dtype=torch.bfloat16,
    device_map="auto"
    #quantization_config=quantization_config
)

# Move the model to the appropriate device
model.to(device)

def stream_chat(
    message: str,
    history: list,
    system_prompt: str,
    temperature: float = 0.8,
    max_new_tokens: int = 1024,
    top_p: float = 1.0,
    top_k: int = 20,
    penalty: float = 1.2,
):
    print(f'Message: {message}')
    print(f'History: {history}')

    conversation = [
        {"role": "system", "content": system_prompt}
    ]
    for prompt, answer in history:
        conversation.extend([
            {"role": "user", "content": prompt},
            {"role": "assistant", "content": answer},
        ])

    conversation.append({"role": "user", "content": message})

    # Prepare input IDs
    input_ids = tokenizer.apply_chat_template(
        conversation,
        #tokenize=True,
        add_generation_prompt=True,
        return_tensors="pt"
    ).to(device)

    streamer = TextIteratorStreamer(
        tokenizer,
        timeout=60.0,
        skip_prompt=True,
        skip_special_tokens=True
    )

    generate_kwargs = dict(
        input_ids=input_ids,
        max_new_tokens=max_new_tokens,
        do_sample=False if temperature == 0 else True,
        top_p=top_p,
        top_k=top_k,
        temperature=temperature,
        eos_token_id=[128001, 128008, 128009],
        streamer=streamer,
    )

    # Generate the response in a background thread
    with torch.no_grad():
        thread = Thread(target=model.generate, kwargs=generate_kwargs)
        thread.start()

    # Stream the response back to the user
    buffer = ""
    for new_text in streamer:
        buffer += new_text
        yield buffer

# Create the Gradio interface
chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)

with gr.Blocks(css=CSS) as demo:
    gr.HTML(TITLE)
    gr.ChatInterface(
        fn=stream_chat,
        chatbot=chatbot,
        fill_height=True,
        additional_inputs_accordion=gr.Accordion(
            label="⚙️ Parameters",
            open=False,
            render=False
        ),
        additional_inputs=[
            gr.Textbox(
                value="You are a helpful assistant",
                label="System Prompt",
                render=False,
            ),
            gr.Slider(
                minimum=0,
                maximum=1,
                step=0.1,
                value=0.8,
                label="Temperature",
                render=False,
            ),
            gr.Slider(
                minimum=128,
                maximum=8192,
                step=1,
                value=1024,
                label="Max new tokens",
                render=False,
            ),
            gr.Slider(
                minimum=0.0,
                maximum=1.0,
                step=0.1,
                value=1.0,
                label="Top_p",
                render=False,
            ),
            gr.Slider(
                minimum=1,
                maximum=20,
                step=1,
                value=20,
                label="Top_k",
                render=False,
            ),
            gr.Slider(
                minimum=0.0,
                maximum=2.0,
                step=0.1,
                value=1.2,
                label="Repetition penalty",
                render=False,
            ),
        ],
        examples=[
            ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."],
            ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."],
            ["Tell me a random fun fact about the Roman Empire."],
            ["Show me a code snippet of a website's sticky header in CSS and JavaScript."],
        ],
        cache_examples=False,
    )

if __name__ == "__main__":
    demo.launch(debug=True)

But output:
To create a public link, set `share=True` in `launch()`.
Message: hi
History: []
Traceback (most recent call last):
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\queueing.py", line 624, in process_events
    response = await route_utils.call_process_api(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\route_utils.py", line 323, in call_process_api
    output = await app.get_blocks().process_api(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\blocks.py", line 2015, in process_api
    result = await self.call_function(
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\blocks.py", line 1574, in call_function
    prediction = await utils.async_iteration(iterator)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration
    return await anext(iterator)
           ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\utils.py", line 815, in asyncgen_wrapper
    response = await iterator.__anext__()
               ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\chat_interface.py", line 678, in _stream_fn
    first_response = await async_iteration(generator)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration
    return await anext(iterator)
           ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\utils.py", line 704, in __anext__
    return await anyio.to_thread.run_sync(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\anyio\to_thread.py", line 56, in run_sync
    return await get_async_backend().run_sync_in_worker_thread(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\anyio\_backends\_asyncio.py", line 2441, in run_sync_in_worker_thread
    return await future
           ^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\anyio\_backends\_asyncio.py", line 943, in run
    result = context.run(func, *args)
             ^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\gradio\utils.py", line 687, in run_sync_iterator_async
    return next(iterator)
           ^^^^^^^^^^^^^^
  File "c:\Users\shermanhonsf\Desktop\Python\llama_develop\chat_gr.py", line 79, in stream_chat
    input_ids = tokenizer.apply_chat_template(
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\transformers\tokenization_utils_base.py", line 1803, in apply_chat_template
    chat_template = self.get_chat_template(chat_template, tools)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\shermanhonsf\Desktop\Python\.conda\Lib\site-packages\transformers\tokenization_utils_base.py", line 1967, in get_chat_template
    raise ValueError(
ValueError: Cannot use chat template functions because tokenizer.chat_template is not set and no template argument was passed! For information about writing templates and setting the tokenizer.chat_template attribute, please see the documentation at https://huggingface.co/docs/transformers/main/en/chat_templating

I saw the prompt format from meta website:

Prompt Template
Special Tokens
Tokens

Description



Specifies the start of the prompt.



Model will cease to generate more tokens. This token is generated only by the base models.



This token is used for padding text sequences to the same length in a batch.





These tokens enclose the role for a particular message. The possible roles are: [system, user, assistant, and ipython]


End of message. A message represents a possible stopping point for execution where the model can inform the executor that a tool call needs to be made. This is used for multi-step interactions between the model and any available tools. This token is emitted by the model when the Environment: ipython instruction is used in the system prompt, or if the model calls for a built-in tool.


End of turn. Represents when the model has determined that it has finished interacting with the user message that initiated its response. This is used in two scenarios:

at the end of a direct interaction between the model and the user
at the end of multiple interactions between the model and any available tools
This token signals to the executor that the model has finished generating a response.



Special tag used in the model’s response to signify a tool call.

Supported Roles
There are 4 different roles that are supported by Llama text models:

system: Sets the context in which to interact with the AI model. It typically includes rules, guidelines, or necessary information that help the model respond effectively.
user: Represents the human interacting with the model. It includes the inputs, commands, and questions to the model.
ipython: A new role introduced in Llama 3.1. Semantically, this role means "tool". This role is used to mark messages with the output of a tool call when sent back to the model from the executor.
assistant: Represents the response generated by the AI model based on the context provided in the system, ipython and user prompts.
[system, assistant, user, ipython]

User and assistant conversation
Here is a regular multi-turn user assistant conversation and how its formatted.

Input Prompt Format


system


You are a helpful assistant

user


Who are you?

assistant



Model Response Format
I'm an AI assistant, which means I'm a computer program designed to simulate conversation and answer questions to the best of my ability. I'm here to help you with any questions or tasks you may have, and I'll do my best to provide you with accurate and helpful information.

I don't have a personal name, but I'm often referred to as a "virtual assistant" or a "chatbot." I'm a machine learning model, which means I've been trained on a large dataset of text and can generate responses based on patterns and context.

I can help with a wide range of topics, from general knowledge and trivia to more specialized subjects like science, history, and technology. I can also assist with tasks like language translation, text summarization, and even generating creative content like stories or poetry.

So, what can I help you with today?


How to fix it? Show full code if applicable