My code for local llama model inference:
"
# Gradio, work

import os
import time
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
import gradio as gr
from threading import Thread

MODEL_PATH = r"C:\Users\shermanhonsf\Desktop\Python\llama_develop\model\Llama3.2-1B-Instruct-HF-Model" #Your model path

TITLE = """
# Llama 3.2 1B
"""

PLACEHOLDER = """
Hi, I'm Llama. Ask me anything.
"""

CSS = """
.duplicate-button {
    margin: auto !important;
    color: white !important;
    background: black !important;
    border-radius: 100vh !important;
}
h3 {
    text-align: center;
}
"""

# Set the device to "cuda" if you have a compatible GPU, otherwise "cpu"
device = "cuda" if torch.cuda.is_available() else "cpu"

# Configure quantization if needed (optional)
quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4"
)

# Load the tokenizer and model from the local directory
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_PATH,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    #quantization_config=quantization_config
)

# Move the model to the appropriate device
model.to(device)

def stream_chat(message: str, history: list, system_prompt: str = 'You are a helpful assistant', temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2):
    print(f'message: {message}')
    print(f'history: {history}')
    print(f'system_prompt: {system_prompt}')
    print(f'temperature: {temperature}')
    print(f'max_new_tokens: {max_new_tokens}')
    print(f'top_p: {top_p}')
    print(f'top_k: {top_k}')
    print(f'penalty: {penalty}')
    print(f'Message: {message}')
    print(f'History: {history}')

    conversation = [
        {"role": "system", "content": system_prompt}
    ]
    for prompt, answer in history:
        conversation.extend([
            {"role": "user", "content": prompt},
            {"role": "assistant", "content": answer},
        ])

    conversation.append({"role": "user", "content": message})

    # Prepare input IDs
    input_ids = tokenizer.apply_chat_template(
        conversation,
        #tokenize=True,
        add_generation_prompt=True,
        return_tensors="pt"
    ).to(device)

    streamer = TextIteratorStreamer(
        tokenizer,
        timeout=60.0,
        skip_prompt=True,
        skip_special_tokens=True
    )

    generate_kwargs = dict(
        input_ids=input_ids,
        max_new_tokens=max_new_tokens,
        do_sample=False if temperature == 0 else True,
        top_p=top_p,
        top_k=top_k,
        temperature=temperature,
        eos_token_id=[128001, 128008, 128009],
        streamer=streamer,
    )

    # Generate the response in a background thread
    with torch.no_grad():
        thread = Thread(target=model.generate, kwargs=generate_kwargs)
        thread.start()

    # Stream the response back to the user
    buffer = ""
    for new_text in streamer:
        buffer += new_text
        yield buffer

# Create the Gradio interface
chatbot = gr.Chatbot(height=600, show_copy_button=True, layout="panel", placeholder=PLACEHOLDER)

with gr.Blocks(css=CSS) as demo:
    gr.HTML(TITLE)
    gr.ChatInterface(
        fn=stream_chat,
        chatbot=chatbot,
        fill_height=True,
        additional_inputs_accordion=gr.Accordion(
            label="⚙️ Parameters",
            open=False,
            render=False
        ),
        
        additional_inputs=[
            
            gr.Textbox(
                value="You are a helpful assistant",
                label="System Prompt",
                render=False,
            ),
            gr.Slider(
                minimum=0,
                maximum=1,
                step=0.1,
                value=0.8,
                label="Temperature",
                render=False,
            ),
            gr.Slider(
                minimum=128,
                maximum=8192,
                step=1,
                value=1024,
                label="Max new tokens",
                render=False,
            ),
            gr.Slider(
                minimum=0.0,
                maximum=1.0,
                step=0.1,
                value=1.0,
                label="Top_p",
                render=False,
            ),
            gr.Slider(
                minimum=1,
                maximum=20,
                step=1,
                value=20,
                label="Top_k",
                render=False,
            ),
            gr.Slider(
                minimum=0.0,
                maximum=2.0,
                step=0.1,
                value=1.2,
                label="Repetition penalty",
                render=False,
            )
        ],
        examples=[
            ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."],
            ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."],
            ["Tell me a random fun fact about the Roman Empire."],
            ["Show me a code snippet of a website's sticky header in CSS and JavaScript."],
        ],
        cache_examples=False,
    )

if __name__ == "__main__":
    demo.launch(debug=True)
"
I want to add the chat with pdf function: the function will allow user input pdf, extract all text inside the pdf, pass the pdf input and user input to the llm, then the llm should be able to understand the pdf and give reasonable response. Take the following as reference:
"
PyPDF2 is the most popular library for parsing PDF files in Python.

We use a PdfReader object which allows us to access the PDF's content.

We can extract the metadata of the file:

from PyPDF2 import PdfReader
reader = PdfReader(path)
meta = reader.metadata
print("Total Pages: ", len(reader.pages))
# All of the following could be None!
print("Author: ", meta.author)
print("Creator: ", meta.creator)
print("Producer: ", meta.producer)
print("Subject: ", meta.subject)
print("Title: ", meta.title)
We can extract the content from any page:

page = reader.pages[0]
print(page.extract_text())
"