Chatbot with TinyLlama

app.py

@@ -0,0 +1,120 @@
+import gradio as gr
+from ctransformers import AutoModelForCausalLM, AutoConfig, Config #import for GGUF/GGML models
+import datetime
+
+modelfile="TinyLlama/TinyLlama-1.1B-Chat-v0.6"
+
+i_temperature = 0.30 
+i_max_new_tokens=1100
+i_repetitionpenalty = 1.2
+i_contextlength=12048
+logfile = 'TinyLlama.1B.txt'
+
+print("loading model...")
+
+stt = datetime.datetime.now()
+conf = AutoConfig(Config(temperature=i_temperature, 
+                         repetition_penalty=i_repetitionpenalty, 
+                         batch_size=64,
+                         max_new_tokens=i_max_new_tokens, 
+                         context_length=i_contextlength))
+llm = AutoModelForCausalLM.from_pretrained(modelfile,
+                                           model_type="llama",
+                                           config=conf)
+dt = datetime.datetime.now() - stt
+print(f"Model loaded in {dt}")
+
+def writehistory(text):
+    with open(logfile, 'a', encoding='utf-8') as f:
+        f.write(text)
+        f.write('\n')
+    f.close()
+
+with gr.Blocks(theme='ParityError/Interstellar') as demo: 
+    # TITLE SECTION
+    with gr.Row():
+        with gr.Column(scale=12):
+            gr.HTML("<center>"
+            + "<h1>🦙 TinyLlama 1.1B 🐋 4K context window</h2></center>")  
+            gr.Markdown("""
+            **Currently Running**: [TinyLlama/TinyLlama-1.1B-Chat-v0.6](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v0.6) &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;  **Chat History Log File**: *TinyLlama.1B.txt*
+
+            - **Base Model**: TinyLlama/TinyLlama-1.1B-Chat-v0.6, Fine tuned on OpenOrca GPT4 subset for 1 epoch, Using CHATML format. 
+            - **License**: Apache 2.0, following the TinyLlama base model. 
+                        The model output is not censored and the authors do not endorse the opinions in the generated content. Use at your own risk.
+            """)         
+        gr.Image(value='imgs/TinyLlama_logo.png', width=70)
+
+   # chat and parameters settings
+    with gr.Row():
+        with gr.Column(scale=4):
+            chatbot = gr.Chatbot(height = 350, show_copy_button=True, avatar_images = ["imgs/user_logo.png","imgs/TinyLlama_logo.png"])
+            with gr.Row():
+                with gr.Column(scale=14):
+                    msg = gr.Textbox(show_label=False, placeholder="Enter text", lines=2)
+                submitBtn = gr.Button("\n💬 Send\n", size="lg", variant="primary", min_width=140)
+
+        with gr.Column(min_width=50, scale=1):
+                with gr.Tab(label="Parameter Setting"):
+                    gr.Markdown("# Parameters")
+                    top_p = gr.Slider(minimum=-0, 
+                                      maximum=1.0,
+                                      value=0.95,
+                                      step=0.05,
+                                      interactive=True,
+                                      label="Top-p")
+                    temperature = gr.Slider(minimum=0.1,
+                                            maximum=1.0,
+                                            value=0.30,
+                                            step=0.01,
+                                            interactive=True,
+                                            label="Temperature")
+                    max_length_tokens = gr.Slider(minimum=0,
+                                                  maximum=4096,
+                                                  value=1060,
+                                                  step=4,
+                                                  interactive=True,
+                                                  label="Max Generation Tokens")
+                    rep_pen = gr.Slider(minimum=0,
+                                        maximum=5,
+                                        value=1.2,
+                                        step=0.05,
+                                        interactive=True,
+                                        label="Repetition Penalty")
+
+                clear = gr.Button("🗑️ Clear All Messages", variant='secondary')
+    def user(user_message, history):
+        writehistory(f"USER: {user_message}")
+        return "", history + [[user_message, None]]
+
+    def bot(history, t, p, m, r):
+        SYSTEM_PROMPT = """<|im_start|>system
+        You are a helpful bot. Your answers are clear and concise.
+        <|im_end|>
+
+        """    
+        prompt = f"<|im_start|>system<|im_end|><|im_start|>user\n{history[-1][0]}<|im_end|>\n<|im_start|>assistant\n"  
+        print(f"history lenght: {len(history)}")
+        if len(history) == 1:
+            print("this is the first round")
+        else:
+            print("here we should pass more conversations")
+        history[-1][1] = ""
+        for character in llm(prompt,
+                             temperature = t,
+                             top_p = p, 
+                             repetition_penalty = r, 
+                             max_new_tokens=m,
+                             stop = ['<|im_end|>'],
+                             stream = True):
+            history[-1][1] += character
+            yield history
+        writehistory(f"temperature: {t}, top_p: {p}, maxNewTokens: {m}, repetitionPenalty: {r}\n---\nBOT: {history}\n\n")
+        # Log in the terminal the messages
+        print(f"USER: {history[-1][0]}\n---\ntemperature: {t}, top_p: {p}, maxNewTokens: {m}, repetitionPenalty: {r}\n---\nBOT: {history[-1][1]}\n\n")    
+    # Clicking the submitBtn will call the generation with Parameters in the slides
+    submitBtn.click(user, [msg, chatbot], [msg, chatbot], queue=False).then(bot, [chatbot,temperature,top_p,max_length_tokens,rep_pen], chatbot)
+    clear.click(lambda: None, None, chatbot, queue=False)
+    
+demo.queue()  # required to yield the streams from the text generation
+demo.launch(inbrowser=True, share=True)

requirements.txt

@@ -0,0 +1,86 @@
+aiofiles==23.2.1
+aiohttp==3.9.3
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.3.0
+async-timeout==4.0.3
+attrs==23.2.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+cmake==3.28.3
+colorama==0.4.6
+contourpy==1.2.0
+cycler==0.12.1
+exceptiongroup==1.2.0
+fastapi==0.110.0
+ffmpy==0.3.2
+filelock==3.13.1
+fonttools==4.50.0
+frozenlist==1.4.1
+fsspec==2024.3.0
+gradio==4.21.0
+gradio_client==0.12.0
+h11==0.14.0
+httpcore==1.0.4
+httpx==0.27.0
+huggingface-hub==0.21.4
+idna==3.6
+importlib_resources==6.3.0
+Jinja2==3.1.3
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+linkify-it-py==2.0.3
+lit==18.1.1
+markdown-it-py==2.2.0
+MarkupSafe==2.1.5
+matplotlib==3.8.3
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.5
+networkx==3.2.1
+numpy==1.26.4
+orjson==3.9.15
+packaging==24.0
+pandas==2.2.1
+pillow==10.2.0
+pydantic==2.6.4
+pydantic_core==2.16.3
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+referencing==0.33.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.1
+rpds-py==0.18.0
+ruff==0.3.3
+safetensors==0.4.2
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.36.3
+sympy==1.12
+tokenizers==0.13.3
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.0.1
+tqdm==4.66.2
+transformers==4.31.0
+triton==2.0.0
+typer==0.9.0
+typing_extensions==4.10.0
+tzdata==2024.1
+uc-micro-py==1.0.3
+urllib3==2.2.1
+uvicorn==0.28.0
+websockets==11.0.3
+yarl==1.9.4