Update app.py

app.py

@@ -9,6 +9,10 @@ import gradio as gr
 import sentencepiece
 
 
+# os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:126'
+MAX_MAX_NEW_TOKENS = 160000
+DEFAULT_MAX_NEW_TOKENS = 20000
+MAX_INPUT_TOKEN_LENGTH = 160000
 DESCRIPTION = """
 # Welcome to Tonic'sYI-6B-200K
 You can use this Space to test out the current model [01-ai/Yi-6B-200K](https://huggingface.co/01-ai/Yi-6B-200K)
@@ -16,231 +20,63 @@ You can also use YI-200 by cloning this space. Simply click here: <a style="disp
 Join us : TeamTonic is always making cool demos! Join our active builder's community on Discord: [Discord](https://discord.gg/nXx5wbX9) On Huggingface: [TeamTonic](https://huggingface.co/TeamTonic) & [MultiTransformer](https://huggingface.co/MultiTransformer) On Github: [Polytonic](https://github.com/tonic-ai) & contribute to [PolyGPT](https://github.com/tonic-ai/polygpt-alpha)
 """
 
-os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:126'
-
-MAX_MAX_NEW_TOKENS = 160000
-DEFAULT_MAX_NEW_TOKENS = 20000
-MAX_INPUT_TOKEN_LENGTH = 160000
-device = "cuda" if torch.cuda.is_available() else "cpu"
 
+# Set up the model and tokenizer
 model_name = "01-ai/Yi-6B-200K"
-
-tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, device_map="cuda", trust_remote_code=True)
-# tokenizer = YiTokenizer(vocab_file=model_name)
-model = transformers.AutoModelForCausalLM.from_pretrained(model_name,
+device = "cuda" if torch.cuda.is_available() else "cpu"
+tokenizer = AutoTokenizer.from_pretrained(model_name, device_map="cuda", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
     device_map="auto",
     torch_dtype=torch.bfloat16,
     load_in_4bit=True,
     trust_remote_code=True
 )
+model.to(device)
 
-def run(message, chat_history, max_new_tokens=20000, temperature=1.5, top_p=0.9, top_k=900):
-    prompt = get_prompt(message, chat_history)
-
-    # Encode the prompt to tensor
-    input_ids = tokenizer.encode(prompt, return_tensors='pt')
-
-    # Move input_ids to the same device as the model
-    input_ids = input_ids.to(model.device)
-    
-    # Generate a response using the model with adjusted parameters
+def run(prompt, max_new_tokens, temperature, top_p, top_k):
+    input_ids = tokenizer.encode(prompt, return_tensors='pt').to(device)
     response_ids = model.generate(
         input_ids,
         max_length=max_new_tokens + input_ids.shape[1],
-        temperature=temperature,  # Controls randomness. Lower values make text more deterministic.
-        top_p=top_p,              # Nucleus sampling: higher values allow more diversity.
-        top_k=top_k,              # Top-k sampling: limits the number of top tokens considered.
+        temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
         pad_token_id=tokenizer.eos_token_id,
-        do_sample=True            # Enable sampling-based generation
-
+        do_sample=True
     )
-
-    # Decode the response
     response = tokenizer.decode(response_ids[:, input_ids.shape[-1]:][0], skip_special_tokens=True)
     return response
 
-def get_prompt(message, chat_history):
-    texts = []
-
-    do_strip = False
-    for user_input, response in chat_history:
-        user_input = user_input.strip() if do_strip else user_input
-        do_strip = True
-        texts.append(f" {response.strip()} {user_input} ")
-    message = message.strip() if do_strip else message
-    texts.append(f"{message}")
-    return ''.join(texts)
-
-def clear_and_save_textbox(message): return '', message
-
-def display_input(message, history=[]):
-    history.append((message, ''))
-    return history
-
-def delete_prev_fn(history=[]):
-    try:
-        message, _ = history.pop()
-    except IndexError:
-        message = ''
-    return history, message or ''
-
-def generate(message, history_with_input, max_new_tokens, temperature, top_p, top_k):
-    if int(max_new_tokens) > MAX_MAX_NEW_TOKENS:
-        raise ValueError
-
-    history = history_with_input[:-1]
-    response = run(message, history, max_new_tokens, temperature, top_p, top_k)
-    yield history + [(message, response)]
-
-
-def process_example(message):
-    generator = generate(message, [], 4056, 1.9, 0.95, 900)
-    for x in generator:
-        pass
-    return '', x
-
-def check_input_token_length(message, chat_history):
-    input_token_length = len(message) + len(chat_history)
-    if input_token_length > MAX_INPUT_TOKEN_LENGTH:
-        raise gr.Error(f"The accumulated input is too long ({input_token_length} > {MAX_INPUT_TOKEN_LENGTH}). Clear your chat history and try again.")
+def generate(prompt, max_new_tokens, temperature, top_p, top_k):
+    response = run(prompt, max_new_tokens, temperature, top_p, top_k)
+    return response
 
+# Gradio Interface
 with gr.Blocks(theme='ParityError/Anime') as demo:
     gr.Markdown(DESCRIPTION)
-
-
     
     with gr.Group():
-        chatbot = gr.Chatbot(label='TonicYi-30B-200K')
         with gr.Row():
-            textbox = gr.Textbox(
-                container=False,
-                show_label=False,
-                placeholder='As the dawn approached, they leant in and said',
-                scale=10
+            prompt = gr.Textbox(
+                label='Enter your prompt',
+                placeholder='Type something...',
+                lines=5
             )
-            submit_button = gr.Button('Submit', variant='primary', scale=1, min_width=0)
-
-    with gr.Row():
-        retry_button = gr.Button('Retry', variant='secondary')
-        undo_button = gr.Button('Undo', variant='secondary')
-        clear_button = gr.Button('Clear', variant='secondary')
-
-    saved_input = gr.State()
+            submit_button = gr.Button('Generate')
 
     with gr.Accordion(label='Advanced options', open=False):
-#       system_prompt = gr.Textbox(label='System prompt', value=DEFAULT_SYSTEM_PROMPT, lines=5, interactive=False)
         max_new_tokens = gr.Slider(label='Max New Tokens', minimum=1, maximum=MAX_MAX_NEW_TOKENS, step=1, value=DEFAULT_MAX_NEW_TOKENS)
-        temperature = gr.Slider(label='Temperature', minimum=0.1, maximum=2.0, step=0.1, value=0.1)
+        temperature = gr.Slider(label='Temperature', minimum=0.1, maximum=2.0, step=0.1, value=1.2)
         top_p = gr.Slider(label='Top-P (nucleus sampling)', minimum=0.05, maximum=1.0, step=0.05, value=0.9)
-        top_k = gr.Slider(label='Top-K', minimum=1, maximum=1000, step=1, value=10)
+        top_k = gr.Slider(label='Top-K', minimum=1, maximum=1000, step=1, value=900)
 
-    textbox.submit(
-        fn=clear_and_save_textbox,
-        inputs=textbox,
-        outputs=[textbox, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=display_input,
-        inputs=[saved_input, chatbot],
-        outputs=chatbot,
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=check_input_token_length,
-        inputs=[saved_input, chatbot],
-        api_name=False,
-        queue=False,
-    ).success(
-        fn=generate,
-        inputs=[
-            saved_input,
-            chatbot,
-            max_new_tokens,
-            temperature,
-            top_p,
-            top_k,
-        ],
-        outputs=chatbot,
-        api_name="Generate",
-    )
-
-    button_event_preprocess = submit_button.click(
-        fn=clear_and_save_textbox,
-        inputs=textbox,
-        outputs=[textbox, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=display_input,
-        inputs=[saved_input, chatbot],
-        outputs=chatbot,
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=check_input_token_length,
-        inputs=[saved_input, chatbot],
-        api_name=False,
-        queue=False,
-    ).success(
-        fn=generate,
-        inputs=[
-            saved_input,
-            chatbot,
-            max_new_tokens,
-            temperature,
-            top_p,
-            top_k,
-        ],
-        outputs=chatbot,
-        api_name="Cgenerate",
-    )
+    output = gr.Textbox(label='Generated Text', lines=10, readonly=True)
 
-    retry_button.click(
-        fn=delete_prev_fn,
-        inputs=chatbot,
-        outputs=[chatbot, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=display_input,
-        inputs=[saved_input, chatbot],
-        outputs=chatbot,
-        api_name=False,
-        queue=False,
-    ).then(
+    submit_button.click(
         fn=generate,
-        inputs=[
-            saved_input,
-            chatbot,
-            max_new_tokens,
-            temperature,
-            top_p,
-            top_k,
-        ],
-        outputs=chatbot,
-        api_name=False,
-    )
-
-    undo_button.click(
-        fn=delete_prev_fn,
-        inputs=chatbot,
-        outputs=[chatbot, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=lambda x: x,
-        inputs=[saved_input],
-        outputs=textbox,
-        api_name=False,
-        queue=False,
-    )
-
-    clear_button.click(
-        fn=lambda: ([], ''),
-        outputs=[chatbot, saved_input],
-        queue=False,
-        api_name=False,
+        inputs=[prompt, max_new_tokens, temperature, top_p, top_k],
+        outputs=output
     )
 
 demo.queue(max_size=5).launch(show_api=True)