use external inference

README.md

@@ -7,7 +7,7 @@ sdk: gradio
 sdk_version: 3.18.0
 app_file: app.py
 pinned: false
-license: cc
+license: creativeml-openrail-m
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,38 +1,38 @@
-import torch
 import gradio as gr
-from transformers import pipeline
 import os
 from mtranslate import translate
+import requests
 
-device = torch.cuda.current_device() if torch.cuda.is_available() else "cpu"
 HF_AUTH_TOKEN = os.environ.get("HF_AUTH_TOKEN")
-text_generation_model = "cahya/indochat-tiny"
-text_generation = pipeline("text-generation", text_generation_model, use_auth_token=HF_AUTH_TOKEN, device=device)
+indochat_api = 'https://cahya-indonesian-whisperer.hf.space/api/indochat/v1'
+indochat_api_auth_token = os.getenv("INDOCHAT_API_AUTH_TOKEN", "")
 
-
-def get_answer(user_input, decoding_methods, num_beams, top_k, top_p, temperature, repetition_penalty, penalty_alpha):
-    if decoding_methods == "Beam Search":
-        do_sample = False
-        penalty_alpha = 0
-    elif decoding_methods == "Sampling":
-        do_sample = True
-        penalty_alpha = 0
-        num_beams = 1
+def get_answer(user_input, decoding_method, num_beams, top_k, top_p, temperature, repetition_penalty, penalty_alpha):
+    print(user_input, decoding_method, top_k, top_p, temperature, repetition_penalty, penalty_alpha)
+    headers = {'Authorization': 'Bearer ' + indochat_api_auth_token}
+    data = {
+        "text": user_input,
+        "min_length": len(user_input) + 50,
+        "max_length": 300,
+        "decoding_method": decoding_method,
+        "num_beams": num_beams,
+        "top_k": top_k,
+        "top_p": top_p,
+        "temperature": temperature,
+        "seed": -1,
+        "repetition_penalty": repetition_penalty,
+        "penalty_alpha": penalty_alpha
+    }
+    r = requests.post(indochat_api, headers=headers, data=data)
+    if r.status_code == 200:
+        result = r.json()
+        answer = result["generated_text"]
+        user_input_en = translate(user_input, "en", "id")
+        answer_en = translate(answer, "en", "id")
+        return [(f"{user_input}\n", None), (answer, "")], \
+            [(f"{user_input_en}\n", None), (answer_en, "")]
     else:
-        do_sample = False
-        num_beams = 1
-    print(user_input, decoding_methods, do_sample, top_k, top_p, temperature, repetition_penalty, penalty_alpha)
-    prompt = f"User: {user_input}\nAssistant: "
-    generated_text = text_generation(f"{prompt}", min_length=50, max_length=200, num_return_sequences=1,
-                                     num_beams=num_beams, do_sample=do_sample, top_k=top_k, top_p=top_p,
-                                     temperature=temperature, repetition_penalty=repetition_penalty,
-                                     penalty_alpha=penalty_alpha)
-    answer = generated_text[0]["generated_text"]
-    answer_without_prompt = answer[len(prompt)+1:]
-    user_input_en = translate(user_input, "en", "id")
-    answer_without_prompt_en = translate(answer_without_prompt, "en", "id")
-    return [(f"{user_input}\n", None), (answer_without_prompt, "")], \
-        [(f"{user_input_en}\n", None), (answer_without_prompt_en, "")]
+        return "Error: " + r.text
 
 
 css = """
@@ -55,7 +55,7 @@ with gr.Blocks(css=css) as demo:
             user_input = gr.inputs.Textbox(placeholder="",
                                            label="Ask me something in Indonesian or English",
                                            default="Bagaimana cara mendidik anak supaya tidak berbohong?")
-            decoding_methods = gr.inputs.Dropdown(["Beam Search", "Sampling", "Contrastive Search"],
+            decoding_method = gr.inputs.Dropdown(["Beam Search", "Sampling", "Contrastive Search"],
                                                   default="Sampling", label="Decoding Method")
             num_beams = gr.inputs.Slider(label="Number of beams for beam search",
                                      default=1, minimum=1, maximum=10, step=1)
@@ -85,7 +85,7 @@ with gr.Blocks(css=css) as demo:
         gr.Markdown("![visitor badge](https://visitor-badge.glitch.me/badge?page_id=cahya_indochat)")
 
     button_generate_story.click(get_answer,
-                                inputs=[user_input, decoding_methods, num_beams, top_k, top_p, temperature,
+                                inputs=[user_input, decoding_method, num_beams, top_k, top_p, temperature,
                                         repetition_penalty, penalty_alpha],
                                 outputs=[generated_answer, generated_answer_en])