Setting up

README.md

@@ -1,12 +1,16 @@
----
-title: Text To Text Translator
-emoji: 📚
-colorFrom: red
-colorTo: gray
-sdk: gradio
-sdk_version: 4.32.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Text_to_Text_Translator
+app_file: app.py
+sdk: gradio
+sdk_version: 4.32.0
+---
+## Text to Text Translator
+
+Built a Text to Text Translator using NLTK and Transformers.
+- Supports Translation of English to Bengali, Tamil, Telugu, Gujarati, Marathi and Hindi.
+-  Uses BanglaT5 which achieved an exceptional score of <b>25.2</b> on SacreBLEU metric while mt5 (Industry Standard) scored much lower at <b>22.5</b>
+
+
+Future Work:
+- Adding functionality of uploading Images and Files
+- OCR will run on these files and provide translation automatically

app.py

@@ -0,0 +1,22 @@
+#User Interface 
+
+import gradio as gr
+import main
+
+def test(text, src, dest):
+  ans = main.main_translation(text,dest,src)
+  return ans['output']
+demo = gr.Interface(
+    test,
+    ["textbox",
+     gr.Dropdown(
+            [("English", "en_XX"), ("Hindi","hi_IN"), ("Bengali","bn_IN"), ("Gujarati","gu_IN"), ("Tamil","ta_IN"), ("Telugu","te_IN"), ("Marathi","mr_IN")], label="Source", info="Select the Source Language!"
+        ), 
+     gr.Dropdown(
+            [("English", "en_XX"), ("Hindi","hi_IN"), ("Bengali","bn_IN"), ("Gujarati","gu_IN"), ("Tamil","ta_IN"), ("Telugu","te_IN"), ("Marathi","mr_IN")], label="Destination", info="Select the Destination Language!"
+        ), 
+     ],
+    outputs=["textbox"],
+)
+
+demo.launch(share=True)

main.py

@@ -0,0 +1,77 @@
+import requests
+from transformers import pipeline
+import nltk
+from nltk import sent_tokenize
+from transformers import MBartForConditionalGeneration, MBart50TokenizerFast
+from transformers import pipeline
+
+# nltk.download('punkt') # Run only once
+
+tokenizer = MBart50TokenizerFast.from_pretrained("SnypzZz/Llama2-13b-Language-translate", src_lang="en_XX")
+#pipe = pipeline("text2text-generation", model="SnypzZz/Llama2-13b-Language-translate", tokenizer=tokenizer)
+model = None
+model_loaded = False
+
+api_token_header = ""
+with open('./secret.py', 'r') as f:
+  api_token_header = f.read()
+
+def load_model():
+    global model, model_loaded
+    model = MBartForConditionalGeneration.from_pretrained("SnypzZz/Llama2-13b-Language-translate") 
+    model_loaded =True
+    return model
+
+def translation(text,dest_lang,dest_lang_code, src_lang_code):
+  
+    if(dest_lang_code == src_lang_code):
+        return "Please select different languages to translate between."
+    
+    # headers = {"Authorization": f"Bearer {secrets_sih.api_token_header}"}
+    headers = {"Authorization": f"Bearer {api_token_header}"}
+    
+    # Bengali Done
+    if(dest_lang == "Bengali" and src_lang_code == "en_XX"):
+        API_URL = "https://api-inference.huggingface.co/models/csebuetnlp/banglat5_nmt_en_bn"
+        def query(payload):
+            response = requests.post(API_URL, headers=headers, json=payload)
+            return response.json()
+        output = query({
+            "inputs": text,
+        })
+        print(output)
+        return output[0]['translation_text']
+    else:
+        global model
+        if model:
+            pass
+        else:
+            model = load_model()
+        loaded_model = model
+        tokenizer = MBart50TokenizerFast.from_pretrained("SnypzZz/Llama2-13b-Language-translate", src_lang=src_lang_code)
+        #model_inputs = tokenizer(text, return_tensors="pt")
+        loaded_model_inputs = tokenizer(text, return_tensors="pt")
+
+        # translate
+        generated_tokens = loaded_model.generate(
+            **loaded_model_inputs,
+            forced_bos_token_id=tokenizer.lang_code_to_id[dest_lang_code]
+        )
+        output = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
+        print(output)
+        return output[0]
+
+def main_translation(text,dest_lang_code,src_lang_code):
+    
+    codes = {"en_XX":"English","bn_IN":"Bengali", "en_GB":"English","gu_IN":"Gujarati","hi_IN":"Hindi","ta_IN":"Tamil","te_IN":"Telugu","mr_IN":"Marathi"}
+    dest_lang = codes[dest_lang_code]
+    src_lang = codes[src_lang_code]
+
+    sentences = sent_tokenize(text)
+    output = ""
+    for line in sentences:
+        output += translation(line,dest_lang,dest_lang_code, src_lang_code)
+    return {"output":output}
+
+
+print(main_translation("hello world", "hi_IN", "en_XX"))