Intial

app.py

@@ -1,3 +1,85 @@
+import os
+import torch
 import gradio as gr
+import time
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+from flores200_codes import flores_codes
 
-gr.Interface.load("models/ychenNLP/nllb-200-3.3b-ep").launch()
+
+def load_models():
+    # build model and tokenizer
+    model_name_dict = {
+                  'nllb-3.3B': "models/ychenNLP/nllb-200-3.3b-ep",
+                  }
+
+    model_dict = {}
+
+    for call_name, real_name in model_name_dict.items():
+        print('\tLoading model: %s' % call_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(real_name)
+        tokenizer = AutoTokenizer.from_pretrained(real_name)
+        model_dict[call_name+'_model'] = model
+        model_dict[call_name+'_tokenizer'] = tokenizer
+
+    return model_dict
+
+
+def translation(source, target, text):
+    if len(model_dict) == 2:
+        model_name = 'nllb-3.3B'
+
+    start_time = time.time()
+    source = flores_codes[source]
+    target = flores_codes[target]
+
+    model = model_dict[model_name + '_model']
+    tokenizer = model_dict[model_name + '_tokenizer']
+
+    translator = pipeline('translation', model=model, tokenizer=tokenizer, src_lang=source, tgt_lang=target)
+    output = translator(text, max_length=400)
+
+    end_time = time.time()
+
+    full_output = output
+    output = output[0]['translation_text']
+    result = {'inference_time': end_time - start_time,
+              'source': source,
+              'target': target,
+              'result': output,
+              'full_output': full_output}
+    return result
+
+
+if __name__ == '__main__':
+    print('\tinit models')
+
+    global model_dict
+
+    model_dict = load_models()
+    
+    # define gradio demo
+    lang_codes = list(flores_codes.keys())
+    
+    inputs = [gr.inputs.Dropdown(lang_codes, default='English', label='Source'),
+              gr.inputs.Dropdown(lang_codes, default='Chinese (Simplified)', label='Target'),
+              gr.inputs.Textbox(lines=5, label="Input text"),
+              ]
+
+    outputs = gr.outputs.JSON()
+
+    title = "NLLB 3.3B"
+
+    demo_status = "Demo is running on CPU"
+    description = f"{demo_status}"
+    examples = [
+    ['English', 'Chinese (Simplified)', 'i would like to find flights from [0] columbus [\0] to [1] minneapolis [\1] on [2] monday [\2] [3] june [\3] [4] fourteenth [\4] [5] early [\5] in the [6] morning [\6] [7] or [\7] in the [8] evening [\8] [9] sunday [\9] [10] june [\10] [11] thirteenth [\11] thank you']
+    ]
+
+    gr.Interface(translation,
+                 inputs,
+                 outputs,
+                 title=title,
+                 description=description,
+                 examples=examples,
+                 examples_per_page=50,
+                 ).launch()