File size: 1,932 Bytes
aa0c704 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from languages import LANGUANGE_MAP
import gradio as gr
import torch
model_ckpt = "ivanlau/language-detection-fine-tuned-on-xlm-roberta-base"
model = AutoModelForSequenceClassification.from_pretrained(model_ckpt)
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
def detect_language(sentence):
tokenized_sentence = tokenizer(sentence, return_tensors='pt')
output = model(**tokenized_sentence)
predictions = torch.nn.functional.softmax(output.logits, dim=-1)
_, preds = torch.max(predictions, dim=-1)
return LANGUANGE_MAP[preds.item()]
examples = [
"I've been waiting for a HuggingFace course my whole life.",
"恭喜发财!",
"Jumpa lagi, saya pergi kerja.",
"你食咗飯未呀?",
"もう食べましたか?",
"as-tu mangé"
]
inputs=gr.inputs.Textbox(placeholder="Enter your text here", label="Text content", lines=5)
outputs=gr.outputs.Label(num_top_classes=3, label="Language detected:")
article = """
Supported languages:
'Arabic', 'Basque', 'Breton', 'Catalan', 'Chinese_China', 'Chinese_Hongkong', 'Chinese_Taiwan', 'Chuvash', 'Czech',
'Dhivehi', 'Dutch', 'English', 'Esperanto', 'Estonian', 'French', 'Frisian', 'Georgian', 'German', 'Greek', 'Hakha_Chin',
'Indonesian', 'Interlingua', 'Italian', 'Japanese', 'Kabyle', 'Kinyarwanda', 'Kyrgyz', 'Latvian', 'Maltese',
'Mangolian', 'Persian', 'Polish', 'Portuguese', 'Romanian', 'Romansh_Sursilvan', 'Russian', 'Sakha', 'Slovenian',
'Spanish', 'Swedish', 'Tamil', 'Tatar', 'Turkish', 'Ukranian', 'Welsh'
"""
gr.Interface(
fn=detect_language,
inputs=inputs,
outputs=outputs,
verbose=True,
examples = examples,
title="Language Detector",
description="A simple language detector fine-tuned from xlm-roberta-base model which can detect 45 languages.",
article=article,
theme="huggingface"
).launch()
|