ivanlau commited on
Commit
aa0c704
1 Parent(s): 1881b18

added app file

Browse files

Files changed (3) hide show
  1. app.py +49 -0
  2. languages.py +47 -0
  3. requirements.txt +3 -0
app.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import AutoModelForSequenceClassification, AutoTokenizer
2
+ from languages import LANGUANGE_MAP
3
+ import gradio as gr
4
+ import torch
5
+
6
+
7
+ model_ckpt = "ivanlau/language-detection-fine-tuned-on-xlm-roberta-base"
8
+ model = AutoModelForSequenceClassification.from_pretrained(model_ckpt)
9
+ tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
10
+
11
+
12
+
13
+ def detect_language(sentence):
14
+ tokenized_sentence = tokenizer(sentence, return_tensors='pt')
15
+ output = model(**tokenized_sentence)
16
+ predictions = torch.nn.functional.softmax(output.logits, dim=-1)
17
+ _, preds = torch.max(predictions, dim=-1)
18
+ return LANGUANGE_MAP[preds.item()]
19
+
20
+ examples = [
21
+ "I've been waiting for a HuggingFace course my whole life.",
22
+ "恭喜发财!",
23
+ "Jumpa lagi, saya pergi kerja.",
24
+ "你食咗飯未呀?",
25
+ "もう食べましたか?",
26
+ "as-tu mangé"
27
+ ]
28
+
29
+ inputs=gr.inputs.Textbox(placeholder="Enter your text here", label="Text content", lines=5)
30
+ outputs=gr.outputs.Label(num_top_classes=3, label="Language detected:")
31
+ article = """
32
+ Supported languages:
33
+ 'Arabic', 'Basque', 'Breton', 'Catalan', 'Chinese_China', 'Chinese_Hongkong', 'Chinese_Taiwan', 'Chuvash', 'Czech',
34
+ 'Dhivehi', 'Dutch', 'English', 'Esperanto', 'Estonian', 'French', 'Frisian', 'Georgian', 'German', 'Greek', 'Hakha_Chin',
35
+ 'Indonesian', 'Interlingua', 'Italian', 'Japanese', 'Kabyle', 'Kinyarwanda', 'Kyrgyz', 'Latvian', 'Maltese',
36
+ 'Mangolian', 'Persian', 'Polish', 'Portuguese', 'Romanian', 'Romansh_Sursilvan', 'Russian', 'Sakha', 'Slovenian',
37
+ 'Spanish', 'Swedish', 'Tamil', 'Tatar', 'Turkish', 'Ukranian', 'Welsh'
38
+ """
39
+ gr.Interface(
40
+ fn=detect_language,
41
+ inputs=inputs,
42
+ outputs=outputs,
43
+ verbose=True,
44
+ examples = examples,
45
+ title="Language Detector",
46
+ description="A simple language detector fine-tuned from xlm-roberta-base model which can detect 45 languages.",
47
+ article=article,
48
+ theme="huggingface"
49
+ ).launch()
languages.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ LANGUANGE_MAP = {
2
+ 0: 'Arabic',
3
+ 1: 'Basque',
4
+ 2: 'Breton',
5
+ 3: 'Catalan',
6
+ 4: 'Chinese_China',
7
+ 5: 'Chinese_Hongkong',
8
+ 6: 'Chinese_Taiwan',
9
+ 7: 'Chuvash',
10
+ 8: 'Czech',
11
+ 9: 'Dhivehi',
12
+ 10: 'Dutch',
13
+ 11: 'English',
14
+ 12: 'Esperanto',
15
+ 13: 'Estonian',
16
+ 14: 'French',
17
+ 15: 'Frisian',
18
+ 16: 'Georgian',
19
+ 17: 'German',
20
+ 18: 'Greek',
21
+ 19: 'Hakha_Chin',
22
+ 20: 'Indonesian',
23
+ 21: 'Interlingua',
24
+ 22: 'Italian',
25
+ 23: 'Japanese',
26
+ 24: 'Kabyle',
27
+ 25: 'Kinyarwanda',
28
+ 26: 'Kyrgyz',
29
+ 27: 'Latvian',
30
+ 28: 'Maltese',
31
+ 29: 'Mongolian',
32
+ 30: 'Persian',
33
+ 31: 'Polish',
34
+ 32: 'Portuguese',
35
+ 33: 'Romanian',
36
+ 34: 'Romansh_Sursilvan',
37
+ 35: 'Russian',
38
+ 36: 'Sakha',
39
+ 37: 'Slovenian',
40
+ 38: 'Spanish',
41
+ 39: 'Swedish',
42
+ 40: 'Tamil',
43
+ 41: 'Tatar',
44
+ 42: 'Turkish',
45
+ 43: 'Ukranian',
46
+ 44: 'Welsh'
47
+ }
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
1
+ gradio
2
+ transformers
3
+ torch