Hugging Face's logo

Spaces:
dinalzein
/
LanguageIdentifier
Runtime error

App Files Community
LanguageIdentifier
File size: 2,138 Bytes 
409b791
 
 
 
 
 
fafbc3d
409b791
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f4da052
409b791
 
 
 
0eab03f
409b791
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer
import gradio as gr
import torch
import numpy as np
from mapping_labels import languages_map, id2label

model_checkpoint = "dinalzein/xlm-roberta-base-finetuned-language-identification"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)
model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint)
trainer = Trainer(model)

class Dataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels=None):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        if self.labels:
            item["labels"] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.encodings["input_ids"])

def identify_language(txt):
  txt=[txt]
  tokenized_txt = tokenizer(txt, truncation=True, max_length=20)
  txt_dataset = Dataset(tokenized_txt)
  raw_pred, _, _ = trainer.predict(txt_dataset)
  # Preprocess raw predictions
  y_pred = np.argmax(raw_pred, axis=1)
  return languages_map[id2label[str(y_pred[0])]]


#with gr.Row():
examples = [
    "C'est La Vie",
    "So ist das Leben",
    "That is life",
    "Ω‡Ψ°Ω‡ Ω‡ΩŠ Ψ§Ω„Ψ­ΩŠΨ§Ψ©"
]


inputs=gr.inputs.Textbox(placeholder="Enter your text here", label="Text content", lines=5)
outputs=gr.outputs.Label(label="Language Identified:")


article = ('''## Suppoted Langauges \n
Arabic (ar), Bulgarian (bg), German (de), Modern greek (el), English (en), Spanish (es), French (fr), Hindi (hi), Italian (it), Japanese (ja), Dutch (nl), Polish (pl), Portuguese (pt), Russian (ru), Swahili (sw), Thai (th), Turkish (tr), Urdu (ur), Vietnamese (vi), Chinese (zh)
''')


gr.Interface(
    fn=identify_language,
    inputs=inputs,
    outputs=outputs,
    verbose=True,
    examples = examples,
    title="Language Identifier",
    description="It aims at identifing the language a document is written in. It supports 20 different languages.",
    article=article,
    theme="huggingface"
).launch()