kaushikbar
added explain bb5868c
import datetime
import gradio as gr
from huggingface_hub import hf_hub_download
from langdetect import detect, DetectorFactory, detect_langs
import fasttext
from transformers import pipeline
from transformers_interpret import ZeroShotClassificationExplainer
import string, nltk
models = {'en': 'facebook/bart-large-mnli', #'Narsil/deberta-large-mnli-zero-cls', #'microsoft/deberta-xlarge-mnli', # English
#'de': 'Sahajtomar/German_Zeroshot', # German
#'es': 'Recognai/zeroshot_selectra_medium', # Spanish
#'it': 'joeddav/xlm-roberta-large-xnli', # Italian
#'ru': 'DeepPavlov/xlm-roberta-large-en-ru-mnli', # Russian
#'tr': 'vicgalle/xlm-roberta-large-xnli-anli', # Turkish
'no': 'NbAiLab/nb-bert-base-mnli'} # Norsk
hypothesis_templates = {'en': 'This passage talks about {}.', # English
#'de': 'Dieses beispiel ist {}.', # German
#'es': 'Este ejemplo es {}.', # Spanish
#'it': 'Questo esempio è {}.', # Italian
#'ru': 'Этот пример {}.', # Russian
#'tr': 'Bu örnek {}.', # Turkish
'no': 'Dette eksempelet er {}.'} # Norsk
classifiers = {'en': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['en'],
model=models['en']),
'''
'de': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['de'],
model=models['de']),
'es': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['es'],
model=models['es']),
'it': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['it'],
model=models['it']),
'ru': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['ru'],
model=models['ru']),
'tr': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['tr'],
model=models['tr']),
'''
'no': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['no'],
model=models['no'])}
fasttext_model = fasttext.load_model(hf_hub_download("julien-c/fasttext-language-id", "lid.176.bin"))
_ = nltk.download('stopwords', quiet=True)
#_ = nltk.download('wordnet', quiet=True)
#_ = nltk.download('punkt', quiet=True)
def prep_examples():
example_text1 = "Coronavirus disease (COVID-19) is an infectious disease caused by the SARS-CoV-2 virus. Most \
people who fall sick with COVID-19 will experience mild to moderate symptoms and recover without special treatment. \
However, some will become seriously ill and require medical attention."
example_labels1 = "business;;health related;;politics;;climate change"
example_text2 = "Elephants are"
example_labels2 = "big;;small;;strong;;fast;;carnivorous"
example_text3 = "Elephants"
example_labels3 = "are big;;can be very small;;generally not strong enough;;are faster than you think"
example_text4 = "Dogs are man's best friend"
example_labels4 = "positive;;negative;;neutral"
example_text5 = "Şampiyonlar Ligi’nde 5. hafta oynanan karşılaşmaların ardından sona erdi. Real Madrid, \
Inter ve Sporting oynadıkları mücadeleler sonrasında Son 16 turuna yükselmeyi başardı. \
Gecenin dev mücadelesinde ise Manchester City, PSG’yi yenerek liderliği garantiledi."
example_labels5 = "dünya;;ekonomi;;kültür;;siyaset;;spor;;teknoloji"
example_text6 = "Letzte Woche gab es einen Selbstmord in einer nahe gelegenen kolonie"
example_labels6 = "verbrechen;;tragödie;;stehlen"
example_text7 = "El autor se perfila, a los 50 años de su muerte, como uno de los grandes de su siglo"
example_labels7 = "cultura;;sociedad;;economia;;salud;;deportes"
example_text8 = "Россия в среду заявила, что военные учения в аннексированном Москвой Крыму закончились \
и что солдаты возвращаются в свои гарнизоны, на следующий день после того, как она объявила о первом выводе \
войск от границ Украины."
example_labels8 = "новости;;комедия"
example_text9 = "I quattro registi - Federico Fellini, Pier Paolo Pasolini, Bernardo Bertolucci e Vittorio De Sica - \
hanno utilizzato stili di ripresa diversi, ma hanno fortemente influenzato le giovani generazioni di registi."
example_labels9 = "cinema;;politica;;cibo"
example_text10 = "Ja, vi elsker dette landet,\
som det stiger frem,\
furet, værbitt over vannet,\
med de tusen hjem.\
Og som fedres kamp har hevet\
det av nød til seir"
example_labels10 = "helse;;sport;;religion;;mat;;patriotisme og nasjonalisme"
example_text11 = "Amar sonar bangla ami tomay bhalobasi"
example_labels11 = "bhalo;;kharap"
examples = [
[example_text1, example_labels1],
[example_text2, example_labels2],
[example_text3, example_labels3],
[example_text4, example_labels4],
[example_text5, example_labels5],
[example_text6, example_labels6],
[example_text7, example_labels7],
[example_text8, example_labels8],
[example_text9, example_labels9],
[example_text10, example_labels10],
[example_text11, example_labels11]]
return examples
def detect_lang(sequence, labels):
DetectorFactory.seed = 0
seq_lang = 'en'
sequence = sequence.replace('\n', ' ')
try:
#seq_lang = detect(sequence)
#lbl_lang = detect(labels)
seq_lang = fasttext_model.predict(sequence, k=1)[0][0].split("__label__")[1]
lbl_lang = fasttext_model.predict(labels, k=1)[0][0].split("__label__")[1]
except:
print("Language detection failed!",
"Date:{}, Sequence:{}, Labels:{}".format(
str(datetime.datetime.now()),
labels))
if seq_lang != lbl_lang:
print("Different languages detected for sequence and labels!",
"Date:{}, Sequence:{}, Labels:{}, Sequence Language:{}, Label Language:{}".format(
str(datetime.datetime.now()),
sequence,
labels,
seq_lang,
lbl_lang))
if seq_lang in models:
print("Sequence Language detected.",
"Date:{}, Sequence:{}, Sequence Language:{}".format(
str(datetime.datetime.now()),
sequence,
seq_lang))
else:
print("Language not supported. Defaulting to English!",
"Date:{}, Sequence:{}, Sequence Language:{}".format(
str(datetime.datetime.now()),
sequence,
seq_lang))
seq_lang = 'en'
return seq_lang
def sequence_to_classify(sequence, labels):
classifier = classifiers[detect_lang(sequence, labels)]
label_clean = str(labels).split(";;")
response = classifier(sequence, label_clean, multi_label=True)
predicted_labels = response['labels']
print(predicted_labels)
predicted_scores = response['scores']
print(predicted_scores)
clean_output = {idx: float(predicted_scores.pop(0)) for idx in predicted_labels}
print("Date:{}, Sequence:{}, Labels: {}".format(
str(datetime.datetime.now()),
sequence,
predicted_labels))
# Explain word attributes
stop_words = nltk.corpus.stopwords.words('english')
puncts = list(string.punctuation)
model_expl = ZeroShotClassificationExplainer(classifier.model, classifier.tokenizer)
response_expl = model_expl(sequence, label_clean, hypothesis_template="This passage talks about {}.")
print(model_expl.predicted_label)
if len(predicted_labels) == 1:
response_expl = response_expl[model_expl.predicted_label]
for key in response_expl:
for idx, elem in enumerate(response_expl[key]):
if elem[0] in stop_words:
del response_expl[key][idx]
print(response_expl)
return clean_output
iface = gr.Interface(
title="Multilingual Multi-label Zero-shot Classification",
description="Currently supported languages are English, German, Spanish, Italian, Russian, Turkish, Norsk.",
fn=sequence_to_classify,
inputs=[gr.inputs.Textbox(lines=10,
label="Please enter the text you would like to classify...",
placeholder="Text here..."),
gr.inputs.Textbox(lines=2,
label="Please enter the candidate labels (separated by 2 consecutive semicolons)...",
placeholder="Labels here separated by ;;")],
outputs=gr.outputs.Label(num_top_classes=5),
#interpretation="default",
examples=prep_examples())
iface.launch()