Deployment/app.py

# Delete all objects from memory

keys = list(globals().keys())

for o in keys:
    if not o.startswith('_'):
        print(o)
        del globals()[o]

# Imort from a file called Bert-medium.py

from Bert_medium import MediumBert
from Offensive_Bert import BertClassifier
from data_cleaning import cleaning_content
from Dialect_Bert import Dialect_Detection

import torch
device = torch.device("cpu")


from transformers import BertTokenizer, AutoTokenizer, BertTokenizerFast
import streamlit as st

# file path
import os

path_file = os.path.dirname(os.path.abspath(__file__))
parent_path = os.path.dirname(path_file)

##########################FUNCTIONS########################

def predict_off(review_text,model,device,tokenizer):

        encoded_review = tokenizer.encode_plus(
        review_text,
        max_length=256,
        add_special_tokens=True,
        return_token_type_ids=False,
        padding='longest',
        return_attention_mask=True,
        return_tensors='pt',
        )

        input_ids = encoded_review['input_ids'].to(device)
        attention_mask = encoded_review['attention_mask'].to(device)
        output = model(input_ids, attention_mask)
        _, prediction = torch.max(output, dim=1)
        #print(f'Review text: {review_text}')
        index = output.cpu().data.numpy().argmax()
        #print(f'Sentiment  : {index}')
        # decode the output of the model to get the predicted label
        pred = index
        
        return pred
#########################################""
def predict_other(review_text,model,device,tokenizer):
        
        encoded_review = tokenizer.encode_plus(
        review_text,
        max_length=217,
        add_special_tokens=True,
        return_token_type_ids=False,
        padding='longest',
        return_attention_mask=True,
        return_tensors='pt',
        )

        input_ids = encoded_review['input_ids'].to(device)
        attention_mask = encoded_review['attention_mask'].to(device)
        output = model(input_ids, attention_mask)
        _, prediction = torch.max(output, dim=1)
        #print(f'Review text: {review_text}')
        index = output.cpu().data.numpy().argmax()
        #print(f'Sentiment  : {index}')
        # decode the output of the model to get the predicted label

        return index
#########################"##################

def predict_dialect(review_text,model,device,tokenizer):
        
        encoded_review = tokenizer.encode_plus(
        review_text,
        max_length=123,
        add_special_tokens=True,
        return_token_type_ids=False,
        padding='longest',
        return_attention_mask=True,
        return_tensors='pt',
        )

        input_ids = encoded_review['input_ids'].to(device)
        attention_mask = encoded_review['attention_mask'].to(device)
        output = model(input_ids, attention_mask)
        _, prediction = torch.max(output, dim=1)
        #print(f'Review text: {review_text}')
        index = output.cpu().data.numpy().argmax()
        #print(f'Sentiment  : {index}')
        pred = index
        return pred


# Main prediction function

def predict(text,device,offensive_model,offensive_tokenizer,racism_model,misogyny_model,verbalabuse_model,dialect_model,religionhate_model,tokenizer_dialect,other_tokenizer,off_dictionary,racism_dict,misogyny_dict,verbalabuse_dict,dialect_dict,religionhate_dict):
        # clean text
        text = cleaning_content(text)
        
        # predict using offensive model
        off_pred = off_dictionary[predict_off(text,offensive_model,device,offensive_tokenizer)]

        if off_pred == 'offensive':
            # predict using racism model
            rac_pred = racism_dict[predict_other(text,racism_model,device,other_tokenizer)]
            # predict using misogyny model
            misog_pred = misogyny_dict[predict_other(text,misogyny_model,device,other_tokenizer)]
            # predict using verbal abuse model
            ver_pred = verbalabuse_dict[predict_other(text,verbalabuse_model,device,other_tokenizer)]
            # predict using dialect model
            dialect_pred = dialect_dict[predict_dialect(text,dialect_model,device,tokenizer_dialect)]
            # predict using religion hate model
            Religion_Hate_pred = religionhate_dict[predict_other(text,religionhate_model,device,other_tokenizer)]
            # return the prediction
            return {"Offensiveness": off_pred, "Dialect": dialect_pred, "Misogyny": misog_pred, "Racism": rac_pred, "Verbal Abuse": ver_pred, "Religion Hate": Religion_Hate_pred}
        
        # predict using misogyny model
        misog_pred = misogyny_dict[predict_other(text,misogyny_model,device,other_tokenizer)]
        # predict using dialect model
        dialect_pred = dialect_dict[predict_dialect(text,dialect_model,device,tokenizer_dialect)]
        
        # return the prediction  as a dataframe row
        return {"Offensiveness": off_pred, "Dialect": dialect_pred, "Misogyny": misog_pred, "Racism": "Not_Racism", "Verbal Abuse": "Not Verbal Abuse", "Religion Hate": "Not Religion Hate"}
###############################################

from geopy.geocoders import Nominatim
import numpy as np
import pandas as pd

geolocator = Nominatim(user_agent="NLP")

def geolocate(country):
    try:
        # Geolocate the center of the country
        loc = geolocator.geocode(country)
        # And return latitude and longitude
        return (loc.latitude, loc.longitude)
    except:
        # Return missing value
        return np.nan

# Stream lit app

st.title("Arabic Hate Speech Detection")

st.write("This app detects hate speech in Arabic dialect text")

st.write("Please enter your text below")


# Session state
if 'Loaded' not in st.session_state:
    st.markdown('### Loading models ...')
    st.session_state['Loaded'] = False
else:
    print('Model already loaded')
    st.session_state['Loaded'] = True
    

if st.session_state['Loaded'] == False:

    # Offensiveness detection model 

    offensive_model = BertClassifier()
    offensive_model.load_state_dict(torch.load(os.path.join(parent_path,'models/modelv3.pt'), map_location=torch.device('cpu')))
    offensive_tokenizer = BertTokenizer.from_pretrained('aubmindlab/bert-base-arabertv02', do_lower_case=True)

    #send model to device

    offensive_model = offensive_model.to(device)
    st.session_state['Offensive_model'] = offensive_model
    st.session_state['Offensive_tokenizer'] = offensive_tokenizer
    print('Offensive model loaded')
    off_dictionary = {1: 'offensive', 0: 'non_offensive'}
    st.session_state['Offensive_dictionary'] = off_dictionary

    ##############################################################################################################################

    # Other four models

    other_tokenizer =  AutoTokenizer.from_pretrained("asafaya/bert-medium-arabic")
    st.session_state['Other_tokenizer'] = other_tokenizer

    racism_model,religionhate_model,verbalabuse_model,misogyny_model = MediumBert(),MediumBert(),MediumBert(),MediumBert()
    ################################################################

    racism_model.load_state_dict(torch.load(os.path.join(parent_path,'models/racism/racism_arabert.pt'), map_location=torch.device('cpu')))
    racism_dict = {0: 'non_racist', 1: 'racist'}

    racism_model = racism_model.to(device)

    st.session_state['Racism_model'] = racism_model
    st.session_state['Racism_dictionary'] = racism_dict

    print('Racism model loaded')
    ################################################################

    religionhate_model.load_state_dict(torch.load(os.path.join(parent_path,'models/religion_hate/religion_hate_params.pt'), map_location=torch.device('cpu')))
    religionhate_dict = {0: 'Religion Hate', 1: 'Not Religion Hate'}

    religionhate_model = religionhate_model.to(device)

    st.session_state['Religion_hate_model'] = religionhate_model
    st.session_state['Religion_hate_dictionary'] = religionhate_dict

    print('Religion Hate model loaded')
    ################################################################

    verbalabuse_model.load_state_dict(torch.load(os.path.join(parent_path,'models/verbal_abuse/verbal_abuse_arabert.pt'), map_location=torch.device('cpu')))
    verbalabuse_dict = {0: 'Verbal Abuse', 1: 'Not Verbal Abuse'}

    verbalabuse_model=verbalabuse_model.to(device)

    st.session_state['Verbal_abuse_model'] = verbalabuse_model
    st.session_state['Verbal_abuse_dictionary'] = verbalabuse_dict

    print('Verbal Abuse model loaded')
    ################################################################

    misogyny_model.load_state_dict(torch.load(os.path.join(parent_path,'models/misogyny/misogyny.pt'), map_location=torch.device('cpu')))
    misogyny_dict = {0: 'misogyny', 1: 'non_misogyny'}

    misogyny_model=misogyny_model.to(device)

    st.session_state['Misogyny_model'] = misogyny_model
    st.session_state['Misogyny_dictionary'] = misogyny_dict


    print('Misogyny model loaded')
    ################################################################

    # Dialect detection model

    dialect_model = Dialect_Detection(10)
    dialect_model.load_state_dict(torch.load(os.path.join(parent_path,'models/dialect_classifier.pt'), map_location=torch.device('cpu')))

    dialect_model = dialect_model.to(device)

    st.session_state['Dialect_model'] = dialect_model

    print('Dialect model loaded')

    tokenizer_dialect = BertTokenizerFast.from_pretrained('alger-ia/dziribert')

    st.session_state['Dialect_tokenizer'] = tokenizer_dialect

    # load the model
    dialect_dict = {0: 'lebanon', 1: 'egypt', 2: 'morocco', 3: 'tunisia', 4: 'algeria', 5: 'qatar', 6: 'iraq', 7: 'saudi arabia', 8: 'libya', 9: 'jordan'}

    st.session_state['Dialect_dictionary'] = dialect_dict

    st.session_state['Loaded'] = True

text = st.text_area("Enter Text")

if st.button("Predict") and text != '':
    result = predict(text = text, device = device,
                    offensive_model= st.session_state['Offensive_model'],
                    offensive_tokenizer= st.session_state['Offensive_tokenizer'],
                    racism_model= st.session_state['Racism_model'],
                    misogyny_model=st.session_state['Misogyny_model'],
                    verbalabuse_model= st.session_state['Verbal_abuse_model'],
                    dialect_model=st.session_state['Dialect_model'],
                    religionhate_model=st.session_state['Religion_hate_model'],
                    tokenizer_dialect=st.session_state['Dialect_tokenizer'],
                    other_tokenizer=st.session_state['Other_tokenizer'],
                    off_dictionary=st.session_state['Offensive_dictionary'],
                    racism_dict=st.session_state['Racism_dictionary'],
                    misogyny_dict=st.session_state['Misogyny_dictionary'],
                    verbalabuse_dict=st.session_state['Verbal_abuse_dictionary'],
                    dialect_dict=st.session_state['Dialect_dictionary'],
                    religionhate_dict=st.session_state['Religion_hate_dictionary'])

    st.write(result)

    location  = geolocate(result['Dialect'])

    # map with contry highlited
    location = pd.DataFrame({'lat': [location[0]], 'lon': [location[1]]})
    st.map(data= location , zoom=5)
    
elif text == '':
    st.write('Please enter text to predict')