File size: 9,378 Bytes
9d4bfa4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d3aaa0b
56cd27c
 
 
6ade7cf
 
 
 
 
1d33810
 
 
9d4bfa4
 
 
 
1d33810
d3aaa0b
 
56cd27c
d3aaa0b
56cd27c
163695f
56cd27c
9d4bfa4
 
 
268dd78
6ade7cf
 
9d4bfa4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d3aaa0b
877dc8a
d3aaa0b
 
 
 
d87c0e6
d3aaa0b
 
 
9d4bfa4
7110d08
95dabf2
 
 
 
6ade7cf
 
 
 
 
 
 
35f2fb6
6ade7cf
f2cee81
 
 
6ade7cf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f2cee81
6ade7cf
 
 
 
1d33810
 
 
 
8fe11ef
 
640c929
b29af5a
c92a43f
b29af5a
 
79bce5c
b29af5a
1daf5e4
79bce5c
b29af5a
1d33810
 
 
 
 
ffecd68
 
1d33810
 
 
 
1daf5e4
1d33810
 
 
9d4bfa4
1d33810
9d4bfa4
 
 
2808cb2
 
 
d3aaa0b
 
 
 
 
 
 
6ade7cf
 
 
 
 
 
 
 
 
 
89d073d
6ade7cf
 
 
 
 
c8ef5d9
6ade7cf
 
2808cb2
9d4bfa4
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
import tensorflow as tf
import numpy as np
import pandas as pd
import swifter
import json
import re
import requests
import time
from keras.preprocessing.text import Tokenizer
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
# from keras.optimizers.optimizer_v2.rmsprop import RMSProp
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Dense, Conv1D, Embedding, MaxPooling1D, GlobalMaxPooling1D, GlobalAveragePooling1D, SpatialDropout1D, LSTM, Dropout, SimpleRNN, Bidirectional, Attention, Activation, GRU, TextVectorization, Input
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.preprocessing.sequence import pad_sequences
import arabicstopwords.arabicstopwords as stp
from nltk.stem.isri import ISRIStemmer
from pyarabic.araby import strip_tashkeel, strip_tatweel
from huggingface_hub import from_pretrained_keras
from collections import Counter
from fastapi import FastAPI, Request, HTTPException
import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore
import threading
# Import the Firebase Admin SDK
import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore
from transformers import BertTokenizer, AutoModelForSeq2SeqLM, pipeline
from arabert.preprocess import ArabertPreprocessor
from transformers import AutoTokenizer, AutoModelForCausalLM
import re
import smtplib
from email.mime.text import MIMEText
import os
import math
import random
import threading
import Classes

# Firebase ininlaziton
cred = credentials.Certificate(
    "text-to-emotions-firebase-adminsdk-8isbn-dffbdf01e8.json")
firebase_admin.initialize_app(cred)
db = firestore.client()
# Model summury
model_name="abdalrahmanshahrour/auto-arabic-summarization"
preprocessor = ArabertPreprocessor(model_name="")

tokenizer = AutoTokenizer.from_pretrained(model_name)
modelsummary =AutoModelForSeq2SeqLM.from_pretrained(model_name)
pipeline1 = pipeline("text2text-generation",model=modelsummary,tokenizer=tokenizer)

# Model inilization
isristemmer = ISRIStemmer()
model = from_pretrained_keras('MahmoudNasser/GRU-MODEL-EMOTION-AR-TEXT-73P')
# dictinarties for email OTP
emailOTP={}


def stemming(txt):
    return isristemmer.stem(txt)


def remove_singleCharacter(text):
    text_tokenized = ar.tokenize(text)
    clean_txt = ''
    for word in text_tokenized:
        if len(word) != 1:
            clean_txt = clean_txt + word + ' '

    return clean_txt[:-1]

# remove_punctuations


def remove_punctuations(text):
    punc = '''()-[]{};:'"\,<>./@#$%^&*،؛_~'''
    arabic_punctuations = '''`÷×؛_ـ،/:".,'~¦+|”…“–ـ=﴾﴿ ﹱ ﹹ ⸀˓• ב'''
    punctuations_list = punc + arabic_punctuations
    for x in punctuations_list:
        text = text.replace(x, ' ')
    return text


def normalize_text(txt):
    txt = strip_tashkeel(txt)
    txt = strip_tatweel(txt)
    txt = ''.join(txt[i] for i in range(len(txt)) if i ==
                  0 or txt[i-1] != txt[i])  # remove repeated characters
    return txt


def remove_stopwords(txt, path="stopword.txt"):
    text_tokenized = txt.split(' ')
    clean_txt = ''
#   useful_words=[]
#   filtered_sentence=" "
    arabic_stop_words_file = open(path, 'r', encoding='utf-8')
    arabic_stop_words = arabic_stop_words_file.read().split('\n')
    for word in text_tokenized:
        if word not in arabic_stop_words:
            clean_txt = clean_txt + word + ' '

    return clean_txt[:-1]


def Remove_unwanted(text):
    # removing the extra spacing and links

    text = re.sub(r'^https?:\/\/.*[\r\n]*', ' ', text, flags=re.MULTILINE)
    text = re.sub(r'^http?:\/\/.*[\r\n]*', ' ', text, flags=re.MULTILINE)
    text = re.sub(r"http\S+", " ", text)
    text = re.sub(r"https\S+", " ", text)
    text = re.sub(r'\s+', ' ', text)
    text = re.sub(r'[a-zA-Z]+', ' ', text)
    text = re.sub(r"^\s+|\s+$", "", text)
    text = re.sub(r"(\s\d+)", " ", text)
    text = re.sub(r"$\d+\W+|\b\d+\b|\W+\d+$", " ", text)
    text = re.sub(r"\d+", " ", text)
    text = re.sub(r'[إأٱآا]', 'ا', text)
    text = re.sub(r'ى', '[ي]', text)
    text = re.sub(r'ء', '[ؤئ]', text)
    text = re.sub(r' +', ' ', text)
    return text


def txt_preprocess(text):
    text = normalize_text(text)
    text = stemming(text)
    text = remove_stopwords(text)
    text = remove_punctuations(text)
    text = Remove_unwanted(text)
    return text


def see_if_thereupdates():
    f = open("updates.txt", "r")
    return f.readline()


def getmodel():
    m = from_pretrained_keras('MahmoudNasser/GRU-MODEL-EMOTION-AR-TEXT-72P')
    return m


def original_values(num):
    if num == 0:
        return 'anger'
    elif num == 1:
        return 'sadness'
    elif num == 2:
        return 'joy'
    elif num == 3:
        return 'surprise'
    elif num == 4:
        return 'love'
    elif num == 5:
        return 'sympathy'
    elif num == 6:
        return 'fear'
def modelsummary(data):
    result = pipeline1(data,
    pad_token_id= tokenizer.eos_token_id,
    num_beams=4,
    repetition_penalty=3.0,
    max_length=600,
    length_penalty=.50,
    no_repeat_ngram_size = 3)[0]['generated_text']
    result = remove_punctuations(result)
    return { 'summary':result}

def modelpredict(data):
  map = {0:'anger', 1:'sadness', 2:'joy', 3:'surprise', 4:'love', 5:'sympathy', 6:'fear'}
  text = txt_preprocess(data)
  pred=model.predict(pd.Series([data])) 
  return map[np.argmax(pred,axis=-1)[0]]
    
#OTP code 
def genereteotp (email):    
  digits = "0123456789"
  OTP = ""
  for i in range(6):
      OTP += digits[math.floor(random.random()*10)]
  emailOTP[email]=OTP
  otp = "your otp is "+OTP
  print("---------------------****---------------------------------------")
  print(email)
  print(otp)
  s = smtplib.SMTP('smtp.gmail.com', 587)
        # start TLS for security
  s.starttls()
  # Authentication
  s.login("youssefmk1214@gmail.com", "lipnacjbsxmjpjxm")
  # message to be sent
  message = otp
  # instance of MIMEText
  msg = MIMEText(message)
  # sender's email address
  msg['From'] = "youssefmk1214@gmail.com"
  # recipient's email address
  msg['To'] = email
  # subject of the email
  msg['Subject'] = " Shakwa Textual OTP"
  print("here to send messege")  
  # send the message via SMTP server
  s.sendmail(msg['From'], msg['To'], msg.as_string())
  # terminate the SMTP session
  s.quit()    
#Threading functions
queuedUnSummurizedShawkas = []
semphoreShakwas=threading.Semaphore(0)
def summrizedshakwas():
   global queuedUnSummurizedShawkas
   global semphoreShakwas
   global db
   while True:  
       semphoreShakwas.acquire()
       shawka=queuedUnSummurizedShawkas.pop(0)
       tmpdict= modelsummary(shawka.complaintbody)
       print(tmpdict)
       shawka.summary=tmpdict['summary']
       db.collection("complaints").document(shawka.id).update({"summary":shawka.summary})
       
thread = threading.Thread(target=summrizedshakwas)
thread.start()
    
#lithening to changes of documnts
callback_done = threading.Event()
def on_snapshot(doc_snapshot, changes, read_time):
    global queuedUnSummurizedShawkas
    global semphoreShakwas
    for doc in doc_snapshot:
        # print(doc.to_dict())
        shakw = Classes.Shakwa.from_dict(source=doc.to_dict())
        shakw.complaintbody
        if shakw.summary==None:
            queuedUnSummurizedShawkas.append(shakw)
            semphoreShakwas.release()
    callback_done.set()

docwatch= db.collection("complaints").on_snapshot(on_snapshot,)    
# Main Server inilization
app = FastAPI()

@app.get("/")
def index():
    return "Hello World"
@app.post("/summary")
async def read_root(request:Request):
    json_data = await request.json()
    if 'text'in json_data:
        return modelsummary(json_data['text'])
    else:
        raise HTTPException(status_code=400, detail="Missing text value")
@app.post("/getOTPCode")
async def read_root(request:Request):
    json_data = await request.json()
    if 'email' in json_data:
        genereteotp(json_data["email"])
        return "message was sent succesufully to "+json_data['email']
    else:
       raise HTTPException(status_code=400, detail="Missing email value")
@app.post("/verifyOTP")
async def read_root(request:Request):
       json_data = await request.json()
       if 'email' in json_data and 'otpcode' in json_data:
            if json_data['otpcode'] ==emailOTP[json_data['email']]  :
                return "OTP verified succesufully "
            else:
                return "OTP Code is wrong "
       else:
           raise HTTPException(status_code=400, detail="Missing email value")
    
@app.post("/predict")
async def read_root(request: Request):
    json_data = await request.json()
    if "mathod" in json_data and json_data["mathod"] == "emotion_predict" and 'text' in json_data:
        return modelpredict(json_data["text"])
    else:
        raise HTTPException(status_code=400, detail="Missing mathod value")
@app.get("/commonwords")
def getcommonwords():
    return {'التسجيل': 23, 'مش': 19, 'تطبيق': 18, 'التطبيق': 18, 'التفعيل': 17, 'كود': 13, 'ارسال': 12, 'تسجيل': 12, 'يتم': 12, 'الرقم': 12}