from firebase_admin import credentials from firebase_admin import firestore import firebase_admin from firebase_admin import db cred = credentials.Certificate( "text-to-emotions-firebase-adminsdk-8isbn-dffbdf01e8.json") firebase_admin.initialize_app(cred) # Import the Firebase Admin SDK import threading import firebase_admin from datetime import datetime import time import Cleaning from collections import Counter import models from datetime import datetime import numpy as np import pytz date_format = '%d-%m-%Y %H:%M:%S' import Classes as Classes db = firestore.client() # Firebase ininlaziton # for doc in docs: # print(f"{doc.id} => {doc.to_dict()}") all_reviews = db.collection("complaints") # Create a query against the collection documents_to_summarize = all_reviews.get() feedbacks = db.collection("feedbacks").where("feedbacks","!=",'').get() allfeedbacks=[] emotioncount={"anger":0,"sadness":0,"joy":0,"surprise":0,"love":0,"sympathy":0,"fear":0,"count":0} documents=[] documnetskey=set() #get all documents for today that have no summary # def get_all_document(): # for doc in documents_to_summarize: # document =Classes.Shakwa.from_dict(source=doc.to_dict()) # documents.append(document) # return documents def get_num_of_words(): for doc in documents_to_summarize: print(len(doc.complaintbody)) def shakwa_common_words(): allcomplaints = [] most_common_words = {} global documents for document in documents: allcomplaints.append(document.complaintbody) documents.clear() # print(allcomplaints) words_in_docs = " ".join(allcomplaints) words_in_docs = Cleaning.txt_preprocess(words_in_docs) words_in_docs = words_in_docs.split(" ") for word in words_in_docs: if word in most_common_words: most_common_words[word] += 1 else: most_common_words[word] = 1 most_common_words = sorted(most_common_words.items(), key=lambda x: x[1]) tz = pytz.timezone('EET') db.collection("per_day_common_words").document(str(datetime.now(tz).date())).set(dict(most_common_words)) def feedback_common_words(): allfeedbacks= [] most_common_words = {} for doc in feedbacks: document = Classes.Feedback.from_dict(source=doc.to_dict()) allcomplaints.append(document.feedback ) # print(allcomplaints) words_in_docs = " ".join(allfeedbacks) words_in_docs = Cleaning.txt_preprocess(words_in_docs) words_in_docs = words_in_docs.split(" ") for word in words_in_docs: if word in most_common_words: most_common_words[word] += 1 else: most_common_words[word] = 1 most_common_words = sorted(most_common_words.items(), key=lambda x: x[1]) return dict(most_common_words) def get_most_common_places(): dic_place_count = {} for doc in all_reviews.get(): document = Classes.Shakwa.from_dict(source=doc.to_dict()) if document.governorate not in dic_place_count.keys(): dic_place_count[document.governorate] = 1 else: dic_place_count[document.governorate] += 1 return dic_place_count queuedUnSummurizedShawkas = [] queuedUnLabeledFeedbacks=[] semphoreShakwas=threading.Semaphore(0) semphoreFeedback=threading.Semaphore(0) def summrizedshakwas(): global queuedUnSummurizedShawkas global semphoreShakwas global db while True: semphoreShakwas.acquire() shawka=queuedUnSummurizedShawkas.pop(0) tmpdict= models.modelsummary(shawka.complaintbody) shawka.summary=tmpdict['summary'] db.collection("complaints").document(shawka.id).update({"summary":shawka.summary}) def LabelingFeedback(): global queuedUnLabeledFeedbacks global semphoreFeedback global db while True: semphoreFeedback.acquire() feedback=queuedUnLabeledFeedbacks.pop(0) tmpdict=models.modelpredict(feedback.feedback) feedback.label=tmpdict["label"] db.collection("feedbacks").document(feedback.id).update(tmpdict) thread = threading.Thread(target=summrizedshakwas) thread.start() threadlabelingfeedback= threading.Thread(target=LabelingFeedback) threadlabelingfeedback.start() #lithening to changes of documnts callback_done = threading.Event() callback_donefeedback=threading.Event() def on_snapshot(doc_snapshot, changes, read_time): global queuedUnSummurizedShawkas global semphoreShakwas global documnetskey global documents tz = pytz.timezone('EET') for doc in doc_snapshot: # print(doc.to_dict()) shakw = Classes.Shakwa.from_dict(source=doc.to_dict()) datetime_obj = datetime.strptime(shakw.date, date_format) if datetime_obj.date() == datetime.now(tz).date() and shakw.id not in documnetskey: documents.append(shakw) documnetskey.add(shakw.id) if shakw.summary==None: queuedUnSummurizedShawkas.append(shakw) semphoreShakwas.release() threadofcommonwords = threading.Thread(target=shakwa_common_words) threadofcommonwords.start() callback_done.set() def on_snapshotfeedback(doc_snapshot, changes, read_time): global queuedUnLabeledFeedbacks global semphoreFeedback global allfeedbacks for doc in doc_snapshot: feedback=Classes.Feedback.from_dict(source=doc.to_dict()) if(feedback.label==None): queuedUnLabeledFeedbacks.append(feedback) semphoreFeedback.release() elif feedback not in allfeedbacks: allfeedbacks.append(feedback) emotioncount[feedback.label]+=1 emotioncount["count"]+=1 callback_donefeedback.set() docwatch= db.collection("complaints").on_snapshot(on_snapshot,) docwatch2=db.collection("feedbacks") .on_snapshot(on_snapshotfeedback,)