Spaces:
No application file
No application file
File size: 3,854 Bytes
21b78eb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
from common_module import *
import tomotopy as tp
# Loading NLTK Modules
import nltk
# nltk.download('all')
nltk.download('stopwords')
from nltk.corpus import stopwords
nltk.download("punkt")
class topicModel:
def __init__(self, data_path):
self.topic_models = {}
self.model_path_global = os.path.join(data_path,"models/topicmodels/global")
self.model_path_local = os.path.join(data_path,"models/topicmodels/local")
for i in os.listdir(self.model_path_global):
self.topic_models[int(i.split(".")[0].split("_")[-1])] = [tp.LDAModel.load(os.path.join(self.model_path_global, i)),
tp.LDAModel.load(os.path.join(self.model_path_local, i.replace("global", "local")))]
# perform pre-processing steps using lemmatization, stop-words and unnecessary punctuation removal
def preprocess_article_text(self, doc_article):
"""
Accept pandas series, then:
1. Apply Stop Word removal
"""
# clean
doc_article = doc_article.lower()
# remove stop words
words = nltk.word_tokenize(doc_article)
stop_words = stopwords.words('english')
stop_words = stop_words + ["said", "says", "just", "like", "would", "could", "use", "told", "new", "also", "thats", "even","dont"]
words = [word for word in words if word not in stop_words and len(word) > 3]
doc_article = ' '.join(words)
doc_article = doc_article.replace('\xa0', '')
doc_article = re.sub('[!"#$%&\'()’*+,-./:;<=>?—@[\\]^_`{|}~’]', '', doc_article)
# remove digits
doc_article = re.sub("^\d+\s|\s\d+\s|\s\d+$", " ", doc_article)
return doc_article
def LdaModel_train(self, doc_list):
# k_g is th number of global topics, while k_l is the number of local topics
num_doc = len(doc_list)
mdl = tp.LDAModel(k=5, min_cf= int(num_doc * 0.25), min_df= int(num_doc * 0.33))
for document in doc_list:
mdl.add_doc(document.split())
iterations = 100
for i in range(0, 10000, iterations):
mdl.train(iterations)
print('Iteration: {}\tLog-likelihood: {}'.format(i, mdl.ll_per_word))
result_dict_train = self.extract_topic(mdl)
return result_dict_train, mdl
def extract_topic(self, mdl):
result_dict = {}
topic_dict = {}
extractor = tp.label.PMIExtractor(max_len=5, max_cand=10000)
cands = extractor.extract(mdl)
# ranking the candidates of labels for a specific topic
labeler = tp.label.FoRelevance(mdl, cands, smoothing=1e-2, mu=0.25)
for k in range(mdl.k):
cur_topic = "topic#"+str(k)
result_dict[cur_topic] = {}
result_dict[cur_topic]["labels"] = (', '.join(label for label, score in labeler.get_topic_labels(k, top_n=5)))
result_dict[cur_topic]['topics'] = ' ,'.join([i[0] for i in mdl.get_topic_words(k, top_n=5)])
return result_dict
def LdaModel_predict(self, doc_list, mdl):
pred_result = {}
docs_words = []
for doc in doc_list:
docs_words = docs_words + doc.strip().split()
doc_inst = mdl.make_doc(docs_words)
topic_dist, ll = mdl.infer(doc_inst)
# sort the topic dist and take index
topic_dist_arr = np.array(topic_dist)
topic_dist_idx = topic_dist_arr.argsort()[::-1]
mdl_topic = self.extract_topic(mdl)
idx = 0
for i in topic_dist_idx:
if topic_dist[i]>0:
pred_result["topic#"+str(idx)] = mdl_topic["topic#"+str(i)]
idx+=1
return pred_result
def do_pridict(self, article, clas_label):
topic_result = {}
mdl_global = self.topic_models[int(clas_label)][0]
mdl_local = self.topic_models[int(clas_label)][1]
topic_result['global'] = self.LdaModel_predict(article.tolist(), mdl_global)
topic_result['local'] = self.LdaModel_predict(article.tolist(), mdl_local)
return topic_result
|