Update news_category_similar_news_prediction.py

news_category_similar_news_prediction.py

@@ -128,8 +128,15 @@ def predict_news_category_similar_news(old_news: pd.DataFrame, new_news: pd.Data
                 raise Exception("New and old cols don't match")
             final_df = pd.concat([old_news, new_news], axis=0, ignore_index=True)
             final_df.drop_duplicates(subset='url', keep='first', inplace=True)
+
+            
             headlines = [*final_df['title']].copy()
-            label, prob = inference(headlines, interpreter, label_encoder, tokenizer)
+            # label, prob = inference(headlines, interpreter, label_encoder, tokenizer)
+
+            headlines_desc = [*final_df['title'] + ". " + final_df['description']].copy()
+            label, prob = inference(headlines_desc, interpreter, label_encoder, tokenizer)
+
+            
             sent_embs = vectorizer.vectorize_(headlines, sent_model)
             sim_news = [find_similar_news(text, search_vec, collection, vectorizer, sent_model, ce_model) for search_vec, text in zip(sent_embs, headlines)]
             final_df['category'] = label
@@ -146,8 +153,15 @@ def predict_news_category_similar_news(old_news: pd.DataFrame, new_news: pd.Data
             old_urls = [*old_news['url']]
             new_news = new_news.loc[new_news['url'].isin(old_urls) == False, :]
             if len(new_news) > 0:
+                
+                
                 headlines = [*new_news['title']].copy()
-                label, prob = inference(headlines, interpreter, label_encoder, tokenizer)
+
+                headlines_desc = [*final_df['title'] + ". " + final_df['description']].copy()
+                label, prob = inference(headlines_desc, interpreter, label_encoder, tokenizer)
+
+            
+                # label, prob = inference(headlines, interpreter, label_encoder, tokenizer)
                 sent_embs = vectorizer.vectorize_(headlines, sent_model)
                 sim_news = [find_similar_news(text, search_vec, collection, vectorizer, sent_model, ce_model) for search_vec, text in zip(sent_embs, headlines)]
                 new_news['category'] = label