Update news_category_similar_news_prediction.py

news_category_similar_news_prediction.py

@@ -130,10 +130,10 @@ def predict_news_category_similar_news(old_news: pd.DataFrame, new_news: pd.Data
             final_df.drop_duplicates(subset='url', keep='first', inplace=True)
 
             
-            headlines = [*final_df['title']].copy()
+            headlines = [*final_df['title'].fillna("").str.strip()]
+            descriptions = [*final_df['description'].fillna("").str.strip()]
             # label, prob = inference(headlines, interpreter, label_encoder, tokenizer)
-
-            headlines_desc = [*final_df['title'] + ". " + final_df['description']].copy()
+            headlines_desc = [h if (h == d) else f"{h}. {d}" for h, d in zip(headlines, descriptions)]
             label, prob = inference(headlines_desc, interpreter, label_encoder, tokenizer)
 
             
@@ -154,14 +154,14 @@ def predict_news_category_similar_news(old_news: pd.DataFrame, new_news: pd.Data
             new_news = new_news.loc[new_news['url'].isin(old_urls) == False, :]
             if len(new_news) > 0:
                 
-                
-                headlines = [*new_news['title']].copy()
 
-                headlines_desc = [*new_news['title'] + ". " + new_news['description']].copy()
+                headlines = [*new_news['title'].fillna("").str.strip()]
+                descriptions = [*new_news['description'].fillna("").str.strip()]
+                # label, prob = inference(headlines, interpreter, label_encoder, tokenizer)
+                headlines_desc = [h if (h == d) else f"{h}. {d}" for h, d in zip(headlines, descriptions)]
                 label, prob = inference(headlines_desc, interpreter, label_encoder, tokenizer)
 
             
-                # label, prob = inference(headlines, interpreter, label_encoder, tokenizer)
                 sent_embs = vectorizer.vectorize_(headlines, sent_model)
                 sim_news = [find_similar_news(text, search_vec, collection, vectorizer, sent_model, ce_model) for search_vec, text in zip(sent_embs, headlines)]
                 new_news['category'] = label