Update news_category_similar_news_prediction.py

news_category_similar_news_prediction.py

@@ -10,6 +10,39 @@ from find_similar_news import find_similar_news
 
 logger = get_logger()
 
+##########################
+
+from dateutil import parser
+def correct_date(x):
+    if (not isinstance(x, str)) or (str(x).find(":") == -1):
+        return "2020-11-07 00:36:44+05:30"
+    return x
+
+def date_time_parser(dt):
+    """
+    Computes the minutes elapsed since published time.
+    :param dt: date
+    :return: int, minutes elapsed.
+    """
+    try:
+        return int(np.round((dt.now(dt.tz) - dt).total_seconds() / 60, 0))
+    except:
+        return 100000
+
+def delete_outdated_news(final_df: pd.DataFrame):
+    final_df = final_df.copy()
+    final_df["parsed_date"] = final_df["parsed_date"].map(correct_date)
+    final_df["parsed_date"] = final_df["parsed_date"].map(parser.parse)
+    final_df["elapsed_time"] = final_df["parsed_date"].apply(date_time_parser)
+    final_df = final_df.loc[final_df["elapsed_time"] <= 720, :].copy() # 1440=24 hrs and 720=12 hrs
+    final_df.drop(columns='elapsed_time', inplace=True)
+    return final_df
+
+
+
+############################################
+
+
 def parse_prediction(tflite_pred, label_encoder):
     tflite_pred_argmax = np.argmax(tflite_pred, axis=1)
     tflite_pred_label = label_encoder.inverse_transform(tflite_pred_argmax)
@@ -53,12 +86,14 @@ def predict_news_category_similar_news(old_news: pd.DataFrame, new_news: pd.Data
         if not isinstance(new_news, pd.DataFrame):
             raise Exception('No New News Found')
         else:
-            new_news = new_news.copy()
+            # new_news = new_news.copy()
+            new_news = delete_outdated_news(new_news)
             logger.warning(f'new news columns: {[*new_news.columns]}')
             logger.warning(f'{len(new_news)} new news items found')
         
         if isinstance(old_news, pd.DataFrame):
-            old_news = old_news.copy()
+            # old_news = old_news.copy()
+            old_news = delete_outdated_news(old_news)
             logger.warning(f'old news columns: {[*old_news.columns]}')
             logger.warning(f'{len(old_news)} old news items found')
             old_news.drop(columns='_id', inplace=True)