Update news_extractor/news_extractor.py

news_extractor/news_extractor.py

@@ -102,7 +102,14 @@ def news_agg(rss):
             rss_df = pd.concat([rss_df, rss_parser(i)], axis=0)
         rss_df.reset_index(drop=True, inplace=True)
         rss_df["description"] = rss_df["description"].replace([" NULL", ''], np.nan)
-        rss_df.dropna(inplace=True)
+
+
+        #### UNCOMMENT IN CASE OF OOM ERROR IN RENDER
+        # rss_df.dropna(inplace=True)
+
+        ####
+
+        
         rss_df["src"] = src_parse(rss)
         rss_df["elapsed_time"] = rss_df["parsed_date"].apply(date_time_parser)
         rss_df["parsed_date"] = rss_df["parsed_date"].astype("str")
@@ -143,7 +150,14 @@ def get_news_rss(url):
     # final_df['src_time'] = final_df['src'] + (" " * 5) + final_df["elapsed_time_str"]
     # final_df.drop(columns=['date', 'parsed_date', 'src', 'elapsed_time', 'elapsed_time_str'], inplace=True)
     final_df.drop(columns=['elapsed_time'], inplace=True)
-    final_df.drop_duplicates(subset='description', inplace=True)
+
+
+    #### UNCOMMENT 1ST AND REMOVE 2ND STATEMENT IN CASE OF OOM ERROR IN RENDER
+    # final_df.drop_duplicates(subset='description', inplace=True)
+    final_df.drop_duplicates(subset='url', inplace=True)
+    
+    ####
+    
     final_df = final_df.loc[(final_df["title"] != ""), :].copy()
 
     final_df.loc[(final_df['description'].isna()) | (final_df['description']=='')| (final_df['description']==' '), 'description'] = final_df.loc[(final_df['description'].isna()) | (final_df['description']=='')| (final_df['description']==' '), 'title']