Update news_extractor/news_extractor.py

news_extractor/news_extractor.py

@@ -18,7 +18,10 @@ def date_time_parser(dt):
     :param dt: date
     :return: int, minutes elapsed.
     """
-    return int(np.round((dt.now(dt.tz) - dt).total_seconds() / 60, 0))
+    try:
+        return int(np.round((dt.now(dt.tz) - dt).total_seconds() / 60, 0))
+    except:
+        return 100000
 
 def text_clean(desc):
     """
@@ -26,13 +29,16 @@ def text_clean(desc):
     :param desc: string containing description
     :return: str, cleaned description.
     """
-    desc = desc.replace("&lt;", "<")
-    desc = desc.replace("&gt;", ">")
-    desc = re.sub("<.*?>", "", desc)
-    desc = desc.replace("#39;", "'")
-    desc = desc.replace('&quot;', '"')
-    desc = desc.replace('&nbsp;', ' ')
-    desc = desc.replace('#32;', ' ')
+    try:
+        desc = desc.replace("&lt;", "<")
+        desc = desc.replace("&gt;", ">")
+        desc = re.sub("<.*?>", "", desc)
+        desc = desc.replace("#39;", "'")
+        desc = desc.replace('&quot;', '"')
+        desc = desc.replace('&nbsp;', ' ')
+        desc = desc.replace('#32;', ' ')
+    except:
+        desc = ""
     return desc
 
 
@@ -42,18 +48,22 @@ def rss_parser(i):
     :param i: single news item in RSS feed.
     :return: Data frame of parsed news item.
     """
-    b1 = BeautifulSoup(str(i), "xml")
-    title = "" if b1.find("title") is None else b1.find("title").get_text()
-    title = text_clean(title)
-    url = "" if b1.find("link") is None else b1.find("link").get_text()
-    desc = "" if b1.find("description") is None else b1.find("description").get_text()
-    desc = text_clean(desc)
-    desc = f'{desc[:300]}...' if len(desc) >= 300 else desc
-    date = "Sat, 12 Aug 2000 13:39:15 +05:30" if ((b1.find("pubDate") == "") or (b1.find("pubDate") is None)) else b1.find("pubDate").get_text()
-    if url.find("businesstoday.in") >= 0:
-        date = date.replace("GMT", "+0530")
-    
-    date1 = parser.parse(date)
+    try:
+        b1 = BeautifulSoup(str(i), "xml")
+        title = "" if b1.find("title") is None else b1.find("title").get_text()
+        title = text_clean(title)
+        url = "" if b1.find("link") is None else b1.find("link").get_text()
+        desc = "" if b1.find("description") is None else b1.find("description").get_text()
+        desc = text_clean(desc)
+        desc = f'{desc[:300]}...' if len(desc) >= 300 else desc
+        date = "Sat, 12 Aug 2000 13:39:15 +05:30" if ((b1.find("pubDate") == "") or (b1.find("pubDate") is None)) else b1.find("pubDate").get_text()
+        if url.find("businesstoday.in") >= 0:
+            date = date.replace("GMT", "+0530")
+        
+        date1 = parser.parse(date)
+    except Exception as e:
+        logger.warning(f'Skipping item {i} due to an error {e}')
+        return None
     return pd.DataFrame({"title": title,
                          "url": url,
                          "description": desc,
@@ -100,23 +110,21 @@ def news_agg(rss):
         b = BeautifulSoup(resp.content, "xml")
         items = b.find_all("item")
         for i in items:
-            rss_df = pd.concat([rss_df, rss_parser(i)], axis=0)
+            parsed_item = rss_parser(i)
+            if parsed_item is not None:
+                rss_df = pd.concat([rss_df, parsed_item], axis=0)
         rss_df.reset_index(drop=True, inplace=True)
         rss_df["description"] = rss_df["description"].replace([" NULL", ''], np.nan)
-
-
-        #### UNCOMMENT IN CASE OF OOM ERROR IN RENDER
-        # rss_df.dropna(inplace=True)
-
-        ####
-
         
         rss_df["src"] = src_parse(rss)
         rss_df["elapsed_time"] = rss_df["parsed_date"].apply(date_time_parser)
         rss_df["parsed_date"] = rss_df["parsed_date"].astype("str")
+
+        if len(rss_df) == 0:
+            rss_df = None
     except Exception as e:
-        logger.warning(f'Skipping {rss} due to an error {e}')
-        pass
+        logger.warning(f'Skipping {rss} feed extraction due to an error {e}')
+        return None
     return rss_df
 
 
@@ -125,25 +133,28 @@ rss = RSS_FEEDS_TO_EXTRACT
 
 
 def get_news_rss(url):
-    final_df = news_agg(url)
-    final_df.reset_index(drop=True, inplace=True)
-
-    final_df.sort_values(by="elapsed_time", inplace=True)
-    final_df.drop(columns=['elapsed_time'], inplace=True)
-
-
-    #### UNCOMMENT 1ST STATEMENT AND REMOVE 2ND STATEMENT IN CASE OF OOM ERROR IN RENDER
-    # final_df.drop_duplicates(subset='description', inplace=True)
-    final_df.drop_duplicates(subset='url', inplace=True)
-    
-    ####
-    
-    final_df = final_df.loc[(final_df["title"] != ""), :].copy()
+    '''
+    Function that is used in multiprocessing
+    '''
+    try:
+        final_df = news_agg(url)
+        if final_df is not None:
+            final_df.reset_index(drop=True, inplace=True)
+        
+            final_df.sort_values(by="elapsed_time", inplace=True)
+            final_df.drop(columns=['elapsed_time'], inplace=True)
 
-    final_df.loc[(final_df['description'].isna()) | (final_df['description']=='')| (final_df['description']==' '), 'description'] = final_df.loc[(final_df['description'].isna()) | (final_df['description']=='')| (final_df['description']==' '), 'title']
-    
+            final_df.drop_duplicates(subset='url', inplace=True)
+            
+            final_df = final_df.loc[(final_df["title"] != ""), :].copy()
+        
+            final_df.loc[(final_df['description'].isna()) | (final_df['description']=='')| (final_df['description']==' '), 'description'] = final_df.loc[(final_df['description'].isna()) | (final_df['description']=='')| (final_df['description']==' '), 'title']
+    except Exception as e:
+        logger.warning(f'Skipping {url} feed processing due to an error {e}')
+        return None
     return final_df
 
+
 def get_news_multi_process(urls):
     logger.warning('Entering get_news_multi_process() to read news')
     '''
@@ -159,13 +170,18 @@ def get_news_multi_process(urls):
         
     final_df = pd.DataFrame()
     for f in results:
-        final_df = pd.concat([final_df, f.get(timeout=120)], axis=0) # getting output of each parallel job
+        rss_df = f.get(timeout=120)
+        if rss_df is not None:
+            final_df = pd.concat([final_df, rss_df], axis=0) # getting output of each parallel job
     
     final_df.reset_index(drop=True, inplace=True)
-    logging.warning(final_df['src'].unique())
     pool.close()
     pool.join()
     logger.warning('Exiting get_news_multi_process()')
+
+    if len(final_df) == 0:
+        final_df = None
+        
     return final_df