Update news_extractor/news_extractor.py

news_extractor/news_extractor.py

@@ -5,6 +5,7 @@ import requests as r
 import regex as re
 from dateutil import parser
 import logging
+import multiprocessing
 
 
 def date_time_parser(dt):
@@ -128,11 +129,12 @@ rss = ['https://www.economictimes.indiatimes.com/rssfeedstopstories.cms',
        'https://www.timesofindia.indiatimes.com/rssfeedmostrecent.cms']
 
 
-def get_news():
-    final_df = pd.DataFrame()
-    for i in rss:
-        # final_df = final_df.append(news_agg(i))
-        final_df = pd.concat([final_df, news_agg(i)], axis=0)
+def get_news_rss(url):
+    # final_df = pd.DataFrame()
+    # for i in rss:
+    #     # final_df = final_df.append(news_agg(i))
+    #     final_df = pd.concat([final_df, news_agg(i)], axis=0)
+    final_df = news_agg(url)
     final_df.reset_index(drop=True, inplace=True)
 
     logging.warning(final_df['src'].unique())
@@ -147,3 +149,29 @@ def get_news():
     final_df.loc[(final_df['description'].isna()) | (final_df['description']=='')| (final_df['description']==' '), 'description'] = final_df.loc[(final_df['description'].isna()) | (final_df['description']=='')| (final_df['description']==' '), 'title']
     
     return final_df
+
+def get_news_multi_process(urls):
+    '''
+    Get the data shape by parallely calculating lenght of each chunk and 
+    aggregating them to get lenght of complete training dataset
+    '''
+    pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
+    
+    results = []
+    for url in urls:
+        f = pool.apply_async(get_news, [url]) # asynchronously applying function to chunk. Each worker parallely begins to work on the job
+        results.append(f) # appending result to results
+        
+    final_df = pd.DataFrame()
+    for f in results:
+#         print(f.get())
+        final_df = pd.concat([final_df, f.get(timeout=120)], axis=0) # getting output of each parallel job
+    
+    final_df.reset_index(drop=True, inplace=True)
+    pool.close()
+    pool.join()
+    return final_df
+
+
+def get_news():
+   return get_data(rss)