lalithadevi
commited on
Commit
•
a78212f
1
Parent(s):
51a30bb
Update news_extractor/news_extractor.py
Browse files
news_extractor/news_extractor.py
CHANGED
@@ -160,7 +160,7 @@ def get_news_rss(url):
|
|
160 |
|
161 |
|
162 |
def get_news_multi_process(urls):
|
163 |
-
logger.warning('Entering get_news_multi_process() to
|
164 |
'''
|
165 |
Get the data shape by parallely calculating lenght of each chunk and
|
166 |
aggregating them to get lenght of complete training dataset
|
@@ -181,6 +181,7 @@ def get_news_multi_process(urls):
|
|
181 |
final_df.reset_index(drop=True, inplace=True)
|
182 |
pool.close()
|
183 |
pool.join()
|
|
|
184 |
logger.warning('Exiting get_news_multi_process()')
|
185 |
|
186 |
if len(final_df) == 0:
|
|
|
160 |
|
161 |
|
162 |
def get_news_multi_process(urls):
|
163 |
+
logger.warning('Entering get_news_multi_process() to extract new news articles')
|
164 |
'''
|
165 |
Get the data shape by parallely calculating lenght of each chunk and
|
166 |
aggregating them to get lenght of complete training dataset
|
|
|
181 |
final_df.reset_index(drop=True, inplace=True)
|
182 |
pool.close()
|
183 |
pool.join()
|
184 |
+
logger.warning(f'Extracted {len(final_df)} new news articles.')
|
185 |
logger.warning('Exiting get_news_multi_process()')
|
186 |
|
187 |
if len(final_df) == 0:
|