Upload 7 files

app.py

@@ -0,0 +1,26 @@
+from news_extractor import get_news
+from db_operations import DBOperations
+import json
+from flask import Flask
+from flask_cors import cross_origin, CORS
+
+app = Flask(__name__)
+CORS(app)
+
+
+@app.route("/")
+@cross_origin()
+def update_news():
+    status = "success"
+    try:
+        news_df = get_news()
+        news_json = [*json.loads(news_df.reset_index(drop=True).to_json(orient="index")).values()]
+        db = DBOperations()
+        db.insert_news_into_db(news_json)
+    except:
+        status = "failure"
+    return status
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=5002, timeout=120, workers=3, threads=3)

db_operations/__init__.py

@@ -0,0 +1 @@
+from db_operations.db_operations import *

db_operations/db_operations.py

@@ -0,0 +1,50 @@
+import pymongo
+import os
+
+
+class DBOperations:
+    """
+    Inserts processed news into MongoDB
+    """
+    def __init__(self):
+        self.url = os.getenv('DB_URL')
+        self.database = "rss_news_db"
+        self.collection = "rss_news"
+        self.__client = None
+        self.__error = 0
+
+    def __connect(self):
+        try:
+            self.__client = pymongo.MongoClient(self.url)
+            _ = self.__client.list_database_names()
+        except Exception as conn_exception:
+            self.__error = 1
+            self.__client = None
+            raise
+
+    def __insert(self, documents):
+        try:
+
+            db = self.__client[self.database]
+            coll = db[self.collection]
+            coll.drop()
+            coll.insert_many(documents=documents)
+        except Exception as insert_err:
+            self.__error = 1
+            raise
+
+    def __close_connection(self):
+        if self.__client is not None:
+            self.__client.close()
+            self.__client = None
+
+    def insert_news_into_db(self, documents: list):
+        if self.url is not None:
+            if self.__error == 0:
+                self.__connect()
+            if self.__error == 0:
+                self.__insert(documents=documents)
+            if self.__error == 0:
+                print("Insertion Successful")
+            if self.__client is not None:
+                self.__close_connection()

docker/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.9-slim
+WORKDIR /webapp
+COPY . .
+RUN rm -r ./docker
+RUN pip install --no-cache-dir -r requirements.txt
+CMD ["gunicorn", "app:app", "--timeout", "120", "--workers=3", "--threads=3", "--worker-connections=1000"]
+EXPOSE 5002

news_extractor/__init__.py

@@ -0,0 +1 @@
+from news_extractor.news_extractor import *

news_extractor/news_extractor.py

@@ -0,0 +1,118 @@
+import pandas as pd
+import numpy as np
+from bs4 import BeautifulSoup
+import requests as r
+import regex as re
+from dateutil import parser
+
+
+def date_time_parser(dt):
+    """
+    Computes the minutes elapsed since published time.
+    :param dt: date
+    :return: int, minutes elapsed.
+    """
+    return int(np.round((dt.now(dt.tz) - dt).total_seconds() / 60, 0))
+
+def text_clean(desc):
+    """
+    Cleans the text by removing special chars.
+    :param desc: string containing description
+    :return: str, cleaned description.
+    """
+    desc = desc.replace("&lt;", "<")
+    desc = desc.replace("&gt;", ">")
+    desc = re.sub("<.*?>", "", desc)
+    desc = desc.replace("#39;", "'")
+    desc = desc.replace('&quot;', '"')
+    desc = desc.replace('&nbsp;', ' ')
+    desc = desc.replace('#32;', ' ')
+    return desc
+
+
+def rss_parser(i):
+    """
+    Returns a data frame of parsed news item.
+    :param i: single news item in RSS feed.
+    :return: Data frame of parsed news item.
+    """
+    b1 = BeautifulSoup(str(i), "xml")
+    title = "" if b1.find("title") is None else b1.find("title").get_text()
+    title = text_clean(title)
+    url = "" if b1.find("link") is None else b1.find("link").get_text()
+    desc = "" if b1.find("description") is None else b1.find("description").get_text()
+    desc = text_clean(desc)
+    desc = f'{desc[:300]}...' if len(desc) >= 300 else desc
+    date = "Sat, 12 Aug 2000 13:39:15 +0530" if ((b1.find("pubDate") is "") or (b1.find("pubDate") is None)) else b1.find("pubDate").get_text()
+    if url.find("businesstoday.in") >= 0:
+        date = date.replace("GMT", "+0530")
+    date1 = parser.parse(date)
+    return pd.DataFrame({"title": title,
+                         "url": url,
+                         "description": desc,
+                         "parsed_date": date1}, index=[0])
+
+
+def src_parse(rss):
+    """
+    Returns the root domain name (eg. livemint.com is extracted from www.livemint.com
+    :param rss: RSS URL
+    :return: str, string containing the source name
+    """
+    if rss.find('ndtvprofit') >= 0:
+        rss = 'ndtv profit'
+    rss = rss.replace("https://www.", "")
+    rss = rss.split("/")
+    return rss[0]
+
+
+def news_agg(rss):
+    """
+    Returns feeds from each 'rss' URL.
+    :param rss: RSS URL.
+    :return: Data frame of processed articles.
+    """
+    try:
+        rss_df = pd.DataFrame()
+        resp = r.get(rss, headers={
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
+                          "(KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36"})
+        b = BeautifulSoup(resp.content, "xml")
+        items = b.find_all("item")
+        for i in items:
+            rss_df = rss_df.append(rss_parser(i)).copy()
+        rss_df["description"] = rss_df["description"].replace([" NULL", ''], np.nan)
+        rss_df.dropna(inplace=True)
+        rss_df["src"] = src_parse(rss)
+        rss_df["elapsed_time"] = rss_df["parsed_date"].apply(date_time_parser)
+        rss_df["parsed_date"] = rss_df["parsed_date"].astype("str")
+        # rss_df["elapsed_time_str"] = rss_df["elapsed_time"].apply(elapsed_time_str)
+    except Exception as e:
+        print(e)
+        pass
+    return rss_df
+
+
+# List of RSS feeds
+rss = ['https://www.economictimes.indiatimes.com/rssfeedstopstories.cms',
+       
+       
+       'https://www.moneycontrol.com/rss/latestnews.xml',
+       'https://www.livemint.com/rss/news',
+       
+       'https://www.zeebiz.com/latest.xml/feed',
+       'https://www.timesofindia.indiatimes.com/rssfeedmostrecent.cms']
+
+
+def get_news():
+    final_df = pd.DataFrame()
+    for i in rss:
+        final_df = final_df.append(news_agg(i))
+
+    final_df.sort_values(by="elapsed_time", inplace=True)
+    # final_df['src_time'] = final_df['src'] + ("&nbsp;" * 5) + final_df["elapsed_time_str"]
+    # final_df.drop(columns=['date', 'parsed_date', 'src', 'elapsed_time', 'elapsed_time_str'], inplace=True)
+    final_df.drop(columns=['elapsed_time'], inplace=True)
+    final_df.drop_duplicates(subset='description', inplace=True)
+    final_df = final_df.loc[(final_df["title"] != ""), :].copy()
+    return final_df

requirements.txt

@@ -9,4 +9,4 @@ flask==2.2.2
 flask_cors==3.0.10
 gunicorn==20.1.0
 pymongo==4.3.3
-Werkzeug==2.2.2
+Werkzeug==2.2.2