import feedparser from pathlib import Path import json from db import Database from apscheduler.schedulers.background import BackgroundScheduler from datetime import datetime scheduler = BackgroundScheduler() database = Database(Path("./")) def get_feed(feed_url): feed = feedparser.parse(feed_url) return {"entries": feed["entries"]} def cache_news(): print("Caching news") data = [] for feed in TOP_NEWS_FEEDS: url = feed["url"] label = feed["label"] print(f"Fetching {label} from {url}") try: feed = get_feed(url) data.append((url, label, json.dumps(feed))) except Exception as e: print(f"Failed to fetch {label} from {url}: {e}") database.insert(data) scheduler.add_job( cache_news, "interval", hours=1, id="cache_news", next_run_time=datetime.now() ) TOP_NEWS_FEEDS = [ {"label": "BBC World News", "url": "http://feeds.bbci.co.uk/news/world/rss.xml"}, { "label": "Reddit World News", "url": "https://www.reddit.com/r/worldnews/top/.rss", }, {"label": "Vox", "url": "http://www.vox.com/rss/index.xml"}, {"label": "CBS News", "url": "https://www.cbsnews.com/latest/rss/main"}, {"label": "ABC News", "url": "http://abcnews.go.com/abcnews/topstories"}, {"label": "CNN Top Stories", "url": "http://rss.cnn.com/rss/cnn_topstories.rss"}, {"label": "CNN World News", "url": "http://rss.cnn.com/rss/cnn_world.rss"}, { "label": "The New York Times", "url": "http://www.nytimes.com/services/xml/rss/nyt/HomePage.xml", }, { "label": "The Economist", "url": "http://www.economist.com/sections/business-finance/rss.xml", }, {"label": "The Guardian", "url": "https://www.theguardian.com/international/rss"}, {"label": "NPR", "url": "http://www.npr.org/rss/rss.php?id=1001"}, {"label": "Al Jazeera", "url": "https://www.aljazeera.com/xml/rss/all.xml"}, { "label": "The Guardian World News", "url": "https://www.theguardian.com/world/rss", }, {"label": "The Atlantic", "url": "https://www.theatlantic.com/feed/all/"}, {"label": "Vice", "url": "http://www.vice.com/rss"}, { "label": "The New York Times", "url": "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml", }, { "label": "The New Yorker", "url": "http://www.newyorker.com/services/rss/feeds/everything.xml", }, {"label": "Pew Research Center", "url": "http://www.pewresearch.org/feed/"}, {"label": "Fox News", "url": "http://feeds.feedburner.com/foxnews/latest"}, { "label": "The Washington Post", "url": "http://feeds.washingtonpost.com/rss/world", }, {"label": "The Guardian UK", "url": "https://www.theguardian.com/uk/rss"}, {"label": "TIME", "url": "http://rss.time.com/web/time/rss/top/index.xml"}, { "label": "The New York Times", "url": "http://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml", }, {"label": "NPR", "url": "https://feeds.npr.org/1001/rss.xml"}, {"label": "Fortune", "url": "http://fortune.com/feed/"}, {"label": "Fox News", "url": "http://feeds.foxnews.com/foxnews/latest"}, { "label": "BBC World News", "url": "http://newsrss.bbc.co.uk/rss/newsonline_world_edition/front_page/rss.xml", }, {"label": "Al Jazeera", "url": "http://www.aljazeera.com/xml/rss/all.xml"}, {"label": "Le Monde", "url": "https://www.lemonde.fr/rss/une.xml"}, {"label": "Vox", "url": "https://www.vox.com/rss/index.xml"}, { "label": "The New York Times", "url": "http://rss.nytimes.com/services/xml/rss/nyt/World.xml", }, {"label": "The Guardian US", "url": "https://www.theguardian.com/us/rss"}, {"label": "ProPublica", "url": "http://feeds.propublica.org/propublica/main"}, {"label": "The Washington Post", "url": "https://feedx.net/rss/washingtonpost.xml"}, {"label": "Axios", "url": "https://api.axios.com/feed/top/"}, {"label": "RT", "url": "https://www.rt.com/rss/"}, {"label": "ABC News US", "url": "http://feeds.abcnews.com/abcnews/usheadlines"}, {"label": "CNN US", "url": "http://rss.cnn.com/rss/cnn_topstories.rss"}, {"label": "CBS News", "url": "http://www.cbsnews.com/latest/rss/main"}, { "label": "The Wall Street Journal", "url": "http://online.wsj.com/xml/rss/3_7085.xml", }, { "label": "USA Today", "url": "http://content.usatoday.com/marketing/rss/rsstrans.aspx?feedId=news2", }, { "label": "The Christian Science Monitor", "url": "http://rss.csmonitor.com/feeds/usa", }, { "label": "NBC News Top Stories", "url": "http://feeds.nbcnews.com/feeds/topstories", }, {"label": "NBC News World News", "url": "http://feeds.nbcnews.com/feeds/worldnews"}, { "label": "Reuters World News", "url": "http://feeds.reuters.com/Reuters/worldNews", }, { "label": "Reuters US News", "url": "http://feeds.reuters.com/Reuters/domesticNews", }, { "label": "Associated Press US Headlines", "url": "http://hosted.ap.org/lineups/USHEADS.rss", }, { "label": "Associated Press World Headlines", "url": "http://hosted.ap.org/lineups/WORLDHEADS.rss", }, { "label": "HuffPost World News", "url": "http://www.huffingtonpost.com/feeds/verticals/world/index.xml", }, { "label": "BBC News US and Canada", "url": "http://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml", }, {"label": "Yahoo News US", "url": "http://news.yahoo.com/rss/us"}, {"label": "Yahoo News World", "url": "http://rss.news.yahoo.com/rss/world"}, {"label": "Newsweek", "url": "http://www.newsweek.com/rss"}, { "label": "The Daily Beast", "url": "http://feeds.feedburner.com/thedailybeast/articles", }, {"label": "Quartz", "url": "http://qz.com/feed"}, {"label": "The Guardian USA", "url": "http://www.theguardian.com/world/usa/rss"}, {"label": "Politico", "url": "http://www.politico.com/rss/politicopicks.xml"}, {"label": "The New Yorker News", "url": "http://www.newyorker.com/feed/news"}, {"label": "PBS NewsHour", "url": "http://feeds.feedburner.com/NationPBSNewsHour"}, {"label": "PBS NewsHour World", "url": "http://feeds.feedburner.com/NewshourWorld"}, {"label": "NPR Politics", "url": "http://www.npr.org/rss/rss.php?id=1003"}, {"label": "NPR World", "url": "http://www.npr.org/rss/rss.php?id=1004"}, { "label": "The Atlantic National", "url": "http://feeds.feedburner.com/AtlanticNational", }, { "label": "The Atlantic Wire", "url": "http://feeds.feedburner.com/TheAtlanticWire", }, { "label": "Los Angeles Times US", "url": "http://www.latimes.com/nation/rss2.0.xml", }, { "label": "Los Angeles Times World", "url": "http://www.latimes.com/world/rss2.0.xml", }, { "label": "Breaking News", "url": "http://api.breakingnews.com/api/v1/item/?format=rss", }, {"label": "VICE News", "url": "https://news.vice.com/rss"}, { "label": "Talking Points Memo", "url": "http://talkingpointsmemo.com/feed/livewire", }, {"label": "TIME Newsfeed", "url": "http://time.com/newsfeed/feed/"}, {"label": "Fox News", "url": "http://feeds.foxnews.com/foxnews/latest?format=xml"}, {"label": "Mashable US & World", "url": "http://mashable.com/us-world/rss/"}, ]