Check_1

feature_pipeline.ipynb

@@ -0,0 +1,535 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "import os "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "            1. open  2. high    3. low  4. close    5. volume\n",
+      "date                                                         \n",
+      "2024-04-26   168.85   172.12  166.3700    168.29  109815725.0\n",
+      "2024-04-25   158.96   170.88  158.3600    170.18  126427521.0\n",
+      "2024-04-24   162.84   167.97  157.5100    162.13  181178020.0\n",
+      "2024-04-23   143.33   147.26  141.1100    144.68  124545104.0\n",
+      "2024-04-22   140.56   144.44  138.8025    142.05  107097564.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from alpha_vantage.timeseries import TimeSeries\n",
+    "import pandas as pd\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "api_key = os.environ.get('stocks_api') # Replace this with your actual API key\n",
+    "ts = TimeSeries(key=api_key, output_format='pandas')\n",
+    "\n",
+    "# Fetch daily adjusted stock prices; adjust the symbol as needed\n",
+    "data, meta_data = ts.get_daily(symbol='TSLA', outputsize='full')\n",
+    "\n",
+    "print(data.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Feature Group created successfully, explore it at \n",
+      "https://c.app.hopsworks.ai:443/p/549016/fs/544838/fg/752979\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "394c6ab7da624ed388df0b9b8bff469a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Uploading Dataframe: 0.00% |          | Rows 0/3479 | Elapsed Time: 00:00 | Remaining Time: ?"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Launching job: tsla_stock_1_offline_fg_materialization\n",
+      "Job started successfully, you can follow the progress at \n",
+      "https://c.app.hopsworks.ai/p/549016/jobs/named/tsla_stock_1_offline_fg_materialization/executions\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(<hsfs.core.job.Job at 0x158c80fd0>, None)"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "DatetimeIndex: 3479 entries, 2024-04-24 to 2010-06-29\n",
+      "Data columns (total 5 columns):\n",
+      " #   Column     Non-Null Count  Dtype  \n",
+      "---  ------     --------------  -----  \n",
+      " 0   1. open    3479 non-null   float64\n",
+      " 1   2. high    3479 non-null   float64\n",
+      " 2   3. low     3479 non-null   float64\n",
+      " 3   4. close   3479 non-null   float64\n",
+      " 4   5. volume  3479 non-null   float64\n",
+      "dtypes: float64(5)\n",
+      "memory usage: 163.1 KB\n"
+     ]
+    }
+   ],
+   "source": [
+    "data.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'1. Information': 'Daily Prices (open, high, low, close) and Volumes',\n",
+       " '2. Symbol': 'TSLA',\n",
+       " '3. Last Refreshed': '2024-04-24',\n",
+       " '4. Output Size': 'Full size',\n",
+       " '5. Time Zone': 'US/Eastern'}"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "meta_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data saved to /Users/manos/Documents/BDS/MLops_mod/TSLA_stock_price.csv\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define your file path and name\n",
+    "file_path = '/Users/manos/Documents/BDS/MLops_mod/TSLA_stock_price.csv'  # Customize the path and filename\n",
+    "\n",
+    "# Save the DataFrame to CSV\n",
+    "data.to_csv(file_path)\n",
+    "\n",
+    "print(f\"Data saved to {file_path}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "         date  1. open  2. high    3. low  4. close    5. volume\n",
+      "0  2024-04-24   162.84   167.97  157.5100    162.13  181178020.0\n",
+      "1  2024-04-23   143.33   147.26  141.1100    144.68  124545104.0\n",
+      "2  2024-04-22   140.56   144.44  138.8025    142.05  107097564.0\n",
+      "3  2024-04-19   148.97   150.94  146.2200    147.05   87074500.0\n",
+      "4  2024-04-18   151.25   152.20  148.7000    149.93   96098830.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load and display the data from CSV to confirm\n",
+    "tsla_df = pd.read_csv(file_path)\n",
+    "print(tsla_df.head())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Connected. Call `.close()` to terminate connection gracefully.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "UserWarning: The installed hopsworks client version 3.4.4 may not be compatible with the connected Hopsworks backend version 3.7.1. \n",
+      "To ensure compatibility please install the latest bug fix release matching the minor version of your backend (3.7) by running 'pip install hopsworks==3.7.*'\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/549016\n",
+      "Connected. Call `.close()` to terminate connection gracefully.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import hopsworks\n",
+    "\n",
+    "project = hopsworks.login()\n",
+    "fs = project.get_feature_store()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clean_column_name(name):\n",
+    "    # Remove all non-letter characters\n",
+    "    cleaned_name = re.sub(r'[^a-zA-Z]', '', name)\n",
+    "    return cleaned_name\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>1. open</th>\n",
+       "      <th>2. high</th>\n",
+       "      <th>3. low</th>\n",
+       "      <th>4. close</th>\n",
+       "      <th>5. volume</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2024-04-24</td>\n",
+       "      <td>162.84</td>\n",
+       "      <td>167.9700</td>\n",
+       "      <td>157.5100</td>\n",
+       "      <td>162.13</td>\n",
+       "      <td>181178020.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2024-04-23</td>\n",
+       "      <td>143.33</td>\n",
+       "      <td>147.2600</td>\n",
+       "      <td>141.1100</td>\n",
+       "      <td>144.68</td>\n",
+       "      <td>124545104.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2024-04-22</td>\n",
+       "      <td>140.56</td>\n",
+       "      <td>144.4400</td>\n",
+       "      <td>138.8025</td>\n",
+       "      <td>142.05</td>\n",
+       "      <td>107097564.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2024-04-19</td>\n",
+       "      <td>148.97</td>\n",
+       "      <td>150.9400</td>\n",
+       "      <td>146.2200</td>\n",
+       "      <td>147.05</td>\n",
+       "      <td>87074500.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2024-04-18</td>\n",
+       "      <td>151.25</td>\n",
+       "      <td>152.2000</td>\n",
+       "      <td>148.7000</td>\n",
+       "      <td>149.93</td>\n",
+       "      <td>96098830.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3474</th>\n",
+       "      <td>2010-07-06</td>\n",
+       "      <td>20.00</td>\n",
+       "      <td>20.0000</td>\n",
+       "      <td>15.8300</td>\n",
+       "      <td>16.11</td>\n",
+       "      <td>6866900.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3475</th>\n",
+       "      <td>2010-07-02</td>\n",
+       "      <td>23.00</td>\n",
+       "      <td>23.1000</td>\n",
+       "      <td>18.7100</td>\n",
+       "      <td>19.20</td>\n",
+       "      <td>5139800.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3476</th>\n",
+       "      <td>2010-07-01</td>\n",
+       "      <td>25.00</td>\n",
+       "      <td>25.9200</td>\n",
+       "      <td>20.2700</td>\n",
+       "      <td>21.96</td>\n",
+       "      <td>8218800.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3477</th>\n",
+       "      <td>2010-06-30</td>\n",
+       "      <td>25.79</td>\n",
+       "      <td>30.4192</td>\n",
+       "      <td>23.3000</td>\n",
+       "      <td>23.83</td>\n",
+       "      <td>17187100.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3478</th>\n",
+       "      <td>2010-06-29</td>\n",
+       "      <td>19.00</td>\n",
+       "      <td>25.0000</td>\n",
+       "      <td>17.5400</td>\n",
+       "      <td>23.89</td>\n",
+       "      <td>18766300.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>3479 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            date  1. open   2. high    3. low  4. close    5. volume\n",
+       "0     2024-04-24   162.84  167.9700  157.5100    162.13  181178020.0\n",
+       "1     2024-04-23   143.33  147.2600  141.1100    144.68  124545104.0\n",
+       "2     2024-04-22   140.56  144.4400  138.8025    142.05  107097564.0\n",
+       "3     2024-04-19   148.97  150.9400  146.2200    147.05   87074500.0\n",
+       "4     2024-04-18   151.25  152.2000  148.7000    149.93   96098830.0\n",
+       "...          ...      ...       ...       ...       ...          ...\n",
+       "3474  2010-07-06    20.00   20.0000   15.8300     16.11    6866900.0\n",
+       "3475  2010-07-02    23.00   23.1000   18.7100     19.20    5139800.0\n",
+       "3476  2010-07-01    25.00   25.9200   20.2700     21.96    8218800.0\n",
+       "3477  2010-06-30    25.79   30.4192   23.3000     23.83   17187100.0\n",
+       "3478  2010-06-29    19.00   25.0000   17.5400     23.89   18766300.0\n",
+       "\n",
+       "[3479 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tsla_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Assuming 'tsla_df' is your DataFrame\n",
+    "tsla_df.columns = [clean_column_name(col) for col in tsla_df.columns]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Index(['date', 'open', 'high', 'low', 'close', 'volume'], dtype='object')\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tsla_df.columns)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define a feature group\n",
+    "tesla_fg = fs.get_or_create_feature_group(\n",
+    "    name=\"tsla_stock\",\n",
+    "    description=\"Tesla stock dataset from alpha vantage\",\n",
+    "    version=1,\n",
+    "    primary_key=[\"date\"],\n",
+    "    online_enabled=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "91ef74ded4714a1492bdc24b176c4f1e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Uploading Dataframe: 0.00% |          | Rows 0/3479 | Elapsed Time: 00:00 | Remaining Time: ?"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Launching job: tsla_stock_1_offline_fg_materialization\n",
+      "Job started successfully, you can follow the progress at \n",
+      "https://c.app.hopsworks.ai/p/549016/jobs/named/tsla_stock_1_offline_fg_materialization/executions\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(<hsfs.core.job.Job at 0x177b01510>, None)"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tesla_fg.insert(tsla_df, write_options={\"wait_for_job\" : False})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

feature_preprocessing.ipynb

@@ -0,0 +1,116 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from calendar import monthrange\n",
+    "from feature_engineering import *\n",
+    "import glob\n",
+    "import pandas as pd\n",
+    "from dotenv import load_dotenv\n",
+    "import os\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "# Set the API endpoint and your API key\n",
+    "endpoint = \"https://api.marketaux.com/v1/news/all?symbols=TSLA&filter_entities=true&published_after=2021&language=en&api_token=iy6rRX4oxFrouZocXr8JNpOzaxZLk3UvMfoMGxYs\"\n",
+    "api_key = os.environ.get('news_api')\n",
+    "\n",
+    "# Set the ticker symbol\n",
+    "ticker = \"TSLA\" #TSLA"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def getNews_historical(api_key,endpoint,ticker,year,month,num=1000):\n",
+    "  \n",
+    "    for start,end in zip([1,15],[16,monthrange(year, month)[1]]):\n",
+    "    \n",
+    "        from_date = '{}-{:02d}-{:02d}'.format(year,month,start)\n",
+    "        to_date = '{}-{:02d}-{:02d}'.format(year,month,end)\n",
+    "        \n",
+    "        print('Grabbing News data between {}-{}'.format(from_date,to_date))    \n",
+    "        news = getNews(api_key,endpoint,ticker,from_date,to_date)\n",
+    "        \n",
+    "        print('Number of articles: ',len(news.index))\n",
+    "        news.head(n=num)\n",
+    "\n",
+    "        # Store the dataframe as a CSV file\n",
+    "        news.to_csv(\"/Users/manos/Documents/BDS/MLops_mod/TSLA_news_{}_to_{}.csv\".format(from_date,to_date))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'meta': {'found': 58203, 'returned': 3, 'limit': 3, 'page': 1}, 'data': [{'uuid': 'a2f5f0e0-937a-4333-9aa7-da32fb0ede1f', 'title': \"What's next for Big Tech? See what SA analysts have to say\", 'description': 'Technology stocks have dropped over the past couple of weeks. See what SA analysts have to say about the overall state of tech and the economy.', 'keywords': '', 'snippet': 'Technology stocks have dropped over the past couple of weeks, and it was further seen with the selloff in Meta Platforms (META) and weak GDP data, as the two ac...', 'url': 'https://seekingalpha.com/news/4094186-tech-stocks-dive-see-what-sa-analysts-have-to-say', 'image_url': 'https://static.seekingalpha.com/cdn/s3/uploads/getty_images/184997191/image_184997191.jpg?io=getty-c-w750', 'language': 'en', 'published_at': '2024-04-26T12:20:54.000000Z', 'source': 'seekingalpha.com', 'relevance_score': None, 'entities': [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'exchange': None, 'exchange_long': None, 'country': 'us', 'type': 'equity', 'industry': 'Consumer Cyclical', 'match_score': 11.309888, 'sentiment_score': 0, 'highlights': [{'highlight': '<em>Tesla</em> (TSLA) -3.5% .\\n\\nTech ETFs', 'sentiment': 0, 'highlighted_in': 'main_text'}]}], 'similar': []}, {'uuid': '650adf2f-d62f-478d-9322-05d3e7d7532d', 'title': 'Stellantis And Tesla: Combine These Stocks For The Ultimate Automotive Portfolio (STLA)', 'description': 'Tesla and Stellantis are two automakers that complement each other. Find out why I see both STLA and TSLA stocks as currently undervalued.', 'keywords': '', 'snippet': 'Tramino/iStock Unreleased via Getty Images\\n\\nStellantis N.V. (NYSE:STLA) and Tesla, Inc. (TSLA) are two very distinct automakers that, in my view, perfectly comp...', 'url': 'https://seekingalpha.com/article/4686610-stellantis-tesla-combine-these-stocks-for-ultimate-automotive-portfolio', 'image_url': 'https://static.seekingalpha.com/cdn/s3/uploads/getty_images/1305717707/image_1305717707.jpg?io=getty-c-w1536', 'language': 'en', 'published_at': '2024-04-26T10:58:06.000000Z', 'source': 'seekingalpha.com', 'relevance_score': None, 'entities': [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'exchange': None, 'exchange_long': None, 'country': 'us', 'type': 'equity', 'industry': 'Consumer Cyclical', 'match_score': 22.866589, 'sentiment_score': 0.173982, 'highlights': [{'highlight': '(NYSE:STLA) and <em>Tesla</em>, <em>Inc</em>. (TSLA) are two very distinct automakers that, in my view, perfectly complement each other. By entering a 50/50 balanced position in the two companies, you can create an “artificial” automaker in your portfolio that is bound to dominate the industry and provide superior returns for shareholders.', 'sentiment': 0.8519, 'highlighted_in': 'main_text'}, {'highlight': 'The brands of “TESSA” include:\\n\\n<em>Tesla</em>, the leading global EV brand and #1 most valuable car brand in the world. Because of Tesla’s aggressive price policy lately, I believe it makes almost no economic sense to buy an EV that is not a <em>Tesla</em>, for the majority of consumers. More on this shortly.', 'sentiment': 0.2089, 'highlighted_in': 'main_text'}, {'highlight': 'The two overall car brand portfolios encompass all market segments\\n\\nGoing beyond EVs, I see “TESSA’s” car portfolio to cover all segments, again because of the complementarity of <em>Tesla</em> and Stellantis. The below chart outlines how all car segments are covered by either <em>Tesla</em> or Stellantis.', 'sentiment': 0, 'highlighted_in': 'main_text'}, {'highlight': 'To be fair, both Stellantis and <em>Tesla</em> margins declined in 2023, and in the case of <em>Tesla</em>, the company just reported that margins are now down to 5.5% after Q1 price cuts.\\n\\nHowever, I believe that Tesla’s margins at the moment do not tell the full story.', 'sentiment': 0.0258, 'highlighted_in': 'main_text'}, {'highlight': 'It is precisely because it enjoyed a 25%+ operating margin back in 2021 that <em>Tesla</em> was able to grow its company with aggressive pricing in the past 2 years.\\n\\nToday, for the majority of use cases, I believe buying an EV that is not a <em>Tesla</em> does not make rational sense.', 'sentiment': 0.4019, 'highlighted_in': 'main_text'}, {'highlight': 'These are cars that have starting prices that are significantly higher than <em>Tesla</em>, but with worse reviews, worse technology and limited access to Tesla’s SuperCharger system. Even EV-native car brands, such as Rivian and Polestar, have difficulty in competing with <em>Tesla</em>.', 'sentiment': -0.9294, 'highlighted_in': 'main_text'}, {'highlight': 'A Rivian R2 starts at $45,000, which is almost $7,000 more than the base <em>Tesla</em> Model 3.\\n\\nI believe that <em>Tesla</em> is using its margins to grow the EV category, converting ICE consumers, and simultaneously gain the monster share of that growing market.', 'sentiment': 0.743, 'highlighted_in': 'main_text'}, {'highlight': \"Key Financial Metrics for <em>Tesla</em>, Q1 24 (Tesla's Q1 Shareholders Presentation)\\n\\nKey Financial Metrics for Stellantis, Q1 24 (Stellantis' Q1 Shareholder Presentation)\\n\\nThis financial data tells the same story: <em>Tesla</em> and Stellantis complement each other.\", 'sentiment': 0, 'highlighted_in': 'main_text'}, {'highlight': 'VOO since 2021 (Seeking Alpha)\\n\\nWhat matters for my thesis is that <em>Tesla</em> and Stellantis are complementary in how they reward shareholders and how the market prices their stocks. <em>Tesla</em> is a tech company, looking at the long term, and shareholders need to be patient to see returns.', 'sentiment': 0.5859, 'highlighted_in': 'main_text'}, {'highlight': 'In that case, <em>Tesla</em> might generate returns significantly higher than Stellantis, to the point that it would have seemed silly to “dilute” a <em>Tesla</em> investment with another stock.', 'sentiment': 0.0258, 'highlighted_in': 'main_text'}, {'highlight': 'Stellantis And <em>Tesla</em>: Combine These Stocks For The Ultimate Automotive Portfolio (STLA)', 'sentiment': 0, 'highlighted_in': 'title'}]}], 'similar': []}, {'uuid': '47a58bd4-3a8d-40fe-8a89-934d0d695ea4', 'title': 'Tesla is being investigated by the NHTSA for Autopilot software fix (NASDAQ:TSLA)', 'description': \"The National Highway Traffic Safety Administration is investigating whether Tesla's recall of 2 million vehicles for Autopilot safeguards is sufficient.\", 'keywords': '', 'snippet': \"The National Highway Traffic Safety Administration confirmed on Friday that the safety regulator has opened an investigation into whether Tesla's (NASDAQ:TSLA) ...\", 'url': 'https://seekingalpha.com/news/4094754-tesla-is-being-investigated-by-the-nhtsa-for-autopilot-software-fix', 'image_url': 'https://static.seekingalpha.com/cdn/s3/uploads/getty_images/1415090444/image_1415090444.jpg?io=getty-c-w750', 'language': 'en', 'published_at': '2024-04-26T10:50:20.000000Z', 'source': 'seekingalpha.com', 'relevance_score': None, 'entities': [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'exchange': None, 'exchange_long': None, 'country': 'us', 'type': 'equity', 'industry': 'Consumer Cyclical', 'match_score': 51.845444, 'sentiment_score': 0.42985, 'highlights': [{'highlight': \"The National Highway Traffic Safety Administration confirmed on Friday that the safety regulator has opened an investigation into whether Tesla's (<em>NASDAQ:TSLA</em>) recall of more than 2 million vehicles announced in December to install new Autopilot safeguards is adequate.\", 'sentiment': 0.836, 'highlighted_in': 'main_text'}, {'highlight': \"While <em>Tesla</em> has released software updates to address potential issues, NHTSA cited Tesla's statement that a portion of the remedy both requires the owner to opt in and allows a driver to readily reverse it.\", 'sentiment': 0, 'highlighted_in': 'main_text'}, {'highlight': 'In December, <em>Tesla</em> (TSLA) said its largest-ever recall was to better ensure drivers pay attention when using its advanced driver assistance system.\\n\\nShares of <em>Tesla</em> (TSLA) rose 1.17% in premarket trading on Friday to $172.17. The EV stock is down 31.51% on a year-to-date basis. Short interest stands at 3.84% of the total float.', 'sentiment': 0.8834, 'highlighted_in': 'main_text'}, {'highlight': '<em>Tesla</em> is being investigated by the NHTSA for Autopilot software fix (<em>NASDAQ:TSLA</em>)', 'sentiment': 0, 'highlighted_in': 'title'}]}], 'similar': [{'uuid': 'b269d18a-6ea0-4554-a20e-047c623513f9', 'title': 'US probes Tesla recall of 2 million vehicles over Autopilot, citing concerns By Reuters', 'description': 'US probes Tesla recall of 2 million vehicles over Autopilot, citing concerns', 'keywords': '', 'snippet': \"WASHINGTON (Reuters) - U.S. auto safety regulators said Friday they have opened an investigation into whether Tesla (NASDAQ: )'s recall of more than 2 million v...\", 'url': 'https://www.investing.com/news/stock-market-news/us-probes-tesla-recall-of-2-million-vehicles-over-autopilot-citing-concerns-3400236', 'image_url': 'https://i-invdn-com.investing.com/news/moved_LYNXMPEJ580NE_L.jpg', 'language': 'en', 'published_at': '2024-04-26T09:51:10.000000Z', 'source': 'investing.com', 'relevance_score': None, 'entities': [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'exchange': None, 'exchange_long': None, 'country': 'us', 'type': 'equity', 'industry': 'Consumer Cyclical', 'match_score': 25.2132, 'sentiment_score': 0.432933, 'highlights': [{'highlight': \"WASHINGTON (Reuters) - U.S. auto safety regulators said Friday they have opened an investigation into whether <em>Tesla</em> (NASDAQ: )'s recall of more than 2 million vehicles announced in December to install new Autopilot safeguards is adequate.\", 'sentiment': 0.7269, 'highlighted_in': 'main_text'}, {'highlight': 'The agency said <em>Tesla</em> has issued software updates to address issues that appear related to its concerns but has not made them \"a part of the recall or otherwise determined to remedy a defect that poses an unreasonable safety risk.\"', 'sentiment': 0.5719, 'highlighted_in': 'main_text'}, {'highlight': 'US probes <em>Tesla</em> recall of 2 million vehicles over Autopilot, citing concerns By Reuters', 'sentiment': 0, 'highlighted_in': 'title'}]}]}]}]}\n"
+     ]
+    }
+   ],
+   "source": [
+    "response = requests.get(endpoint)\n",
+    "data = response.json()\n",
+    "print(data)  # See what the data looks like\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Grabbing News data between 2022-01-01-2022-01-16\n"
+     ]
+    },
+    {
+     "ename": "ValueError",
+     "evalue": "All arrays must be of the same length",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "\u001b[1;32m/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb Cell 4\u001b[0m line \u001b[0;36m4\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39mfor\u001b[39;00m year \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m2022\u001b[39m,\u001b[39m2023\u001b[39m):\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m     \u001b[39mfor\u001b[39;00m month \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m1\u001b[39m,\u001b[39m13\u001b[39m):\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m         getNews_historical(api_key,endpoint,ticker,year,month)\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m         \u001b[39mif\u001b[39;00m year \u001b[39m==\u001b[39m \u001b[39m2023\u001b[39m \u001b[39mand\u001b[39;00m month \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m             \u001b[39mbreak\u001b[39;00m\n",
+      "\u001b[1;32m/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb Cell 4\u001b[0m line \u001b[0;36m9\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m to_date \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m-\u001b[39m\u001b[39m{:02d}\u001b[39;00m\u001b[39m-\u001b[39m\u001b[39m{:02d}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mformat(year,month,end)\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mGrabbing News data between \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m-\u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mformat(from_date,to_date))    \n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m news \u001b[39m=\u001b[39m getNews(api_key,endpoint,ticker,from_date,to_date)\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mNumber of articles: \u001b[39m\u001b[39m'\u001b[39m,\u001b[39mlen\u001b[39m(news\u001b[39m.\u001b[39mindex))\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/manos/Documents/BDS/MLops_mod/feature_preprocessing.ipynb#X14sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m news\u001b[39m.\u001b[39mhead(n\u001b[39m=\u001b[39mnum)\n",
+      "File \u001b[0;32m~/Documents/BDS/MLops_mod/feature_engineering.py:27\u001b[0m, in \u001b[0;36mgetNews\u001b[0;34m(api_key, endpoint, ticker, from_date, to_date, num)\u001b[0m\n\u001b[1;32m     21\u001b[0m response \u001b[39m=\u001b[39m requests\u001b[39m.\u001b[39mget(endpoint, params\u001b[39m=\u001b[39mparams)\n\u001b[1;32m     23\u001b[0m \u001b[39m# Print the response from the API\u001b[39;00m\n\u001b[1;32m     24\u001b[0m \u001b[39m#print(response.json())\u001b[39;00m\n\u001b[1;32m     25\u001b[0m \n\u001b[1;32m     26\u001b[0m \u001b[39m#Return a Pandas dataframe from the response\u001b[39;00m\n\u001b[0;32m---> 27\u001b[0m \u001b[39mreturn\u001b[39;00m pd\u001b[39m.\u001b[39mDataFrame(response\u001b[39m.\u001b[39mjson())\n",
+      "File \u001b[0;32m/Applications/anaconda3/lib/python3.11/site-packages/pandas/core/frame.py:662\u001b[0m, in \u001b[0;36mDataFrame.__init__\u001b[0;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[1;32m    656\u001b[0m     mgr \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_init_mgr(\n\u001b[1;32m    657\u001b[0m         data, axes\u001b[39m=\u001b[39m{\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m: index, \u001b[39m\"\u001b[39m\u001b[39mcolumns\u001b[39m\u001b[39m\"\u001b[39m: columns}, dtype\u001b[39m=\u001b[39mdtype, copy\u001b[39m=\u001b[39mcopy\n\u001b[1;32m    658\u001b[0m     )\n\u001b[1;32m    660\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, \u001b[39mdict\u001b[39m):\n\u001b[1;32m    661\u001b[0m     \u001b[39m# GH#38939 de facto copy defaults to False only in non-dict cases\u001b[39;00m\n\u001b[0;32m--> 662\u001b[0m     mgr \u001b[39m=\u001b[39m dict_to_mgr(data, index, columns, dtype\u001b[39m=\u001b[39mdtype, copy\u001b[39m=\u001b[39mcopy, typ\u001b[39m=\u001b[39mmanager)\n\u001b[1;32m    663\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, ma\u001b[39m.\u001b[39mMaskedArray):\n\u001b[1;32m    664\u001b[0m     \u001b[39mimport\u001b[39;00m \u001b[39mnumpy\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mma\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mmrecords\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mmrecords\u001b[39;00m\n",
+      "File \u001b[0;32m/Applications/anaconda3/lib/python3.11/site-packages/pandas/core/internals/construction.py:493\u001b[0m, in \u001b[0;36mdict_to_mgr\u001b[0;34m(data, index, columns, dtype, typ, copy)\u001b[0m\n\u001b[1;32m    489\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    490\u001b[0m         \u001b[39m# dtype check to exclude e.g. range objects, scalars\u001b[39;00m\n\u001b[1;32m    491\u001b[0m         arrays \u001b[39m=\u001b[39m [x\u001b[39m.\u001b[39mcopy() \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(x, \u001b[39m\"\u001b[39m\u001b[39mdtype\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39melse\u001b[39;00m x \u001b[39mfor\u001b[39;00m x \u001b[39min\u001b[39;00m arrays]\n\u001b[0;32m--> 493\u001b[0m \u001b[39mreturn\u001b[39;00m arrays_to_mgr(arrays, columns, index, dtype\u001b[39m=\u001b[39mdtype, typ\u001b[39m=\u001b[39mtyp, consolidate\u001b[39m=\u001b[39mcopy)\n",
+      "File \u001b[0;32m/Applications/anaconda3/lib/python3.11/site-packages/pandas/core/internals/construction.py:118\u001b[0m, in \u001b[0;36marrays_to_mgr\u001b[0;34m(arrays, columns, index, dtype, verify_integrity, typ, consolidate)\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[39mif\u001b[39;00m verify_integrity:\n\u001b[1;32m    116\u001b[0m     \u001b[39m# figure out the index, if necessary\u001b[39;00m\n\u001b[1;32m    117\u001b[0m     \u001b[39mif\u001b[39;00m index \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 118\u001b[0m         index \u001b[39m=\u001b[39m _extract_index(arrays)\n\u001b[1;32m    119\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    120\u001b[0m         index \u001b[39m=\u001b[39m ensure_index(index)\n",
+      "File \u001b[0;32m/Applications/anaconda3/lib/python3.11/site-packages/pandas/core/internals/construction.py:666\u001b[0m, in \u001b[0;36m_extract_index\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m    664\u001b[0m lengths \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(\u001b[39mset\u001b[39m(raw_lengths))\n\u001b[1;32m    665\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(lengths) \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m--> 666\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mAll arrays must be of the same length\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    668\u001b[0m \u001b[39mif\u001b[39;00m have_dicts:\n\u001b[1;32m    669\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m    670\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mMixing dicts with non-Series may lead to ambiguous ordering.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    671\u001b[0m     )\n",
+      "\u001b[0;31mValueError\u001b[0m: All arrays must be of the same length"
+     ]
+    }
+   ],
+   "source": [
+    "# Grab old data\n",
+    "for year in range(2022,2023):\n",
+    "    for month in range(1,13):\n",
+    "        getNews_historical(api_key,endpoint,ticker,year,month)\n",
+    "        if year == 2023 and month == 1:\n",
+    "            break"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

news_experimenting.ipynb

@@ -14,10 +14,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 41,
    "metadata": {},
    "outputs": [],
    "source": [
+    "from dotenv import load_dotenv\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "load_dotenv()\n",
+    "\n",
     "def fetch_tesla_news(api_key, start_date, end_date):\n",
     "    url = \"https://api.marketaux.com/v1/news/all?symbols=TSLA&filter_entities=true&language=en&api_token=iy6rRX4oxFrouZocXr8JNpOzaxZLk3UvMfoMGxYs\"  # Modify this based on the exact endpoint you need\n",
     "    headers = {\n",
@@ -43,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 44,
    "metadata": {},
    "outputs": [
     {
@@ -51,44 +63,39 @@
      "output_type": "stream",
      "text": [
       "                                   uuid  \\\n",
-      "0  daf76e3e-caea-4c92-a461-6b3132655788   \n",
-      "1  8dab10ca-5b23-465a-aa86-360bc987a774   \n",
-      "2  b8c381b9-4187-433e-ad15-cecc9d227b13   \n",
+      "0  99ce54fb-eb5b-4c9d-be66-6a56aa9f6de3   \n",
+      "1  9286f959-f5ea-4b95-88e8-7ba99a472ca5   \n",
+      "2  92e41393-f923-4c98-ba38-5bab89fedabd   \n",
       "\n",
       "                                               title  \\\n",
-      "0  Stock market today: US futures climb as earnin...   \n",
-      "1  5 things to know before the stock market opens...   \n",
-      "2  Wall Street Breakfast Podcast: UNH: Personal D...   \n",
-      "\n",
-      "                                         description  \\\n",
-      "0  The wait for Tesla results is on as investors ...   \n",
-      "1  Here are the most important news items that in...   \n",
-      "2  UnitedHealth confirms personal data compromise...   \n",
+      "0  S&P 500 gains as investors digest positive ear...   \n",
+      "1  3 Defensive Stocks to Protect Your Portfolio i...   \n",
+      "2  Tesla to lay off more than 3,000 employees in ...   \n",
       "\n",
-      "                                            keywords  \\\n",
-      "0                                                      \n",
-      "1  Investment strategy, Economy, Markets, Busines...   \n",
-      "2                                                      \n",
+      "                                         description keywords  \\\n",
+      "0  Benchmark S&P 500 rose on\\nTuesday following p...  Markets   \n",
+      "1  Stocks Analysis by The Tokenist (Timothy Fries...            \n",
+      "2  Tesla is set to cut 3,332\\njobs in California,...  Markets   \n",
       "\n",
       "                                             snippet  \\\n",
-      "0  US stocks climbed on Tuesday, on track for fur...   \n",
-      "1  In this article CPRI Follow your favorite stoc...   \n",
-      "2  JHVEPhoto/iStock Editorial via Getty Images\\n\\...   \n",
+      "0  * Tesla set to kick off Magnificent Seven earn...   \n",
+      "1  After the Iran-Israel situation fizzled out fo...   \n",
+      "2  Tesla, Inc. designs, builds, and sells electri...   \n",
       "\n",
       "                                                 url  \\\n",
-      "0  https://finance.yahoo.com/news/stock-market-to...   \n",
-      "1  https://www.cnbc.com/2024/04/23/5-things-to-kn...   \n",
-      "2  https://seekingalpha.com/article/4685243-wall-...   \n",
+      "0  https://www.marketscreener.com/quote/stock/TES...   \n",
+      "1  https://www.investing.com/analysis/3-defensive...   \n",
+      "2  https://www.marketscreener.com/quote/stock/TES...   \n",
       "\n",
       "                                           image_url language  \\\n",
-      "0  https://s.yimg.com/ny/api/res/1.2/mqjC0VUO61dY...       en   \n",
-      "1  https://image.cnbcfm.com/api/v1/image/10692170...       en   \n",
-      "2  https://static.seekingalpha.com/cdn/s3/uploads...       en   \n",
+      "0  https://img.zonebourse.com/reuters/2016-04/201...       en   \n",
+      "1  https://i-invdn-com.investing.com/redesign/ima...       en   \n",
+      "2  https://www.marketscreener.com/images/reuters/...       en   \n",
       "\n",
-      "                  published_at             source relevance_score  \\\n",
-      "0  2024-04-23T11:22:53.000000Z  finance.yahoo.com            None   \n",
-      "1  2024-04-23T11:16:00.000000Z           cnbc.com            None   \n",
-      "2  2024-04-23T11:00:00.000000Z   seekingalpha.com            None   \n",
+      "                  published_at              source relevance_score  \\\n",
+      "0  2024-04-23T19:10:43.000000Z  marketscreener.com            None   \n",
+      "1  2024-04-23T19:06:00.000000Z       investing.com            None   \n",
+      "2  2024-04-23T18:50:54.000000Z  marketscreener.com            None   \n",
       "\n",
       "                                            entities  \\\n",
       "0  [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'ex...   \n",
@@ -96,24 +103,172 @@
       "2  [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'ex...   \n",
       "\n",
       "                                             similar  \n",
-      "0  [{'uuid': '01cd65de-530d-407a-96c9-5b3359e98a0...  \n",
+      "0                                                 []  \n",
       "1                                                 []  \n",
-      "2                                                 []  \n"
+      "2  [{'uuid': 'df8f1cf3-89ca-4430-975a-131b9c31245...  \n"
      ]
     }
    ],
    "source": [
-    "api_key = \"iy6rRX4oxFrouZocXr8JNpOzaxZLk3UvMfoMGxYs\"\n",
+    "api_key = os.environ.get('news_api')\n",
     "tesla_news_df = fetch_tesla_news(api_key, \"2017-01-01\", datetime.now().strftime('%Y-%m-%d'))\n",
     "print(tesla_news_df.head())"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 40,
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>uuid</th>\n",
+       "      <th>title</th>\n",
+       "      <th>description</th>\n",
+       "      <th>keywords</th>\n",
+       "      <th>snippet</th>\n",
+       "      <th>url</th>\n",
+       "      <th>image_url</th>\n",
+       "      <th>language</th>\n",
+       "      <th>published_at</th>\n",
+       "      <th>source</th>\n",
+       "      <th>relevance_score</th>\n",
+       "      <th>entities</th>\n",
+       "      <th>similar</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>daf76e3e-caea-4c92-a461-6b3132655788</td>\n",
+       "      <td>Stock market today: US futures climb as earnin...</td>\n",
+       "      <td>The wait for Tesla results is on as investors ...</td>\n",
+       "      <td></td>\n",
+       "      <td>US stocks climbed on Tuesday, on track for fur...</td>\n",
+       "      <td>https://finance.yahoo.com/news/stock-market-to...</td>\n",
+       "      <td>https://s.yimg.com/ny/api/res/1.2/mqjC0VUO61dY...</td>\n",
+       "      <td>en</td>\n",
+       "      <td>2024-04-23T11:22:53.000000Z</td>\n",
+       "      <td>finance.yahoo.com</td>\n",
+       "      <td>None</td>\n",
+       "      <td>[{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'ex...</td>\n",
+       "      <td>[{'uuid': '01cd65de-530d-407a-96c9-5b3359e98a0...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>8dab10ca-5b23-465a-aa86-360bc987a774</td>\n",
+       "      <td>5 things to know before the stock market opens...</td>\n",
+       "      <td>Here are the most important news items that in...</td>\n",
+       "      <td>Investment strategy, Economy, Markets, Busines...</td>\n",
+       "      <td>In this article CPRI Follow your favorite stoc...</td>\n",
+       "      <td>https://www.cnbc.com/2024/04/23/5-things-to-kn...</td>\n",
+       "      <td>https://image.cnbcfm.com/api/v1/image/10692170...</td>\n",
+       "      <td>en</td>\n",
+       "      <td>2024-04-23T11:16:00.000000Z</td>\n",
+       "      <td>cnbc.com</td>\n",
+       "      <td>None</td>\n",
+       "      <td>[{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'ex...</td>\n",
+       "      <td>[]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>b8c381b9-4187-433e-ad15-cecc9d227b13</td>\n",
+       "      <td>Wall Street Breakfast Podcast: UNH: Personal D...</td>\n",
+       "      <td>UnitedHealth confirms personal data compromise...</td>\n",
+       "      <td></td>\n",
+       "      <td>JHVEPhoto/iStock Editorial via Getty Images\\n\\...</td>\n",
+       "      <td>https://seekingalpha.com/article/4685243-wall-...</td>\n",
+       "      <td>https://static.seekingalpha.com/cdn/s3/uploads...</td>\n",
+       "      <td>en</td>\n",
+       "      <td>2024-04-23T11:00:00.000000Z</td>\n",
+       "      <td>seekingalpha.com</td>\n",
+       "      <td>None</td>\n",
+       "      <td>[{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'ex...</td>\n",
+       "      <td>[]</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                   uuid  \\\n",
+       "0  daf76e3e-caea-4c92-a461-6b3132655788   \n",
+       "1  8dab10ca-5b23-465a-aa86-360bc987a774   \n",
+       "2  b8c381b9-4187-433e-ad15-cecc9d227b13   \n",
+       "\n",
+       "                                               title  \\\n",
+       "0  Stock market today: US futures climb as earnin...   \n",
+       "1  5 things to know before the stock market opens...   \n",
+       "2  Wall Street Breakfast Podcast: UNH: Personal D...   \n",
+       "\n",
+       "                                         description  \\\n",
+       "0  The wait for Tesla results is on as investors ...   \n",
+       "1  Here are the most important news items that in...   \n",
+       "2  UnitedHealth confirms personal data compromise...   \n",
+       "\n",
+       "                                            keywords  \\\n",
+       "0                                                      \n",
+       "1  Investment strategy, Economy, Markets, Busines...   \n",
+       "2                                                      \n",
+       "\n",
+       "                                             snippet  \\\n",
+       "0  US stocks climbed on Tuesday, on track for fur...   \n",
+       "1  In this article CPRI Follow your favorite stoc...   \n",
+       "2  JHVEPhoto/iStock Editorial via Getty Images\\n\\...   \n",
+       "\n",
+       "                                                 url  \\\n",
+       "0  https://finance.yahoo.com/news/stock-market-to...   \n",
+       "1  https://www.cnbc.com/2024/04/23/5-things-to-kn...   \n",
+       "2  https://seekingalpha.com/article/4685243-wall-...   \n",
+       "\n",
+       "                                           image_url language  \\\n",
+       "0  https://s.yimg.com/ny/api/res/1.2/mqjC0VUO61dY...       en   \n",
+       "1  https://image.cnbcfm.com/api/v1/image/10692170...       en   \n",
+       "2  https://static.seekingalpha.com/cdn/s3/uploads...       en   \n",
+       "\n",
+       "                  published_at             source relevance_score  \\\n",
+       "0  2024-04-23T11:22:53.000000Z  finance.yahoo.com            None   \n",
+       "1  2024-04-23T11:16:00.000000Z           cnbc.com            None   \n",
+       "2  2024-04-23T11:00:00.000000Z   seekingalpha.com            None   \n",
+       "\n",
+       "                                            entities  \\\n",
+       "0  [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'ex...   \n",
+       "1  [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'ex...   \n",
+       "2  [{'symbol': 'TSLA', 'name': 'Tesla, Inc.', 'ex...   \n",
+       "\n",
+       "                                             similar  \n",
+       "0  [{'uuid': '01cd65de-530d-407a-96c9-5b3359e98a0...  \n",
+       "1                                                 []  \n",
+       "2                                                 []  "
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tesla_news_df"
+   ]
   },
   {
    "cell_type": "code",