feb8 meeting update

.gitignore

@@ -1,3 +1,4 @@
 secrets.json
 edgar-crawler/
-.venv/
+.venv/
+.env

app.py

@@ -35,13 +35,8 @@ conn.close()
 df = pd.DataFrame(rows, columns=column_names)
 
 # setup pygwalker configuration: https://github.com/Kanaries/pygwalker, https://docs.kanaries.net/pygwalker/use-pygwalker-with-streamlit.en
-#pyg_html = pyg.to_html(df, dark="dark")
 pyg_html = pyg.walk(df, dark = 'dark', return_html=True)
 
 components.html(pyg_html, height=1000, scrolling=True)
 
-# show the dataframe just to test
-st.dataframe(df)
-
-
 

pages/1_Fundamentals.py

@@ -1,4 +1,4 @@
-from pages.scraped_data import get_alpha_vantage_data, get_finnhub_data
+from scraped_data import get_alpha_vantage_data, get_finnhub_data
 import datetime
 import pandas as pd
 import streamlit as st
@@ -13,9 +13,9 @@ def get_industry(ticker):
 
 def get_company_metrics(ticker):
   res_basic_fins = get_finnhub_data(f'/stock/metric?symbol={ticker}&metric=all')
-  metric_data = res_basic_fins['metric']
-  annual_series_data = res_basic_fins['series']['annual']
-  quarterly_series_data = res_basic_fins['series']['quarterly']
+  metric_data = res_basic_fins['metric'] if 'metric' in res_basic_fins else 'N/A'
+  annual_series_data = res_basic_fins['series']['annual'] if ('series' in res_basic_fins and 'annual' in res_basic_fins['series']) else 'N/A'
+  quarterly_series_data = res_basic_fins['series']['quarterly'] if ('series' in res_basic_fins and 'quarterly' in res_basic_fins['series']) else 'N/A'
   return metric_data, annual_series_data, quarterly_series_data
 
 
@@ -59,9 +59,9 @@ def get_equity_gains(ticker, period):
   recent_div = sp500.sort_values('Date', ascending=False)
   first_non_zero_dividend_row = recent_div[recent_div['Dividends'] > 0.0]
   if len(first_non_zero_dividend_row) == 0:
-    return val['5Y_change'], 0
+    return val['5Y_change'], 0, recent_div['Close'].iloc[0]
   else:
-    return val['5Y_change'], first_non_zero_dividend_row.iloc[0]['Dividends']
+    return val['5Y_change'], first_non_zero_dividend_row.iloc[0]['Dividends'], recent_div['Close'].iloc[0]
 
 
 def get_90_day_tbill():
@@ -90,10 +90,11 @@ symbols = []
 list_of_tickers = get_list_of_tickers()
 
 with st.form(key="selecting columns"):
-    symbols = st.multiselect(label='Select Tickers:', options=list_of_tickers, placeholder='MSFT, AAPL, ...')
-    submit_button = st.form_submit_button(label='Evaluate')
+    symbols = st.multiselect(label='Enter Tickers Here. Cannot check metrics for Funds.', options=list_of_tickers, placeholder='MSFT, AAPL, ...')
+    strategy_selection = st.radio("Select Strategy", ('Value', 'Growth'), horizontal=True)
+    submit_button = st.form_submit_button(label='Compute Metrics')
 
-    if submit_button and symbols:
+    if submit_button and symbols and strategy_selection == 'Value':
         beta_dfs = []
         gains_data = {}
         hash_map = {}
@@ -105,12 +106,12 @@ with st.form(key="selecting columns"):
             metric_data, annual_series_data, quarterly_series_data  = get_company_metrics(ticker)
             
             # reformat all JSON returns to be flattened dictionaries 
-            roe_dict = {'roe': annual_series_data['roe'][0]['v']}
-            totalDebtToTotalCapital_y_dict = {'totalDebtToTotalCapital_y' : annual_series_data['totalDebtToTotalCapital'][0]['v']}
-            totalDebtToEquity_y_dict = {'totalDebtToEquity_y' :   annual_series_data['totalDebtToEquity'][0]['v']}
-            eps_dict = {'eps' :   annual_series_data['eps'][0]['v']}
-            totalDebtToTotalCapital_q_dict = {'totalDebtToTotalCapital_q' :   quarterly_series_data['totalDebtToTotalCapital'][0]['v']}
-            totalDebtToEquity_q_dict = {'totalDebtToEquity_q' :   quarterly_series_data['totalDebtToEquity'][0]['v']}
+            roe_dict = {'roe': annual_series_data['roe'][0]['v'] if annual_series_data != 'N/A' else 'N/A'}
+            totalDebtToTotalCapital_y_dict = {'totalDebtToTotalCapital_y' : annual_series_data['totalDebtToTotalCapital'][0]['v'] if annual_series_data != 'N/A' else 'N/A'}
+            totalDebtToEquity_y_dict = {'totalDebtToEquity_y' :   annual_series_data['totalDebtToEquity'][0]['v'] if annual_series_data != 'N/A' else 'N/A'}
+            eps_dict = {'eps' :   annual_series_data['eps'][0]['v'] if annual_series_data != 'N/A' else 'N/A'}
+            totalDebtToTotalCapital_q_dict = {'totalDebtToTotalCapital_q' :   quarterly_series_data['totalDebtToTotalCapital'][0]['v'] if quarterly_series_data != 'N/A' else 'N/A'}
+            totalDebtToEquity_q_dict = {'totalDebtToEquity_q' :   quarterly_series_data['totalDebtToEquity'][0]['v'] if quarterly_series_data != 'N/A' else 'N/A'}
             
             # merge all dictionary keys per ticker
             combined_info = basic_info.copy()  # Make a copy of the basic info
@@ -123,26 +124,29 @@ with st.form(key="selecting columns"):
             beta_dfs.append(df_b)
             
             # equity gains
-            _, div = get_equity_gains(ticker=ticker, period=1810)
-            gains_data[ticker] = div
+            _, div, close_price = get_equity_gains(ticker=ticker, period=1810)
+            gains_data[ticker] = [div, close_price]
         
+
         # Now, create a DataFrame from the hash_map
         df_1 = pd.DataFrame.from_dict(hash_map, orient='index')[['finnhubIndustry','eps','roe','dividendGrowthRate5Y','epsGrowth5Y','payoutRatioAnnual','payoutRatioTTM','roeTTM','totalDebtToEquity_y','totalDebtToEquity_q', 'totalDebtToTotalCapital_y','totalDebtToTotalCapital_q']]
         # Create beta df
         beta_df = pd.concat(beta_dfs)
-        df_2 = pd.DataFrame(list(gains_data.items()), columns=['symbol', 'recent_dividend']).set_index('symbol')
-        
+        df_2 = pd.DataFrame.from_dict(gains_data, orient='index', columns=['Recent Dividend','Price'])
+
         df_apis = df_1.join(beta_df)
         df_final = df_apis.join(df_2)
         
         # calculate additional columns 
-        df_final['5Y_SP500_growth'], _ = get_equity_gains(ticker= '^GSPC', period=1810)
-        
+        df_final['5Y_SP500_growth'], _, _ = get_equity_gains(ticker= '^GSPC', period=1810)
         df_final['90_day_tbill'] = 4.06
-        
+        df_final['P/E Ratio'] = df_final['Price'] / df_final['eps']
+        df_final['dividendGrowthRate5Y'] = df_final['dividendGrowthRate5Y']/100
         df_final['CAPM'] = df_final['90_day_tbill']/100 + df_final['Beta']*(df_final['5Y_SP500_growth'] - df_final['90_day_tbill']/100)
-        
-        df_final['DDM'] = (df_final['recent_dividend'] * (1+df_final['dividendGrowthRate5Y']/100)) / (df_final['CAPM'] - df_final['dividendGrowthRate5Y']/100)
-        
+        df_final['DDM'] = (df_final['Recent Dividend'] * (1+df_final['dividendGrowthRate5Y'])) / (df_final['CAPM'] - df_final['dividendGrowthRate5Y'])
+        df_final = df_final[['finnhubIndustry','Price','eps','roe','P/E Ratio','epsGrowth5Y','payoutRatioAnnual','payoutRatioTTM','roeTTM','totalDebtToEquity_y','totalDebtToEquity_q', 'totalDebtToTotalCapital_y','totalDebtToTotalCapital_q','Beta','Recent Dividend','90_day_tbill','5Y_SP500_growth','dividendGrowthRate5Y','CAPM','DDM']]
+        df_final.rename({'finnhubIndustry':'Industry', 'eps':'EPS', 'roe':'ROE'}, inplace=True, axis=1)
         st.write(df_final)
-    
+
+    if submit_button and symbols and strategy_selection == 'Growth':
+       st.write("Not built yet...")

pages/2_Sentiment_Data_Input.py

@@ -7,7 +7,8 @@ import sqlite3
 import pandas as pd
 import streamlit as st
 import os 
-
+import streamlit as st
+from requests.exceptions import HTTPError, RequestException
 
 # API DOC: https://finnhub.io/docs/api/introduction
 
@@ -17,11 +18,25 @@ def get_finnhub_data(example: str) -> json:
     :param1 example: '/company-news?symbol=AAPL&from=2023-08-15&to=2023-08-20'
     """
     base_url = 'https://finnhub.io/api/v1//'
-    
     token = f"&token={os.environ['finnhub_token']}"
 
-    request = requests.get(f"{base_url}{example}{token}")
-    return request.json()
+    try:
+        request = requests.get(f"{base_url}{example}{token}")
+        request.raise_for_status()  # This will raise an HTTPError if the response was an error
+        return request.json()
+    except HTTPError as http_err:
+        st.write(f"HTTP error occurred: {http_err}")  # Python 3.6+
+        # Consider logging the error or handling it further based on your needs
+    except RequestException as err:
+        st.write(f"Other error occurred: {err}")  # Python 3.6+
+        # Handle other types of exceptions (e.g., network issues)
+    except Exception as e:
+        st.write(f"Unexpected error: {e}")
+        # Catch-all for any other exceptions, which is useful for debugging
+
+    return {}  # Return
+
+nlp = pipeline("sentiment-analysis", model="ProsusAI/finbert")
 
 def sentiment_analysis(headline:str) -> str:
     """
@@ -30,10 +45,10 @@ def sentiment_analysis(headline:str) -> str:
     
     :param1 headline: Text string: 'Apple is the best company in the world'
     """
-    nlp = pipeline("sentiment-analysis", model="ProsusAI/finbert")
     return nlp(headline)
 
 
+
 # --------------------------------- get news articles for a company --------------------------------- #
 conn = sqlite3.connect('fin_data.db')
 c = conn.cursor()
@@ -51,51 +66,53 @@ c.execute("""CREATE TABLE IF NOT EXISTS company_news (
         )""")
 
 
-ticker = st.text_input(label='Entering ticker will add Nov23-Jan24 of news sentiment data to database.')
+ticker = st.text_input(label='Entering ticker will add last 5 days of news sentiment data to database.')
 
-if ticker:
+start_date_str = (datetime.datetime.today() - datetime.timedelta(days=5)).strftime('%Y-%m-%d')
+end_date_str = datetime.datetime.today().strftime('%Y-%m-%d')
+dates = set(pd.date_range(start=start_date_str, end=end_date_str).strftime('%Y-%m-%d'))
+
+if st.button('Load') and ticker:
     
-    c.execute('SELECT DISTINCT(ticker) FROM company_news')
+    c.execute('SELECT DISTINCT(ticker), date_stamp FROM company_news where ticker = ?', (ticker,))
     distinct_tickers = c.fetchall()
     distinct_ticker_symbols = [ticker[0] for ticker in distinct_tickers]
+    existing_dates = set([ticker[1] for ticker in distinct_tickers])
+    unique_dates = dates - existing_dates
 
-    if ticker not in distinct_ticker_symbols:
-        try:
-            res_news = get_finnhub_data(f"/company-news?symbol={ticker}&from=2023-12-01&to=2024-02-01")
-        except:
-            RuntimeError('Invalid Ticker.')
-            
-        st.write('Processing ' + str(len(res_news)) + ' articles...')
-        
-        for item in res_news:
-            dt_object = datetime.datetime.fromtimestamp(item['datetime']).strftime("%Y-%m-%d")
-            sentiment = sentiment_analysis(item['headline'])
-            sentiment_label = sentiment[0]['label']
-            sentiment_score = sentiment[0]['score']
-            st.write(sentiment_label, dt_object)
+    if unique_dates:
+        for date in unique_dates:
+            try:
+                res_news = get_finnhub_data(f"/company-news?symbol={ticker}&from={date}&to={date}")
+            except:
+                st.error('Invalid Ticker.')
+            st.write(f"Proessing {len(res_news)} headlines for ", ticker,' on ', date)
+
+            for item in res_news:
+                dt_object = datetime.datetime.fromtimestamp(item['datetime']).strftime("%Y-%m-%d")
+                sentiment = sentiment_analysis(item['headline'])
+                sentiment_label = sentiment[0]['label']
+                sentiment_score = sentiment[0]['score']
+                #st.write(sentiment_label, dt_object)
+                
+                query = """
+                INSERT INTO company_news (ticker, category, headline, date_stamp, sentiment_label, sentiment_score)
+                VALUES (?, ?, ?, ?, ?, ?)
+                """
+                data = (ticker, item['category'], item['headline'], dt_object, sentiment_label, sentiment_score)
             
-            query = """
-            INSERT INTO company_news (ticker, category, headline, date_stamp, sentiment_label, sentiment_score)
-            VALUES (?, ?, ?, ?, ?, ?)
-            """
-            data = (ticker, item['category'], item['headline'], dt_object, sentiment_label, sentiment_score)
-        
-            #Execute the query with the data
-            c.execute(query, data)
+                #Execute the query with the data
+                c.execute(query, data)
 
     c.execute("""
         SELECT * FROM company_news WHERE ticker = ?
     """, (ticker,))
     # Fetch all results
     rows = c.fetchall()
-    
-    # Extract column names from cursor description
-    column_names = [description[0] for description in c.description]
-    
+
     # Create a DataFrame
-    df = pd.DataFrame(rows, columns=column_names)
+    df = pd.DataFrame(rows, columns=[description[0] for description in c.description])
 
-        
     st.write(df)
     
     

pages/scraped_data.py

@@ -1,185 +0,0 @@
-import json
-import requests
-import datetime
-import pandas as pd
-import streamlit as st
-import os 
-import re
-import yfinance as yf
-import plotly.graph_objects as go
-from datetime import timedelta
-
-def get_finnhub_data(example: str) -> json:
-    """
-    Pass in the "example" string from the API documentation. It changes for every endpoint.
-
-    :param1 example: '/company-news?symbol=AAPL&from=2023-08-15&to=2023-08-20'
-    """
-    base_url = 'https://finnhub.io/api/v1//'
-    
-    token = f"&token={os.environ['finnhub_token']}"
-
-    request = requests.get(f"{base_url}{example}{token}")
-    return request.json()
-
-
-def get_alpha_vantage_data(example: str) -> json:
-    """
-    Pass in the "function" string from the API documentation. It changes for every endpoint.
-
-    :param1 example: ''
-    """
-    base_url = 'https://www.alphavantage.co/query?'
-    token = f"&apikey={os.environ['alpha_api_key']}"
-
-    request = requests.get(f"{base_url}{example}{token}")
-    return request.json()
-
-    
-# --------------------------------- list of all tickers ---------------------------------#
-# comp_info = get_finnhub_data('/stock/symbol?exchange=US')
-# list_of_tickers = []
-# for i in range(len(comp_info)-1):
-#   for key in comp_info[i].keys():
-#     if key == 'symbol':
-#       list_of_tickers.append(comp_info[i]['symbol'])
-ticker = []
-ticker = st.text_input('type a ticker')
-
-if ticker:
-    comp_info = get_finnhub_data(f'/stock/profile2?symbol={ticker}')
-    df_industry = pd.DataFrame.from_dict(comp_info, orient='index').T[['finnhubIndustry','name','ticker']]
-    st.write(df_industry)
-                 
-    # --------------------------------- finnhub API --------------------------------- #
-    # get basic financials 
-    #
-    
-    res_basic_fins = get_finnhub_data(f'/stock/metric?symbol={ticker}&metric=all')
-    metric_data = res_basic_fins['metric']
-    annual_series_data = res_basic_fins['series']['annual']
-    quarterly_series_data = res_basic_fins['series']['quarterly']
-    
-    
-    # endpoint 1
-    df_metric_data = pd.DataFrame.from_dict(metric_data, orient='index', columns=['Value'])
-    df_metric_data = df_metric_data.transpose()
-    df_metric_data = df_metric_data[['dividendGrowthRate5Y','epsGrowth5Y','payoutRatioAnnual','payoutRatioTTM','roeTTM']]
-    df_metric_data['symbol'] = res_basic_fins['symbol']
-    st.write(df_metric_data)
-    
-    # endpoint 2
-    df_roe = pd.DataFrame(annual_series_data['roe']).rename(columns={'v': 'bookValue'})
-    df_totalDebtToTotalCapital = pd.DataFrame(annual_series_data['totalDebtToTotalCapital']).rename(columns={'v': 'annual_totalDebtToTotalCapital'})
-    df_totalDebtToEquity = pd.DataFrame(annual_series_data['totalDebtToEquity']).rename(columns={'v': 'annual_totalDebtToEquity'})
-    df_eps = pd.DataFrame(annual_series_data['eps']).rename(columns={'v': 'eps'})
-    
-    # Merge the DataFrames on 'period'
-    dfs = [df_roe, df_totalDebtToTotalCapital, df_totalDebtToEquity, df_eps]
-    df_merged = dfs[0]
-    for df in dfs[1:]:
-        df_merged = df_merged.merge(df, on='period', how='outer')
-    
-    df_annual_series_data = df_merged
-    df_annual_series_data['symbol'] = res_basic_fins['symbol']
-    
-    st.write(df_annual_series_data)
-    
-    
-    # endpoint 3
-    df_totalDebtToTotalCapital = pd.DataFrame(quarterly_series_data['totalDebtToTotalCapital']).rename(columns={'v': 'quarterly_totalDebtToTotalCapital'})
-    df_totalDebtToEquity = pd.DataFrame(quarterly_series_data['totalDebtToEquity']).rename(columns={'v': 'quarterly_totalDebtToEquity'})
-    
-    # Merge the DataFrames on 'period'
-    dfs = [df_totalDebtToTotalCapital, df_totalDebtToEquity]
-    df_merged = dfs[0]
-    for df in dfs[1:]:
-        df_merged = df_merged.merge(df, on='period', how='outer')
-    
-    df_quarterly_series_data = df_merged
-    df_quarterly_series_data['symbol'] = res_basic_fins['symbol']
-    
-    st.write(df_quarterly_series_data)
-
-    # ------------------------ alphavantage API --------------------------------- #
-    # dividend and t-bill data
-    #
-    
-    # endpoint 1
-    q = f'function=TIME_SERIES_MONTHLY_ADJUSTED&symbol={ticker}&interval=5min'
-    dividend_data = get_alpha_vantage_data(q)
-    
-    data = []
-    for date, val in dividend_data['Monthly Adjusted Time Series'].items():
-      # key is the date, val contains another dictionary where the 6th element is a dividend value
-      for label, dividend in val.items():
-        if re.search('dividend', label) and float(dividend) > 0.0000:
-          data.append((date, dividend))
-    df_dividends = pd.DataFrame(data, columns = [['date','monthly_dividend']])
-    st.write(df_dividends)
-
-    # endpoint 2
-    q = 'function=TREASURY_YIELD&'
-    
-    t_bill_data = get_alpha_vantage_data(q)    
-    df_t_bills = pd.DataFrame(t_bill_data['data']).rename(columns={'date':'Date', 'value':'90_day_t_bill'})
-    st.write(df_t_bills)
-
-
-    # ------------------------ yfinance API --------------------------------- #
-    # index and ticker price data (beta calculations)
-    # 
-
-    sp = yf.Ticker("^GSPC")
-    sp_hist = sp.history(period="1y")['Close']
-    
-    sp = yf.Ticker(f'{ticker}')
-    ticker_hist = sp.history(period="1y")['Close']
-
-    sp500 = sp_hist.reset_index().rename(columns={'Close':'sp500_Close'})
-    sp500['sp500_variance'] = sp500['sp500_Close'].var()
-
-    merged_df = sp500.merge(ticker_hist, how='outer', on='Date')
-
-    beta = merged_df.cov().loc['sp500_Close']['Close'] / sp500['sp500_variance'].max()
-    max_date = merged_df['Date'].max()
-    
-    d = {'Date': max_date, 'Beta': beta, 'Symbol': ticker}
-    df_beta = pd.DataFrame(d, index = [0])
-    st.write(df_beta)
-
-    # getting 5 years of history for pct change calculation
-    sp = yf.Ticker("^GSPC")
-    sp_hist = sp.history(period="5y")['Close']
-    sp500 = sp_hist.reset_index().rename(columns={'Close':'sp500_Close'})
-    sp500['Date'] = pd.to_datetime(sp500["Date"].dt.strftime('%Y-%m-%d'))
-    
-    todays_close = sp500.sort_values('Date', ascending=False).iloc[0]
-    
-    sp500_new_index = sp500.set_index('Date')
-    
-    # 1M, 3M, 1Y, 2Y, 3Y & 5Y
-    days = [30, 90, 365, 730, 1095, 1810]
-    
-    val = {}
-    
-    for day in days:
-      if todays_close[0] - timedelta(days=day) in sp500_new_index.index:
-        delta = todays_close['sp500_Close'] - sp500_new_index.loc[todays_close[0] - timedelta(days=day)]['sp500_Close'] 
-        val[f'{day}_day_pct_change'] = (delta / sp500_new_index.loc[todays_close[0] - timedelta(days=day)]['sp500_Close'] *100)
-      else: 
-        val[f'{day}_day_pct_change'] = 'weekend'
-    
-    df_sp500_ytd_change = pd.DataFrame(val, index=['Delta'])
-    st.write(df_sp500_ytd_change)
-
-    
-    # plotting sp500 price over time    
-    price_chart = go.Scatter(
-        x=merged_df.Date,
-        y=merged_df.sp500_Close,
-        name = '1y price history'
-    )
-    
-    fig_candle = go.Figure(price_chart)
-    st.plotly_chart(fig_candle, use_container_width=True) 

requirements.txt

@@ -20,4 +20,5 @@ pygwalker==0.3.9
 streamlit==1.22.0
 regex
 yfinance==0.2.28
-torch
+torch
+python-dotenv

scraped_data.py

@@ -0,0 +1,61 @@
+import json
+import requests
+import datetime
+import pandas as pd
+import streamlit as st
+import os 
+import re
+import yfinance as yf
+import plotly.graph_objects as go
+from datetime import timedelta
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def get_finnhub_data(example: str) -> json:
+    """
+    Pass in the "example" string from the API documentation. It changes for every endpoint.
+
+    :param1 example: '/company-news?symbol=AAPL&from=2023-08-15&to=2023-08-20'
+    """
+    base_url = 'https://finnhub.io/api/v1//'
+    
+    token = f"&token={os.environ['finnhub_token']}"
+
+    request = requests.get(f"{base_url}{example}{token}")
+    return request.json()
+
+
+def get_alpha_vantage_data(example: str) -> json:
+    """
+    Pass in the "function" string from the API documentation. It changes for every endpoint.
+
+    :param1 example: ''
+    """
+    base_url = 'https://www.alphavantage.co/query?'
+    token = f"&apikey={os.environ['alpha_api_key']}"
+
+    request = requests.get(f"{base_url}{example}{token}")
+    return request.json()
+
+
+sp = yf.Ticker("^GSPC")
+sp_hist = sp.history(period="1y")['Close']
+
+sp500 = sp_hist.reset_index().rename(columns={'Close':'sp500_Close'})
+sp500['sp500_variance'] = sp500['sp500_Close'].var()
+
+beta = sp500.cov().loc['sp500_Close']['Close'] / sp500['sp500_variance'].max()
+max_date = sp500['Date'].max()
+
+
+
+# plotting sp500 price over time    
+price_chart = go.Scatter(
+    x=sp500.Date,
+    y=sp500.sp500_Close,
+    name = '1y price history'
+)
+
+fig_candle = go.Figure(price_chart)
+st.plotly_chart(fig_candle, use_container_width=True)