from asyncio.constants import LOG_THRESHOLD_FOR_CONNLOST_WRITES import yfinance as yf import pandas as pd import numpy as np import plotly.graph_objs as go from stocks import * from transformers import AutoModelForSequenceClassification, pipeline, AutoTokenizer import os from random import random os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import tensorflow as tf import math import datetime import random import time #import kaleido from sklearn.preprocessing import MinMaxScaler import matplotlib.pyplot as plt #import warnings import tensorflow as tf from tensorflow import keras from keras.layers import Dropout, Activation from keras import layers from keras.callbacks import EarlyStopping from sklearn.metrics import r2_score import plotly.graph_objs as go import plotly.io as pio pio.templates model = AutoModelForSequenceClassification.from_pretrained("fine_tuned_FinBERT", from_tf=False, config="config.json") tokenizer = AutoTokenizer.from_pretrained("fine_tuned_FinBERT/tokenizer/") class Models(object): def __init__(self): self.stock_data = Stock_Data() def bollinger_bands_20d_2std(self, ticker): ''' This method calculates the Bollinger Bands with a Rolling average of the last 20 days and 2 standard deviations. In a plot, this would be represented as 3 lines: a rolling average, an upper bound (rolling average + 2 standard deviations) and a lower bound (rolling average - 2 standard deviations). When the price of a stock is between the rolling average and lower bound, it is considered as oversold, so it makes sense to buy, if it is between the roll. avg. and the upper bound, it is considered as overbought, so it makes sense to sell, if it is equal to the roll.avg. it is neutral and if it is outside the bounds, it is considered an Unusual Event. The function returns the outlook of the stock (either "Buy", or "Sell" or "Hold" or "Unusual Event") ''' if self.stock_data.status_getter(ticker) != "Open": return "Market Closed" else: data = self.stock_data.stock_data_getter(ticker) low_high_closing_df = pd.DataFrame(data) low_high_closing_df = data.iloc[:, 4:5] # Getting only the "Adj Close" column low_high_closing_df = low_high_closing_df.tail(40) # Getting the last 40 days low_high_closing_df["rolling_avg_20d"] = low_high_closing_df['Adj Close'].rolling(20, min_periods = 20).mean() low_high_closing_df["sd"] = low_high_closing_df["Adj Close"].rolling(20, min_periods = 20).std() low_high_closing_df = low_high_closing_df.tail(20) # Keeping the last 20 days only recent_data = low_high_closing_df.iloc[-1, :].to_list() # Creating a Series object with the most recent data (last row only) upper_bound = recent_data[1] + 2*recent_data[2] # Upper Bound lower_bound = recent_data[1] - 2*recent_data[2] # Lower Bound mean_20d = recent_data[1] # Rolling average of last 20 days if self.stock_data.current_price_getter(ticker) is None: return "Market Closed" else: message = "" if self.stock_data.current_price_getter(ticker) < mean_20d and self.stock_data.current_price_getter(ticker) >= lower_bound: message = "Buy" elif self.stock_data.current_price_getter(ticker) > mean_20d and self.stock_data.current_price_getter(ticker) <= upper_bound: message = "Sell" elif self.stock_data.current_price_getter(ticker) == mean_20d: message = "Hold" elif self.stock_data.current_price_getter(ticker) <= lower_bound or self.stock_data.current_price_getter(ticker) >= upper_bound: message = "Unusual Event" return message def bollinger_bands_10d_1point5std(self, ticker): ''' This method calculates the Bollinger Bands with a Rolling average of the last 10 days and 1.5 standard deviations. In a plot, this would be represented as 3 lines: a rolling average, an upper bound (rolling average + 1.5 standard deviations) and a lower bound (rolling average - 1.5 standard deviations). When the price of a stock is between the rolling average and lower bound, it is considered as oversold, so it makes sense to buy, if it is between the roll. avg. and the upper bound, it is considered as overbought, so it makes sense to sell, if it is equal to the roll.avg. it is neutral and if it is outside the bounds, it is considered an Unusual Event. The function returns the outlook of the stock (either "Buy", or "Sell" or "Hold" or "Unusual Event") ''' if self.stock_data.status_getter(ticker) != "Open": return "Market Closed" else: data = self.stock_data.stock_data_getter(ticker) low_high_closing_df = pd.DataFrame(data) low_high_closing_df = data.iloc[:, 4:5] # Getting only the "Adj Close" column low_high_closing_df = low_high_closing_df.tail(20) # Getting the last 20 days low_high_closing_df["rolling_avg_10d"] = low_high_closing_df['Adj Close'].rolling(10, min_periods = 10).mean() low_high_closing_df["sd"] = low_high_closing_df["Adj Close"].rolling(10, min_periods = 10).std() low_high_closing_df = low_high_closing_df.tail(10) # Keeping the last 10 days only recent_data = low_high_closing_df.iloc[-1, :].to_list() # Creating a Series object with the most recent data (last row only) upper_bound = recent_data[1] + 1.5*recent_data[2] # Upper Bound lower_bound = recent_data[1] - 1.5*recent_data[2] # Lower Bound mean_10d = recent_data[1] # Rolling average of last 10 days if self.stock_data.current_price_getter(ticker) is None: return "Market Closed" else: message = "" if self.stock_data.current_price_getter(ticker) < mean_10d and self.stock_data.current_price_getter(ticker) >= lower_bound: message = "Buy" elif self.stock_data.current_price_getter(ticker) > mean_10d and self.stock_data.current_price_getter(ticker) <= upper_bound: message = "Sell" elif self.stock_data.current_price_getter(ticker) == mean_10d: message = "Hold" elif self.stock_data.current_price_getter(ticker) <= lower_bound or self.stock_data.current_price_getter(ticker) >= upper_bound: message = "Unusual Event" return message def bollinger_bands_50d_3std(self, ticker): ''' This method calculates the Bollinger Bands with a Rolling average of the last 50 days and 3 standard deviations. In a plot, this would be represented as 3 lines: a rolling average, an upper bound (rolling average + 3 standard deviations) and a lower bound (rolling average - 3 standard deviations). When the price of a stock is between the rolling average and lower bound, it is considered as oversold, so it makes sense to buy, if it is between the roll. avg. and the upper bound, it is considered as overbought, so it makes sense to sell, if it is equal to the roll.avg. it is neutral and if it is outside the bounds, it is considered an Unusual Event. The function returns the outlook of the stock (either "Buy", or "Sell" or "Hold" or "Unusual Event") ''' if self.stock_data.status_getter(ticker) != "Open": return "Market Closed" else: data = self.stock_data.stock_data_getter(ticker) low_high_closing_df = pd.DataFrame(data) low_high_closing_df = data.iloc[:, 4:5] # Getting only the "Adj Close" column low_high_closing_df = low_high_closing_df.tail(100) # Getting the last 100 days low_high_closing_df["rolling_avg_50d"] = low_high_closing_df['Adj Close'].rolling(50, min_periods = 50).mean() low_high_closing_df["sd"] = low_high_closing_df["Adj Close"].rolling(50, min_periods = 50).std() low_high_closing_df = low_high_closing_df.tail(50) # Keeping the last 50 days only recent_data = low_high_closing_df.iloc[-1, :].to_list() # Creating a Series object with the most recent data (last row only) upper_bound = recent_data[1] + 3*recent_data[2] # Upper Bound lower_bound = recent_data[1] - 3*recent_data[2] # Lower Bound mean_50d = recent_data[1] # Rolling average of last 50 days # Finding the outlook dependent on the current price if self.stock_data.current_price_getter(ticker) is None: return "Market Closed" else: message = "" if self.stock_data.current_price_getter(ticker) < mean_50d and self.stock_data.current_price_getter(ticker) >= lower_bound: message = "Buy" elif self.stock_data.current_price_getter(ticker) > mean_50d and self.stock_data.current_price_getter(ticker) <= upper_bound: message = "Sell" elif self.stock_data.current_price_getter(ticker) == mean_50d: message = "Hold" elif self.stock_data.current_price_getter(ticker) <= lower_bound or self.stock_data.current_price_getter(ticker) >= upper_bound: message = "Unusual Event" return message def MACD(self, ticker): ''' This method calculates the MACD (Mean Average Convergence Divergence) for a stock. The decision of whether to buy or sell a stock when using this method, depends on the difference of two "lines". The 1st one is called "MACD" and is equal to the difference between the Exponential Moving Average of the adjusted closing price of the last 12 days, and the Moving Average of the adjusted closing price of the last 26 days. The 2nd line is the 9 day moving average of the adj. closing price. When MACD > 9 day M.A. it is considered that there is an uptrend, else, an downtrend. At last, when MACD line crosses the 9 day M.A. from "above", a "Sell" signal is given, while it crosses it from below, a "Buy" signal is given. ''' if self.stock_data.status_getter(ticker) != "Open": return "Market Closed" else: data = self.stock_data.stock_data_getter(ticker) low_high_closing_df = pd.DataFrame(data) low_high_closing_df = data.iloc[:, 4:5] # Getting only the "Adj Close" column low_high_closing_df = low_high_closing_df.tail(52) # Getting the last 52 days # Get the 12-day EMA of the closing price low_high_closing_df['EMA_12d'] = low_high_closing_df['Adj Close'].ewm(span=12, adjust=False, min_periods=12).mean() # Get the 26-day MA of the closing price low_high_closing_df['MA_26d'] = low_high_closing_df['Adj Close'].ewm(span=26, adjust=False, min_periods=26).mean() # Subtract the 26-day EMA from the 12-Day EMA to get the MACD low_high_closing_df['MACD'] = low_high_closing_df['EMA_12d'] - low_high_closing_df['MA_26d'] # Making the signal line low_high_closing_df['MA_9d'] = low_high_closing_df['MACD'].ewm(span=9, adjust=False, min_periods=9).mean() low_high_closing_df['Diff'] = low_high_closing_df['MACD'] - low_high_closing_df['MA_9d'] Diff = low_high_closing_df['Diff'].astype(float) if self.stock_data.current_price_getter(ticker) is None: return "Market Closed" else: message = "" if Diff.iloc[-1] < 0: if Diff.iloc[-2] >= 0: message = "Downtrend and sell signal" else: message = "Downtrend and no signal" else: if Diff.iloc[-2] <= 0: message = "Uptrend and buy signal" else: message = "Uptrend and no signal" return message def finbert_headlines_sentiment(self, ticker): ''' This method uses a the "weights" and the "tokenizer" of a fine-tuned Fin-BERT model, which is a BERT model that was furtherly trained on financial data. The "article_parser()" method scraps www.marketwatch.com and returns the last 17 headers of the chosen stock's articles. The, the FinBERT model classifies each one of them as either "Positive" or "Negative" or "Neutral", and a score is assigned to each header (+100, -100, and 0) correspondingly. At last, a rolling average of window size = 5 is used to "smooth" the sentiment line of the "plotly" plot that is returned. ''' articles_df = self.stock_data.article_parser(ticker) articles_list = articles_df["headline"].tolist() clf = pipeline("text-classification", model=model, tokenizer=tokenizer) outputs_list = clf(articles_list) sentiments = [] for item in outputs_list: sentiments.append(item["label"]) sentiments_df = pd.DataFrame(sentiments) sentiments_df.rename(columns = {0:'sentiment'}, inplace = True) sentiments_df["sentiment"] = sentiments_df["sentiment"].apply(lambda x: 100 if x == "positive" else -100 if x=="negative" else 0) sentiments_df["roll_avg"] = round(sentiments_df["sentiment"].rolling(5, min_periods = 1).mean(), 2) sentiments_df = sentiments_df.tail(12).reset_index() pd.options.plotting.backend = "plotly" fig = sentiments_df["roll_avg"].plot(title="Sentiment Analysis of the last 12 www.marketwatch.com articles about " + ticker, template="plotly_dark", labels=dict(index="12 most recent article headlines", value="sentiment score (rolling avg. of window size 5)")) fig.update_traces(line=dict(color="#3D9140", width=3)) fig.update_layout(yaxis_range=[-100,100]) fig.update_layout(xaxis_range=[0,12]) fig.update_layout(showlegend=False) fig.add_hline(y=0, line_width=1.5, line_color="black") current_sentiment = sentiments_df["roll_avg"].tail(1).values[0] return {'fig': fig, 'current_sentiment': current_sentiment} def LSTM_7_days_price_predictor(self, ticker): ''' This method predicts the price of a chosen stock for the next 7 days as of today, by using the daily adjusted closing prices for the last 2 years. At first, a 60-day window of historical prices (i-60) is created as our feature data (x_train) and the following 60-days window as label data (y_train). For every stock available, we have manually defined different parameters so that they fit as good as it gets to the model. Finally we combute the R2 metric and make the predictions. At last, we proceed with the predictions. The model looks back in our data (60 days back) and predicta for the following 7 days. ''' stock_data = self.stock_data.LSTM_stock_data_getter(ticker) stock_data=pd.DataFrame(data=stock_data).drop(['Open','High','Low','Close', 'Volume'],axis=1).reset_index() stock_data['Date'] = pd.to_datetime(stock_data['Date']) stock_data=stock_data.dropna() # Data Preprocessing random.seed(1997) close_prices = stock_data['Adj Close'] values = close_prices.values training_data_len = math.ceil(len(values)* 0.8) scaler = MinMaxScaler(feature_range=(0,1)) scaled_data = scaler.fit_transform(values.reshape(-1,1)) train_data = scaled_data[0: training_data_len, :] x_train = [] y_train = [] for i in range(60, len(train_data)): x_train.append(train_data[i-60:i, 0]) y_train.append(train_data[i, 0]) x_train, y_train = np.array(x_train), np.array(y_train) x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1)) # Preparation of test set test_data = scaled_data[training_data_len-60: , : ] x_test = [] y_test = values[training_data_len:] for i in range(60, len(test_data)): x_test.append(test_data[i-60:i, 0]) x_test = np.array(x_test) x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1)) ##### Setting Up LSTM Network Architecture and the Training of the LSTM Model def LSTM_trainer(seed, DROPOUT, LSTM_units,patience,batch_size, epochs): tf.random.set_seed(seed) DROPOUT = DROPOUT global model_lstm model_lstm = keras.Sequential() model_lstm.add(layers.LSTM(LSTM_units, return_sequences=True, input_shape=(x_train.shape[1], 1))) model_lstm.add(Dropout(rate=DROPOUT)) model_lstm.add(layers.LSTM(LSTM_units, return_sequences=False)) model_lstm.add(Dropout(rate=DROPOUT)) model_lstm.add(layers.Dense(25)) model_lstm.add(Dropout(rate=DROPOUT)) model_lstm.add(layers.Dense(1)) model_lstm.add(Activation('linear')) print('\n') print("Compiling the LSTM Model for the " + str(ticker) + " stock....\n") t0 = time.time() model_lstm.compile(optimizer='adam', loss='mean_squared_error',metrics=['mae']) callback=EarlyStopping(monitor='val_loss', min_delta=0, patience=patience, verbose=1, mode='auto') model_lstm.fit(x_train, y_train, batch_size= batch_size, epochs=epochs, validation_split=0.1,# ...holding out 10% of the data for validation shuffle=True,verbose=0,callbacks=[callback]) t1 = time.time() global ex_time ex_time = round(t1-t0, 2) print("Compiling took :",ex_time,"seconds") predictions = model_lstm.predict(x_test) predictions = scaler.inverse_transform(predictions) #rmse = np.sqrt(np.mean(((predictions - y_test) ** 2))) global r_squared_score global rmse r_squared_score = round(r2_score(y_test, predictions),2) rmse = np.sqrt(np.mean(((predictions - y_test) ** 2))) #print('Rmse Score: ', round(rmse),2) print('R2 Score: ', r_squared_score) if ticker == 'AAPL': LSTM_trainer(1, 0.2, 100,2, 20, 30) elif ticker == 'NVDA': LSTM_trainer(2, 0.2, 100,2, 30, 50) elif ticker == 'PYPL': LSTM_trainer(6, 0.2, 100,10,25, 30) elif ticker == 'MSFT': LSTM_trainer(4, 0.1, 80, 2,20, 40) elif ticker == 'TSLA': LSTM_trainer(5, 0.1, 120, 4,20, 25) elif ticker == 'AMZN': LSTM_trainer(6, 0.1, 120,2, 20, 25) elif ticker == 'SPOT': LSTM_trainer(9, 0.2, 200,5, 20, 40) #elif ticker == 'TWTR' : # LSTM_trainer(15, 0.2, 100,4,20, 40) elif ticker == 'UBER': LSTM_trainer(15, 0.2, 100,7,20, 40) elif ticker == 'ABNB': LSTM_trainer(15, 0.2, 120,8,20, 40) elif ticker == 'GOOG': LSTM_trainer(15, 0.2, 100,3,20, 25) # Unseen Predictions for the next 7 days close_data = scaled_data look_back = 60 def predict(num_prediction, model): prediction_list = close_data[-look_back:] for _ in range(num_prediction): x = prediction_list[-look_back:] x = x.reshape((1, look_back, 1)) out = model.predict(x)[0][0] prediction_list = np.append(prediction_list, out) prediction_list = prediction_list[look_back-1:] return prediction_list def predict_dates(num_prediction): last_date = stock_data['Date'].values[-1] prediction_dates = pd.date_range(last_date, periods=num_prediction+1).tolist() return prediction_dates num_prediction = 7 forecast = predict(num_prediction, model_lstm) forecast_dates = predict_dates(num_prediction) plt.figure(figsize=(25,10)) forecast = forecast.reshape(-1, 1) forecast_inverse = scaler.inverse_transform(forecast) # Ploting the Actual Prices and the Predictions of them for the next 7 days base = stock_data['Date'].iloc[[-1]] # Here we create our base date (the last existing date with actual prices) testdata = pd.DataFrame(forecast_inverse)# Here we create a data frame that contains the prediction prices and an empty column for their dates testdata['Date'] = "" testdata.columns = ["Adj Close","Date"] testdata = testdata.iloc[1:,:] testdata["Label"] = "" # Let's add a column "Label" that would show if the respective price is a prediction or not testdata["Label"] = "Prediction" testdata = testdata[["Date", "Adj Close", "Label"]] date_list = [base + datetime.timedelta(days=x+1) for x in range(testdata.shape[0]+1)] date_list = pd.DataFrame(date_list) date_list.columns = ["Date"] date_list.reset_index(inplace = True) date_list.drop(["index"], axis = 1, inplace = True) date_list.index = date_list.index + 1 testdata.Date = date_list stock_data["Label"] = "" stock_data["Label"] = "Actual price" finaldf = pd.concat([stock_data,testdata], axis=0) # Here we concatenate the "testdata" and the original data frame "df" into a final one finaldf.reset_index(inplace = True) finaldf.drop(["index"], axis = 1, inplace = True) finaldf['Date'] = pd.to_datetime(finaldf['Date']) plt.rcParams["figure.figsize"] = (25,10) #We create two different data frames, one that contains the actual prices and one that has only the predictions finaldfPredictions = finaldf.iloc[-8:] finaldfActuals = finaldf.iloc[:-7] plot_1 = go.Scatter( x = finaldfActuals['Date'], y = finaldfActuals['Adj Close'], mode = 'lines', name = 'Historical Data (2 years)', line=dict(width=1,color='#3D9140')) plot_2 = go.Scatter( x = finaldfPredictions['Date'], y = finaldfPredictions['Adj Close'], mode = 'lines', name = '7-day Prediction', line=dict(width=1,color="#EE3B3B")) plot_3 = go.Scatter( x = finaldfPredictions['Date'][:1], y = finaldfPredictions['Adj Close'][:1], mode = 'markers', name = 'Latest Actual Closing Price', line=dict(width=1)) layout = go.Layout( title = 'Next 7 days stock price prediction of ' + str(ticker), xaxis = {'title' : "Date"}, yaxis = {'title' : "Price ($)"} ) fig = go.Figure(data=[plot_1, plot_2,plot_3], layout=layout) fig.update_layout(template='plotly_dark',autosize=True) fig.update_layout(legend=dict( orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), annotations = [dict(x=0.5, y=0, xref='paper', yref='paper', text="Current In Sample R- Squared : " + str(r_squared_score*100) + " % \n", showarrow = False)], xaxis=dict(showgrid=False), yaxis=dict(showgrid=False) ) fig.add_annotation(x=0.5, y=0.05, xref='paper', yref='paper', text="Current In Sample Root Mean Square Error : " + str(round(rmse,2)) + " % ", showarrow=False) return fig