```
# src/news_fetcher.py

import requests
import pandas as pd
from datetime import datetime
from textblob import TextBlob
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from .config import NEWS_API_KEY, NEWS_API_ENDPOINT

def fetch_news(ticker, start_date, end_date, page_size=100):
    """
    Fetch news articles related to the given ticker within the date range.

    :param ticker: Stock ticker symbol (e.g., 'AAPL')
    :param start_date: Start date in 'YYYY-MM-DD' format
    :param end_date: End date in 'YYYY-MM-DD' format
    :param page_size: Number of articles per request (max 100)
    :return: List of news articles
    """
    all_articles = []
    page = 1
    while True:
        params = {
            'q': ticker,
            'from': start_date,
            'to': end_date,
            'sortBy': 'relevancy',
            'language': 'en',
            'pageSize': page_size,
            'page': page,
            'apiKey': NEWS_API_KEY
        }
        response = requests.get(NEWS_API_ENDPOINT, params=params)
        if response.status_code != 200:
            print(f"Error fetching news: {response.status_code} - {response.text}")
            break
        data = response.json()
        articles = data.get('articles', [])
        if not articles:
            break
        all_articles.extend(articles)
        if len(articles) < page_size:
            break
        page += 1
    return all_articles

def analyze_sentiment_textblob(articles):
    """
    Analyze sentiment using TextBlob.

    :param articles: List of news articles
    :return: DataFrame with Date and TextBlob sentiment
    """
    sentiments = []
    for article in articles:
        text = (article.get('title') or '') + ' ' + (article.get('description') or '')
        blob = TextBlob(text)
        sentiment = blob.sentiment.polarity  # Range [-1, 1]
        try:
            date = datetime.strptime(article['publishedAt'], '%Y-%m-%dT%H:%M:%SZ').date()
        except ValueError:
            continue  # Skip articles with invalid date formats
        sentiments.append({'Date': date, 'TextBlob_Sentiment': sentiment})
    df_sentiment = pd.DataFrame(sentiments)
    if df_sentiment.empty:
        return pd.DataFrame(columns=['Date', 'TextBlob_Sentiment'])
    df_sentiment = df_sentiment.groupby('Date').mean().reset_index()
    return df_sentiment

def analyze_sentiment_vader(articles):
    """
    Analyze sentiment using VADER.

    :param articles: List of news articles
    :return: DataFrame with Date and VADER sentiment
    """
    analyzer = SentimentIntensityAnalyzer()
    sentiments = []
    for article in articles:
        text = (article.get('title') or '') + ' ' + (article.get('description') or '')
        vs = analyzer.polarity_scores(text)
        sentiment = vs['compound']  # Range [-1, 1]
        try:
            date = datetime.strptime(article['publishedAt'], '%Y-%m-%dT%H:%M:%SZ').date()
        except ValueError:
            continue  # Skip articles with invalid date formats
        sentiments.append({'Date': date, 'VADER_Sentiment': sentiment})
    df_sentiment = pd.DataFrame(sentiments)
    if df_sentiment.empty:
        return pd.DataFrame(columns=['Date', 'VADER_Sentiment'])
    df_sentiment = df_sentiment.groupby('Date').mean().reset_index()
    return df_sentiment

def fetch_and_process_news(ticker, start_date, end_date):
    """
    Fetch news articles and analyze sentiment.

    :param ticker: Stock ticker symbol
    :param start_date: Start date in 'YYYY-MM-DD'
    :param end_date: End date in 'YYYY-MM-DD'
    :return: DataFrame with sentiment features
    """
    print(f"Fetching news for {ticker} from {start_date} to {end_date}...")
    articles = fetch_news(ticker, start_date, end_date)
    if not articles:
        print("No articles found.")
        return pd.DataFrame(columns=['Date', 'TextBlob_Sentiment', 'VADER_Sentiment'])
    print(f"Fetched {len(articles)} articles.")
    df_textblob = analyze_sentiment_textblob(articles)
    df_vader = analyze_sentiment_vader(articles)
    # Merge sentiments on Date
    df_sentiment = pd.merge(df_textblob, df_vader, on='Date', how='outer').fillna(0)
    return df_sentiment
```
```
# src/train_all_models.py

import sys
import os
import pandas as pd
from multiprocessing import Pool
from data_loader import load_data, download_data
from feature_engineering import create_features
from news_fetcher import fetch_and_process_news
from config import RAW_DATA_DIR, PROCESSED_DATA_DIR, START_DATE, END_DATE
from utils import evaluate_model
from sklearn.model_selection import TimeSeriesSplit, GridSearchCV, train_test_split
from sklearn.preprocessing import StandardScaler
from xgboost import XGBRegressor
import joblib
import shap
import matplotlib.pyplot as plt

def train_model(ticker):
    try:
        print(f"Processing ticker: {ticker}")
        
        # Define file paths
        raw_save_path = os.path.join(RAW_DATA_DIR, f"{ticker}.csv")
        processed_save_path = os.path.join(PROCESSED_DATA_DIR, f"{ticker}_processed.csv")
        model_file = os.path.join(PROCESSED_DATA_DIR, f"{ticker}_xgb_model.pkl")
        scaler_file = os.path.join(PROCESSED_DATA_DIR, f"{ticker}_xgb_scaler.pkl")
        
        # Download stock data
        print(f"Downloading data for {ticker}...")
        download_data(ticker, START_DATE, END_DATE, raw_save_path)
        
        # Load stock data
        print(f"Loading data for {ticker}...")
        data = load_data(raw_save_path)
        
        # Feature Engineering
        print(f"Creating features for {ticker}...")
        data_features = create_features(data)
        
        # Fetch and process news sentiment
        print(f"Fetching and processing news sentiment for {ticker}...")
        sentiment_data = fetch_and_process_news(ticker, START_DATE, END_DATE)
        
        # Merge sentiment data
        print(f"Merging sentiment data for {ticker}...")
        data_features.reset_index(inplace=True)
        data_features = pd.merge(data_features, sentiment_data, on='Date', how='left')
        data_features[['TextBlob_Sentiment', 'VADER_Sentiment']] = data_features[['TextBlob_Sentiment', 'VADER_Sentiment']].fillna(0)
        data_features.set_index('Date', inplace=True)
        
        # Save processed data
        data_features.to_csv(processed_save_path)
        print(f"Processed data saved to {processed_save_path}")
        
        # Define features and target
        features = [
            'Close', 'MA10', 'MA50', 'Volatility', 'Returns',
            'RSI', 'MACD', 'MACD_Signal', 'MACD_Diff',
            'Bollinger_High', 'Bollinger_Low', 'ATR', 'ADX', 'SAR',
            'Rolling_Mean_7', 'Rolling_STD_7', 'Rolling_Mean_21', 'Rolling_STD_21',
            'Day_of_Week', 'Month', 'Quarter', 'Is_End_Of_Month', 'MA10_MA50_Ratio',
            'TextBlob_Sentiment', 'VADER_Sentiment'
        ] + [f'Close_Lag_{lag}' for lag in range(1, 15)] + \
            [f'MA10_Lag_{lag}' for lag in range(1, 15)] + \
            [f'MA50_Lag_{lag}' for lag in range(1, 15)] + \
            [f'Volatility_Lag_{lag}' for lag in range(1, 15)]
        
        target = 'Target'
        
        X = data_features[features]
        y = data_features[target]
        
        # Data Splitting
        print(f"Splitting data for {ticker}...")
        tscv = TimeSeriesSplit(n_splits=5)
        split = list(tscv.split(X))[-1]
        X_train, X_test = X.iloc[split[0]], X.iloc[split[1]]
        y_train, y_test = y.iloc[split[0]], y.iloc[split[1]]
        
        # Data Scaling
        print(f"Scaling data for {ticker}...")
        scaler = StandardScaler()
        X_train_scaled = scaler.fit_transform(X_train)
        X_test_scaled = scaler.transform(X_test)
        
        # Hyperparameter Grid
        param_grid = {
            'n_estimators': [100, 200, 300],
            'max_depth': [3, 6, 9],
            'learning_rate': [0.01, 0.1, 0.2],
            'subsample': [0.7, 0.8, 0.9],
            'colsample_bytree': [0.7, 0.8, 0.9]
        }
        
        # Initialize XGBRegressor
        print(f"Initializing XGBRegressor for {ticker}...")
        model = XGBRegressor(objective='reg:squarederror', random_state=42)
        
        # Grid Search
        print(f"Starting Grid Search for {ticker}...")
        grid_search = GridSearchCV(
            estimator=model,
            param_grid=param_grid,
            cv=tscv,
            scoring='neg_mean_squared_error',
            n_jobs=-1,
            verbose=1
        )
        
        grid_search.fit(X_train_scaled, y_train)
        
        # Best Parameters
        best_params = grid_search.best_params_
        print(f'Best parameters for {ticker}: {best_params}')
        
        # Best Model
        best_model = grid_search.best_estimator_
        
        # Train Best Model with Early Stopping
        print(f"Training best model with early stopping for {ticker}...")
        X_train_main, X_valid, y_train_main, y_valid = train_test_split(
            X_train_scaled, y_train, test_size=0.1, shuffle=False
        )
        
        best_model.fit(
            X_train_main, y_train_main,
            eval_set=[(X_valid, y_valid)],
            early_stopping_rounds=50,
            verbose=False
        )
        
        # Make Predictions
        print(f"Making predictions for {ticker}...")
        y_pred = best_model.predict(X_test_scaled)
        
        # Evaluate Model
        print(f"Evaluating model for {ticker}...")
        metrics = evaluate_model(y_test, y_pred)
        print(f'Model Metrics for {ticker}:')
        for key, value in metrics.items():
            print(f'{key}: {value}')
        
        # SHAP Interpretation (Optional)
        print(f"Generating SHAP summary plot for {ticker}...")
        explainer = shap.Explainer(best_model)
        shap_values = explainer(X_test_scaled)
        
        # Save SHAP Summary Plot
        shap.summary_plot(shap_values, X_test, feature_names=features, show=False)
        shap_plot_path = os.path.join(PROCESSED_DATA_DIR, f"{ticker}_shap_summary.png")
        plt.savefig(shap_plot_path, bbox_inches='tight')
        plt.close()
        print(f"SHAP summary plot saved to {shap_plot_path}")
        
        # Plot Actual vs Predicted Prices
        print(f"Plotting Actual vs Predicted prices for {ticker}...")
        plt.figure(figsize=(12,6))
        plt.plot(y_test.index, y_test, label='Actual Price')
        plt.plot(y_test.index, y_pred, label='Predicted Price')
        plt.legend()
        plt.title(f'Actual vs Predicted Stock Prices for {ticker}')
        plt.xlabel('Date')
        plt.ylabel('Price USD')
        plt_path = os.path.join(PROCESSED_DATA_DIR, f"{ticker}_actual_vs_predicted.png")
        plt.savefig(plt_path, bbox_inches='tight')
        plt.close()
        print(f"Actual vs Predicted plot saved to {plt_path}")
        
        # Save the Model and Scaler
        print(f"Saving model and scaler for {ticker}...")
        joblib.dump(best_model, model_file)
        joblib.dump(scaler, scaler_file)
        print(f"Model and scaler saved to {model_file} and {scaler_file} respectively.\n")
    
    def main():
        # List of tickers to process
        tickers = ['AAPL', 'GOOGL', 'MSFT', 'AMZN', 'FB']  # Extend as needed
        
        # Use multiprocessing to handle multiple tickers simultaneously
        print("Starting model training for all tickers...")
        with Pool(processes=4) as pool:  # Adjust based on your CPU cores
            pool.map(train_model, tickers)
        print("Completed training for all tickers.")
    
    if __name__ == "__main__":
        main()
```
```
# notebooks/data_exploration.ipynb

import sys
sys.path.append('../src')
from data_loader import load_data, download_data
from feature_engineering import create_features
from news_fetcher import fetch_and_process_news
from config import RAW_DATA_DIR, PROCESSED_DATA_DIR, START_DATE, END_DATE
import matplotlib.pyplot as plt
import seaborn as sns
import os
import pandas as pd

# Define list of tickers
tickers = ['AAPL', 'GOOGL', 'MSFT', 'AMZN', 'FB']  # Add more tickers as needed

# Function to load processed data
def load_processed_data(ticker):
    processed_file = os.path.join(PROCESSED_DATA_DIR, f"{ticker}_processed.csv")
    if not os.path.exists(processed_file):
        print(f"Processed file for {ticker} does not exist.")
        return None
    df = pd.read_csv(processed_file, parse_dates=['Date'], index_col='Date')
    return df

# Iterate through tickers and explore data
for ticker in tickers:
    print(f"\nExploring data for {ticker}")
    df = load_processed_data(ticker)
    if df is None:
        continue
    
    # Show the first few rows
    print("First 5 rows of data:")
    display(df.head())
    
    # Display data info
    print("\nData Info:")
    print(df.info())
    
    # Show statistical summary
    print("\nData Description:")
    print(df.describe())
    
    # Check for missing values
    print("\nMissing Values:")
    print(df.isnull().sum())
    
    # Plot Closing Price
    plt.figure(figsize=(12,6))
    plt.plot(df['Close'], label='Close Price')
    plt.title(f'Closing Price of {ticker}')
    plt.xlabel('Date')
    plt.ylabel('Close Price (USD)')
    plt.legend()
    plt.show()
    
    # Plot Distribution of Closing Price
    plt.figure(figsize=(10,6))
    sns.histplot(df['Close'], bins=50)
    plt.title('Distribution of Closing Price')
    plt.xlabel('Close Price (USD)')
    plt.ylabel('Frequency')
    plt.show()
    
    # Calculate and Plot Correlation Matrix
    corr_matrix = df.corr()
    plt.figure(figsize=(16,12))
    sns.heatmap(corr_matrix, annot=True, cmap='coolwarm')
    plt.title(f'Correlation Matrix for {ticker}')
    plt.show()
    
    # Plot Moving Averages
    plt.figure(figsize=(12,6))
    plt.plot(df['Close'], label='Closing Price')
    plt.plot(df['MA10'], label='MA10')
    plt.plot(df['MA50'], label='MA50')
    plt.legend()
    plt.title('Moving Averages')
    plt.xlabel('Date')
    plt.ylabel('Price (USD)')
    plt.show()
    
    # Plot Sentiment Scores
    plt.figure(figsize=(14,7))
    plt.plot(df['TextBlob_Sentiment'], label='TextBlob Sentiment', alpha=0.7)
    plt.plot(df['VADER_Sentiment'], label='VADER Sentiment', alpha=0.7)
    plt.title(f'News Sentiment Over Time for {ticker}')
    plt.xlabel('Date')
    plt.ylabel('Sentiment Score')
    plt.legend()
    plt.show()
    
    # Pairplot of Selected Features
    selected_features = ['Close', 'TextBlob_Sentiment', 'VADER_Sentiment', 'RSI', 'ADX', 'ATR']
    sns.pairplot(df[selected_features])
    plt.suptitle(f'Pairplot of Selected Features for {ticker}', y=1.02)
    plt.show()
```
```
KeyError Traceback (most recent call last)
File ~\PycharmProjects_Python_3_12_0\pythonProject2\stock_prediction_project\venv\lib\site-packages\pandas\core\indexes\base.py:3805, in Index.get_loc(self, key)
3804 try:
-> 3805 return self._engine.get_loc(casted_key)
3806 except KeyError as err:

File index.pyx:167, in pandas._libs.index.IndexEngine.get_loc()

File index.pyx:196, in pandas._libs.index.IndexEngine.get_loc()

File pandas\_libs\hashtable_class_helper.pxi:7081, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas\_libs\hashtable_class_helper.pxi:7089, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'TextBlob_Sentiment'

The above exception was the direct cause of the following exception:

KeyError Traceback (most recent call last)
Cell In[2], line 86
84 # Plot Sentiment Scores
85 plt.figure(figsize=(14,7))
---> 86 plt.plot(df['TextBlob_Sentiment'], label='TextBlob Sentiment', alpha=0.7)
87 plt.plot(df['VADER_Sentiment'], label='VADER Sentiment', alpha=0.7)
88 plt.title(f'News Sentiment Over Time for {ticker}')

File ~\PycharmProjects_Python_3_12_0\pythonProject2\stock_prediction_project\venv\lib\site-packages\pandas\core\frame.py:4102, in DataFrame.getitem(self, key)
4100 if self.columns.nlevels > 1:
4101 return self._getitem_multilevel(key)
-> 4102 indexer = self.columns.get_loc(key)
4103 if is_integer(indexer):
4104 indexer = [indexer]

File ~\PycharmProjects_Python_3_12_0\pythonProject2\stock_prediction_project\venv\lib\site-packages\pandas\core\indexes\base.py:3812, in Index.get_loc(self, key)
3807 if isinstance(casted_key, slice) or (
3808 isinstance(casted_key, abc.Iterable)
3809 and any(isinstance(x, slice) for x in casted_key)
3810 ):
3811 raise InvalidIndexError(key)
-> 3812 raise KeyError(key) from err
3813 except TypeError:
3814 # If we have a listlike key, _check_indexing_error will raise
3815 # InvalidIndexError. Otherwise we fall through and re-raise
3816 # the TypeError.
3817 self._check_indexing_error(key)

KeyError: 'TextBlob_Sentiment'
```
Close,Close,High,Low,Open,Volume,MA10,MA50,Volatility,Returns,RSI,MACD,MACD_Signal,MACD_Diff,Bollinger_High,Bollinger_Low,ATR,ADX,SAR,Rolling_Mean_7,Rolling_STD_7,Rolling_Mean_21,Rolling_STD_21,Day_of_Week,Month,Quarter,Is_End_Of_Month,MA10_MA50_Ratio,Target,Close_Lag_1,MA10_Lag_1,MA50_Lag_1,Volatility_Lag_1,Close_Lag_2,MA10_Lag_2,MA50_Lag_2,Volatility_Lag_2,Close_Lag_3,MA10_Lag_3,MA50_Lag_3,Volatility_Lag_3,Close_Lag_4,MA10_Lag_4,MA50_Lag_4,Volatility_Lag_4,Close_Lag_5,MA10_Lag_5,MA50_Lag_5,Volatility_Lag_5,Close_Lag_6,MA10_Lag_6,MA50_Lag_6,Volatility_Lag_6,Close_Lag_7,MA10_Lag_7,MA50_Lag_7,Volatility_Lag_7,Close_Lag_8,MA10_Lag_8,MA50_Lag_8,Volatility_Lag_8,Close_Lag_9,MA10_Lag_9,MA50_Lag_9,Volatility_Lag_9,Close_Lag_10,MA10_Lag_10,MA50_Lag_10,Volatility_Lag_10,Close_Lag_11,MA10_Lag_11,MA50_Lag_11,Volatility_Lag_11,Close_Lag_12,MA10_Lag_12,MA50_Lag_12,Volatility_Lag_12,Close_Lag_13,MA10_Lag_13,MA50_Lag_13,Volatility_Lag_13,Close_Lag_14,MA10_Lag_14,MA50_Lag_14,Volatility_Lag_14
```
# src/feature_engineering.py

import pandas as pd
import ta  # Ensure this library is installed via `pip install ta`


def create_features(data):
    """
    Add new features to the DataFrame.

    :param data: DataFrame with raw stock data.
    :return: DataFrame with additional features.
    """
    df = data.copy()

    # Calculate Moving Averages
    df['MA10'] = df['Close'].rolling(window=10).mean()
    df['MA50'] = df['Close'].rolling(window=50).mean()

    # Calculate Volatility (Rolling Standard Deviation)
    df['Volatility'] = df['Close'].rolling(window=10).std()

    # Calculate Returns
    df['Returns'] = df['Close'].pct_change()

    # Calculate Relative Strength Index (RSI)
    df['RSI'] = ta.momentum.RSIIndicator(close=df['Close'], window=14).rsi()

    # Calculate Moving Average Convergence Divergence (MACD)
    macd = ta.trend.MACD(close=df['Close'])
    df['MACD'] = macd.macd()
    df['MACD_Signal'] = macd.macd_signal()
    df['MACD_Diff'] = macd.macd_diff()

    # Calculate Bollinger Bands
    bollinger = ta.volatility.BollingerBands(close=df['Close'], window=20, window_dev=2)
    df['Bollinger_High'] = bollinger.bollinger_hband()
    df['Bollinger_Low'] = bollinger.bollinger_lband()

    # Calculate Average True Range (ATR)
    df['ATR'] = ta.volatility.AverageTrueRange(high=df['High'], low=df['Low'], close=df['Close'], window=14).average_true_range()

    # Calculate Average Directional Index (ADX)
    df['ADX'] = ta.trend.ADXIndicator(high=df['High'], low=df['Low'], close=df['Close'], window=14).adx()

    # Calculate Parabolic SAR (Stop and Reverse)
    df['SAR'] = ta.trend.PSARIndicator(high=df['High'], low=df['Low'], close=df['Close'], step=0.02, max_step=0.2).psar()

    # Calculate Rolling Window Statistics
    df['Rolling_Mean_7'] = df['Close'].rolling(window=7).mean()
    df['Rolling_STD_7'] = df['Close'].rolling(window=7).std()
    df['Rolling_Mean_21'] = df['Close'].rolling(window=21).mean()
    df['Rolling_STD_21'] = df['Close'].rolling(window=21).std()

    # Add Time-Based Features
    df['Day_of_Week'] = df.index.dayofweek  # 0=Monday, 6=Sunday
    df['Month'] = df.index.month
    df['Quarter'] = df.index.quarter
    df['Is_End_Of_Month'] = df.index.is_month_end.astype(int)

    # Add Interaction Features
    df['MA10_MA50_Ratio'] = df['MA10'] / df['MA50']

    # Define the target variable as the next day's closing price
    df['Target'] = df['Close'].shift(-1)

    # Generate Lag Features for Close, MA10, MA50, and Volatility
    for lag in range(1, 15):  # Create lag features from 1 to 14
        df[f'Close_Lag_{lag}'] = df['Close'].shift(lag)
        df[f'MA10_Lag_{lag}'] = df['MA10'].shift(lag)
        df[f'MA50_Lag_{lag}'] = df['MA50'].shift(lag)
        df[f'Volatility_Lag_{lag}'] = df['Volatility'].shift(lag)

    # Handle missing values resulting from feature engineering steps
    df.dropna(inplace=True)

    return df
```