files Uploaded

config/config.yaml

@@ -0,0 +1,11 @@
+artifacts_root: artifacts
+
+data_ingestion:
+  root_dir: artifacts/data_ingestion
+  source_url: https://drive.google.com/file/d/1kI7vnsPB46Z2grGq9GSaUCaQ3Y1ekI9a/view?usp=sharing
+  data_dir: artifacts/data_ingestion/
+  scaler_path: artifacts/data_preprocessing
+
+model_training:
+  root_dir: artifacts/model_training
+  trained_model_path: artifacts/model_training/model.keras

params.yaml

@@ -0,0 +1,4 @@
+EPOCHS: 300
+BATCH_SIZE: 32
+LEARNING_RATE: 0.001
+TIME_STEPS: 60

requirements.txt

@@ -1,16 +1,16 @@
-tensorflow==2.17.0
-yfinance==0.2.40
-pandas==2.2.2
-numpy
-matplotlib==3.9.1
-seaborn==0.13.2
-python-box==7.2.0
-pyYAML==6.0.1
-tqdm==4.66.4
-ensure==1.0.4
-joblib==1.4.2
-types-PyYAML==6.0.12
-fastapi==0.111.0
-streamlit==1.36.0
-plotly==5.22.0
+tensorflow==2.17.0
+yfinance==0.2.40
+pandas==2.2.2
+numpy
+matplotlib==3.9.1
+seaborn==0.13.2
+python-box==7.2.0
+pyYAML==6.0.1
+tqdm==4.66.4
+ensure==1.0.4
+joblib==1.4.2
+types-PyYAML==6.0.12
+fastapi==0.111.0
+streamlit==1.36.0
+plotly==5.22.0
 scikit-learn

tsForecaster/__init__.py

@@ -0,0 +1,21 @@
+import os
+import sys
+import logging
+
+logging_str = "[%(asctime)s: %(levelname)s: %(module)s: %(message)s]"
+
+log_dir = "logs"
+log_filepath = os.path.join(log_dir, "running_logs.log")
+os.makedirs(log_dir, exist_ok=True)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format=logging_str,
+    
+    handlers=[
+        logging.FileHandler(log_filepath),
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+
+logger = logging.getLogger("tsForecasterLogger")

tsForecaster/components/data_ingestion.py

@@ -0,0 +1,44 @@
+import os
+import opendatasets as od
+import yfinance as yf
+import pandas as pd
+from tsForecaster.entity.config_entity import DataIngestionConfig
+from tsForecaster import logger
+
+class DataIngestion:
+    def __init__(self, config:DataIngestionConfig) -> None:
+        self.config = config
+    
+    def download_file(self) ->None:
+        try:
+            download_dir = self.config.data_dir
+            dataset_url = self.config.source_url
+            os.makedirs(download_dir, exist_ok=True)
+            logger.info(f"Downloading data from {dataset_url} into file {download_dir}")
+            od.download(dataset_url, data_dir=download_dir)
+            logger.info(f"Downloaded data from {dataset_url} into file {download_dir}")
+        except Exception as e:
+            raise e
+    
+    def update_file(self) ->None:
+        try:
+            download_dir = self.config.data_dir
+            ticker = yf.Ticker("^NSEI")
+            history = ticker.history(start='2024-07-08', interval='1d')
+            history.drop(columns=['Volume', 'Dividends', 'Stock Splits'], inplace=True)
+            history = round(history, 2)
+            history['Index Name'] = "NIFTY 50"
+            history.reset_index(inplace=True)
+            history['Date'] = history['Date'].dt.strftime('%d %b %Y')
+            history = history[['Index Name', 'Date', 'Open', 'High', 'Low', 'Close']]
+            history = history.sort_values(by=['Date'], ascending=False)
+            df = pd.read_csv(os.path.join(download_dir ,"NIFTY 50_Historical.csv"))
+            df = pd.concat([history, df], ignore_index=True)
+            df = df[['Index Name', 'Date', 'Open', 'High', 'Low', 'Close']]
+            df['Date'] = pd.to_datetime(df['Date'])
+            df = df.drop_duplicates(subset=['Date'])
+            df.to_csv(os.path.join(download_dir ,"NIFTY 50_Historical.csv"))
+        
+        except Exception as e:
+            raise e
+    

tsForecaster/components/data_preprocessing.py

@@ -0,0 +1,77 @@
+import os
+from pathlib import Path
+import joblib
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import MinMaxScaler
+from tsForecaster.entity.config_entity import DataIngestionConfig
+from tsForecaster import logger
+
+class DataPreProcessing:
+    def __init__(self, config: DataIngestionConfig) -> None:
+        self.config = config
+
+    def process_csv(self) -> pd.DataFrame:
+        logger.info("Processing data from CSV to DataFrame")
+        df_path = Path(self.config.data_dir) / "NIFTY 50_Historical.csv"
+        df = pd.read_csv(df_path, index_col=[0])
+        df.drop(columns=['Index Name', 'Open', 'High', 'Low'], inplace=True)
+        df['Date'] = pd.to_datetime(df['Date'])
+        
+        start_date = '1990-07-03'
+        end_date = pd.Timestamp.today().strftime('%Y-%m-%d')
+        date_range = pd.date_range(start=start_date, end=end_date, freq='D')
+        date_df = pd.DataFrame(date_range, columns=['Date'])
+        df = pd.merge(date_df, df, how='left', on='Date')
+        df = df.ffill()
+        
+        df['Close%'] = ((df['Close'] / df['Close'].shift(1)) - 1) * 100
+        for period, days in {
+            '1D': 1, '2D': 2, '3D': 3, '1W': 7, '2W': 14, '1M': 30, '2M': 60,
+            '3M': 90, '6M': 180, '1Y': 365, '2Y': 730, '3Y': 1095, '5Y': 1825,
+            '7Y': 2555, '10Y': 3650
+        }.items():
+            df[f'Close_{period}_ago'] = df['Close%'].shift(days)
+            
+        df.dropna(inplace=True)
+        df.reset_index(drop=True, inplace=True)
+        df.set_index('Date', inplace=True)
+        
+        logger.info("Done processing data from CSV to DataFrame")
+        return df
+
+    def scaling_data(self, df: pd.DataFrame):
+        logger.info("Scaling data...")
+        scaler = MinMaxScaler()
+        df_scaled = pd.DataFrame(scaler.fit_transform(df), columns=df.columns, index=df.index)
+        joblib.dump(scaler, Path(os.path.join(self.config.scaler_path, 'scaler.pkl')))
+        logger.info("Data scaling done")
+        return df_scaled
+    
+    def create_sequences(self, df: pd.DataFrame, time_steps: int):
+        logger.info("Creating X, y sequences...")
+        X, y = [], []
+        for i in range(len(df) - time_steps):
+            X.append(df.iloc[i: i + time_steps, 1:].values)
+            y.append(df.iloc[i, 0])
+        dates = df.index[time_steps:]
+        logger.info("X, y sequences created")
+        return np.array(X), np.array(y), dates
+
+    def train_test_split(self, X, y, dates, train_len=0.8, val_len=0.1, test_len=0.1):
+        logger.info("Splitting train, val, test data...")
+        total_len = train_len + val_len + test_len
+        if total_len != 1.0:
+            logger.error("Error splitting train, val, test data: Total length is not equal to 1")
+            raise ValueError("Aggregate length of train, validation, and test lengths should be equal to 1")
+        
+        total_size = len(y)
+        train_size = int(total_size * train_len)
+        val_size = int(total_size * val_len)
+        
+        X_train, X_val, X_test = X[:train_size], X[train_size:train_size+val_size], X[train_size+val_size:]
+        y_train, y_val, y_test = y[:train_size], y[train_size:train_size+val_size], y[train_size+val_size:]
+        dates_train, dates_val, dates_test = dates[:train_size], dates[train_size:train_size+val_size], dates[train_size+val_size:]
+        
+        logger.info("Done splitting train, val, test data")
+        return X_train, X_val, X_test, y_train, y_val, y_test, dates_train, dates_val, dates_test

tsForecaster/components/model_evaluation.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+from matplotlib import pyplot as plt
+import tensorflow as tf
+from tsForecaster.entity.config_entity import ModelTrainingConfig
+from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
+
+
+class ModelEvaluation:
+    def __init__(self, config:ModelTrainingConfig) -> None:
+        self.config = config
+    
+    @staticmethod
+    def load_model(path: Path) -> tf.keras.Model:
+        return tf.keras.models.load_model(path)
+    
+    def evaluation(self, X, y, dates):
+        self.model = self.load_model(self.config.training_model_path)
+        y_pred = self.model.predict(X).flatten()
+        
+        plt.figure(figsize=(15, 5))
+        plt.plot(dates, y)
+        plt.plot(dates, y_pred)
+        plt.legend(['y-true', 'y-pred'])
+        plt.show()
+        
+        mae = mean_absolute_error(y, y_pred)
+        mse = mean_squared_error(y, y_pred)
+        rmse = mean_squared_error(y, y_pred, squared=False)
+        r2 = r2_score(y, y_pred)
+        
+        return mae, mse, rmse, r2

tsForecaster/components/model_training.py

@@ -0,0 +1,46 @@
+import tensorflow as tf
+from pathlib import Path
+from tsForecaster.entity.config_entity import ModelTrainingConfig
+from tsForecaster.utils.common import create_directories
+
+
+class ModelTraining:
+    def __init__(self, config: ModelTrainingConfig) -> None:
+        self.config = config
+        self.model = self.create_model()
+        
+        create_directories([config.root_dir])
+    
+    @staticmethod
+    def save_model(path: Path, model: tf.keras.Model):
+        model.save(path)
+    
+    def create_model(self) -> tf.keras.Model:
+        model = tf.keras.models.Sequential([
+                    tf.keras.layers.Input((self.config.params_time_steps, 15)),
+                    tf.keras.layers.GRU(128, return_sequences=True),
+                    tf.keras.layers.GRU(128, return_sequences=False),
+                    tf.keras.layers.Dense(64, activation='relu'),
+                    tf.keras.layers.Dense(64, activation='relu'),
+                    tf.keras.layers.Dense(64, activation='relu'),
+                    tf.keras.layers.Dense(1)
+                    ])
+        
+        model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(learning_rate=self.config.params_learning_rate), metrics=['mean_squared_error'])
+        
+        model.summary()
+        
+        return model
+    
+    def train(self, X_train, y_train, X_val, y_val):
+        early_stop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=10, restore_best_weights=True)
+        history = self.model.fit(
+                        X_train,
+                        y_train,
+                        validation_data=(X_val, y_val),
+                        epochs=self.config.params_epochs,
+                        batch_size=self.config.params_batch_size,
+                        callbacks=[early_stop]
+                    )
+        self.save_model(path=self.config.training_model_path, model=self.model)
+        return history

tsForecaster/config/configuration.py

@@ -0,0 +1,44 @@
+import os
+from pathlib import Path
+from tsForecaster.constants import CONFIG_FILE_PATH, PARAMS_FILE_PATH
+from tsForecaster.utils.common import read_yaml, create_directories
+from tsForecaster.entity.config_entity import DataIngestionConfig, ModelTrainingConfig
+
+class ConfigurationManager:
+    def __init__(self, config_file_path=CONFIG_FILE_PATH, params_file_path=PARAMS_FILE_PATH) -> None:
+        self.config = read_yaml(config_file_path)
+        self.params = read_yaml(params_file_path)
+        
+        create_directories([self.config.artifacts_root])
+    
+    def get_data_ingestion_config(self) -> DataIngestionConfig:
+        config = self.config.data_ingestion
+        
+        create_directories([config.root_dir])
+        
+        data_ingestion_config = DataIngestionConfig(
+            root_dir=config.root_dir,
+            source_url=config.source_url,
+            data_dir=config.data_dir,
+            scaler_path=config.scaler_path
+        )
+        
+        return data_ingestion_config
+    
+    def get_model_training_config(self) -> ModelTrainingConfig:
+        model_training = self.config.model_training
+        params = self.params
+        
+        create_directories([model_training.root_dir])
+        
+        model_training_config = ModelTrainingConfig(
+            root_dir=Path(model_training.root_dir),
+            training_model_path=Path(model_training.trained_model_path),
+            training_data_path=Path(self.config.data_ingestion.data_dir),
+            params_epochs=params.EPOCHS,
+            params_batch_size=params.BATCH_SIZE,
+            params_learning_rate=params.LEARNING_RATE,
+            params_time_steps=params.TIME_STEPS
+        )
+        
+        return model_training_config

tsForecaster/constants/__init__.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+CONFIG_FILE_PATH = Path("config/config.yaml")
+PARAMS_FILE_PATH = Path("params.yaml")

tsForecaster/entity/config_entity.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+@dataclass(frozen=True)
+class DataIngestionConfig:
+    root_dir: Path
+    source_url: Path
+    data_dir: Path
+    scaler_path: Path
+
+@dataclass(frozen=True)
+class ModelTrainingConfig:
+    root_dir: Path
+    training_model_path: Path
+    training_data_path: Path
+    params_time_steps: int
+    params_epochs: int
+    params_batch_size: int
+    params_learning_rate: float
+    

tsForecaster/pipeline/stage_01_data_ingestion.py

@@ -0,0 +1,29 @@
+from tsForecaster.components.data_ingestion import DataIngestion
+from tsForecaster.config.configuration import ConfigurationManager
+from tsForecaster import logger
+
+STAGE_NAME = "Data Ingestion Stage"
+
+class DataIngestionPipeline:
+    def __init__(self) -> None:
+        pass
+    
+    def main(self):
+        try:
+            config = ConfigurationManager()
+            data_ingestion_config = config.get_data_ingestion_config()
+            data_ingestion = DataIngestion(config=data_ingestion_config)
+            data_ingestion.download_file()
+            data_ingestion.update_file()
+        except Exception as e:
+            raise e
+
+if __name__ == '__main__':
+    try:
+        logger.info(f">>>>> stage {STAGE_NAME} started <<<<<")
+        obj = DataIngestionPipeline()
+        obj.main()
+        logger.info(f">>>>> stage {STAGE_NAME} completed <<<<<")
+    except Exception as e:
+        logger.exception(e)
+        raise e

tsForecaster/pipeline/stage_02_data_preprocessing.py

@@ -0,0 +1,36 @@
+from tsForecaster.components.data_preprocessing import DataPreProcessing
+from tsForecaster import logger
+from tsForecaster.config.configuration import ConfigurationManager
+
+STAGE_NAME = "Data Pre-Processing Stage"
+
+class DataPreProcessingPipeline:
+    def __init__(self) -> None:
+        pass
+    
+    def main(self):
+        try:
+            config = ConfigurationManager()
+            data_ingestion_config = config.get_data_ingestion_config()
+            data_preprocessing = DataPreProcessing(config=data_ingestion_config)
+            df = data_preprocessing.process_csv()
+            close_df = df['Close']
+            df.drop(columns=['Close'], inplace=True)
+            scaled_df = data_preprocessing.scaling_data(df)
+            time_steps = 60
+            X, y, dates = data_preprocessing.create_sequences(scaled_df, time_steps)
+            X_train, X_val, X_test, y_train, y_val, y_test, dates_train, dates_val, dates_test = data_preprocessing.train_test_split(X, y, dates, train_len=0.8, val_len=0.1, test_len=0.1)
+            
+            return X_train, X_val, X_test, y_train, y_val, y_test, dates_train, dates_val, dates_test, close_df
+        except Exception as e:
+            raise e
+
+if __name__ == '__main__':
+    try:
+        logger.info(f">>>>> stage {STAGE_NAME} started <<<<<")
+        obj = DataPreProcessingPipeline()
+        obj.main()
+        logger.info(f">>>>> stage {STAGE_NAME} completed <<<<<")
+    except Exception as e:
+        logger.exception(e)
+        raise e

tsForecaster/pipeline/stage_03_model_training.py

@@ -0,0 +1,28 @@
+from tsForecaster.components.model_training import ModelTraining
+from tsForecaster.config.configuration import ConfigurationManager
+from tsForecaster import logger
+
+STAGE_NAME = "Model Training"
+
+class ModelTrainingPipeline:
+    def __init__(self) -> None:
+        pass
+    
+    def main(self, X_train, y_train, X_val, y_val):
+        try:
+            config = ConfigurationManager()
+            model_training_config = config.get_model_training_config()
+            model_training = ModelTraining(config=model_training_config)
+            model_training.train(X_train, y_train, X_val, y_val)
+        except Exception as e:
+            raise e
+
+if __name__ == '__main__':
+    try:
+        logger.info(f">>>>> stage {STAGE_NAME} started <<<<<")
+        obj = ModelTrainingPipeline()
+        history = obj.main()
+        logger.info(f">>>>> stage {STAGE_NAME} completed <<<<<")
+    except Exception as e:
+        logger.exception(e)
+        raise e

tsForecaster/pipeline/stage_04_model_evaluation.py

@@ -0,0 +1,30 @@
+import os
+from tsForecaster import logger
+from tsForecaster.components.model_evaluation import ModelEvaluation
+from tsForecaster.config.configuration import ConfigurationManager
+
+STAGE_NAME = "Model Evaluation Stage"
+
+class ModelEvaluationPipeline:
+    def __init__(self) -> None:
+
+        pass
+    
+    def main(self, X, y, dates):
+        try:
+            config = ConfigurationManager()
+            eval_config = config.get_model_training_config()
+            eval = ModelEvaluation(eval_config)
+            eval.evaluation(X, y, dates)
+        except Exception as e:
+            raise e
+
+if __name__ == '__main__':
+    try:
+        logger.info(f">>>>> stage {STAGE_NAME} started <<<<<")
+        obj = ModelEvaluationPipeline()
+        obj.main()
+        logger.info(f">>>>> stage {STAGE_NAME} completed <<<<<")
+    except Exception as e:
+        logger.exception(e)
+        raise e

tsForecaster/utils/common.py

@@ -0,0 +1,191 @@
+import os
+from box.exceptions import BoxValueError
+import yaml
+from tsForecaster import logger
+import json
+import joblib
+from ensure import ensure_annotations
+from box import ConfigBox
+from pathlib import Path
+from typing import Any
+import base64
+
+@ensure_annotations
+def read_yaml(path_to_yaml: Path) -> ConfigBox:
+    """
+    Read and load data from a YAML file.
+
+    Args:
+        path_to_yaml (Path): Path to the YAML file.
+
+    Returns:
+        Box: A Box object containing the YAML file data.
+
+    Raises:
+        ValueError: If the YAML file is empty or cannot be loaded.
+        Exception: For any other unexpected errors during file reading or parsing.
+    """
+    try:
+        with open(path_to_yaml) as yaml_file:
+            content = yaml.safe_load(yaml_file)
+            logger.info(f"yaml file {path_to_yaml} loaded successfully")
+            return ConfigBox(content)
+    except BoxValueError:
+        raise ValueError("Yaml file is empty")
+    except Exception as e:
+        raise e
+
+@ensure_annotations
+def create_directories(path_to_directories: list, verbose: bool=True):
+    """
+    Create directories specified in the list.
+
+    Args:
+        path_to_directories (list): List of directory paths to create.
+        verbose (bool, optional): Whether to log directory creation (default is True).
+
+    Raises:
+        Exception: For any unexpected errors during directory creation.
+    """
+    try:
+        for dirs in path_to_directories:
+            os.makedirs(dirs, exist_ok=True)
+            if verbose:
+                logger.info(f"Created directory at: {path_to_directories}")
+    except Exception as e:
+        raise e
+
+@ensure_annotations
+def save_json(path: Path, data: dict):
+    """
+    Save JSON data to a file.
+
+    Args:
+        path (Path): Path to the JSON file.
+        data (dict): Dictionary containing JSON serializable data.
+
+    Raises:
+        Exception: For any unexpected errors during file saving.
+    """
+    try:
+        with open(path, 'w') as f:
+            json.dump(data, f, indent=4)
+        logger.info(f"Json saved at: {path}")
+    except Exception as e:
+        raise e
+
+@ensure_annotations
+def load_json(path: Path) -> ConfigBox:
+    """
+    Load JSON data from a file and return as a Box object.
+
+    Args:
+        path (Path): Path to the JSON file.
+
+    Returns:
+        Box: A Box object containing the JSON data.
+
+    Raises:
+        Exception: For any unexpected errors during file loading.
+    """
+    try:
+        with open(path, 'r') as f:
+            content = json.load(f)
+        logger.info(f"Successfully loaded json from: {path}")
+        return ConfigBox(content)
+    except Exception as e:
+        raise e
+
+@ensure_annotations
+def save_bin(data: Any, path: Path) -> None:
+    """
+    Save binary data using joblib.
+
+    Args:
+        data (Any): Data to be saved.
+        path (Path): Path to save the binary file.
+
+    Raises:
+        Exception: For any unexpected errors during file saving.
+    """
+    try:
+        joblib.dump(value=data, filename=path)
+        logger.info(f"Binary file saved at: {path}")
+    except Exception as e:
+        raise e
+
+@ensure_annotations
+def load_bin(path: Path):
+    """
+    Load binary data using joblib.
+
+    Args:
+        path (Path): Path to the binary file.
+
+    Returns:
+        Any: Loaded binary data.
+
+    Raises:
+        Exception: For any unexpected errors during file loading.
+    """
+    joblib.load(filename=path)
+    logger.info(f"Successfully loaded binary from: {path}")
+
+@ensure_annotations
+def get_size(path: Path) -> str:
+    """
+    Get the size of a file in kilobytes.
+
+    Args:
+        path (Path): Path to the file.
+
+    Returns:
+        str: Size of the file in kilobytes formatted as "{size} kb".
+
+    Raises:
+        Exception: For any unexpected errors during file size retrieval.
+    """
+    try:
+        size_in_kb = round(os.path.getsize(path)/1024, 2)
+        return f"{size_in_kb} kb"
+    except Exception as e:
+        raise e
+
+def decodeImage(imgString, fileName) -> None:
+    """
+    Decode base64 encoded image data and save it to a file.
+
+    Args:
+        imgString (str): Base64 encoded image data.
+        fileName (str): Name of the file to save the decoded image data.
+
+    Raises:
+        Exception: For any unexpected errors during decoding or file writing.
+    """
+    try:
+        imgData = base64.b64decode(imgString)
+        with open(fileName, 'wb') as f:
+            f.write(imgData)
+            f.close
+    except Exception as e:
+        raise e
+
+def encodeImage(imagePath) -> bytes:
+    """
+    Encode an image file to base64 bytes.
+
+    Args:
+        imagePath (str): Path to the image file.
+
+    Returns:
+        bytes: Base64 encoded bytes of the image data.
+
+    Raises:
+        Exception: For any unexpected errors during encoding or file reading.
+    """
+    try:
+        with open(imagePath, 'rb') as f:
+            imgData = f.read()
+            return base64.b64encode(imgData)
+    except Exception as e:
+        raise e