"""
# Import Necessary Libraries
import pandas as pd
import numpy as np
import yfinance as yf
from pathlib import Path
import logging
import os
from datetime import datetime
import pandas_datareader.data as web
import matplotlib.pyplot as plt
import pandas_ta as ta  # Install using: pip install pandas_ta
from tqdm import tqdm  # For progress bars

# Setup Logging
def setup_logging(log_file='logs/data_processing.log'):
    """
    Configure logging settings to log INFO and higher-level messages to a file.
    """
    # Create the logs directory if it doesn't exist
    log_path = Path(log_file)
    log_path.parent.mkdir(parents=True, exist_ok=True)

    # Configure logging
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s:%(levelname)s:%(message)s',
        handlers=[
            logging.FileHandler(log_file),
            logging.StreamHandler()  # Also output to the console
        ]
    )

# Initialize Logging
setup_logging()
logger = logging.getLogger()

# Define Paths
RAW_DATA_PATH = Path('data/raw_data')
PROCESSED_DATA_PATH = Path('data/processed_data')
MACRO_DATA_PATH = PROCESSED_DATA_PATH / 'macro'
TECHNICAL_DATA_PATH = PROCESSED_DATA_PATH / 'technical'

# Create Necessary Directories
def create_directories(directories):
    """
    Create directories for storing processed data.
    """
    for directory in directories:
        Path(directory).mkdir(parents=True, exist_ok=True)
        logging.info(f"Directory created or already exists: {directory}")

directories = [
    MACRO_DATA_PATH,
    TECHNICAL_DATA_PATH,
    'logs'
]
create_directories(directories)

# Function to Fetch Macroeconomic Data from FRED
def fetch_macro_data():
    """
    Fetch macroeconomic indicators from FRED and save to CSV files.
    """
    logger.info("Fetching macroeconomic data from FRED.")
    start_date = '2010-01-01'
    end_date = datetime.today().strftime('%Y-%m-%d')

    # Define the indicators and their FRED codes
    macro_indicators = {
        'FEDFUNDS': 'Federal_Funds_Rate',
        'GDP': 'GDP',
        'CPIAUCSL': 'CPI'  # Consumer Price Index for All Urban Consumers
    }

    for code, name in macro_indicators.items():
        try:
            logger.info(f"Fetching {name}")
            data = web.DataReader(code, 'fred', start_date, end_date)
            data.reset_index(inplace=True)
            file_path = MACRO_DATA_PATH / f"{name}.csv"
            data.to_csv(file_path, index=False)
            logger.info(f"Saved {name} data to {file_path}")
        except Exception as e:
            logger.error(f"Error fetching {name}: {e}")

# Function to Read and Process Price Data
def process_price_data():
    """
    Read raw price data CSVs, clean data, calculate technical indicators,
    and save processed data.
    """
    logger.info("Processing price data and calculating technical indicators.")
    timeframes = ['1d', '1wk', '1mo']
    data_types = ['in_sample', 'out_sample']

    for data_type in data_types:
        for timeframe in timeframes:
            raw_data_dir = RAW_DATA_PATH / data_type / timeframe
            processed_data_dir = TECHNICAL_DATA_PATH / data_type / timeframe
            processed_data_dir.mkdir(parents=True, exist_ok=True)

            csv_files = list(raw_data_dir.glob('*.csv'))
            logger.info(f"Processing {len(csv_files)} files in {raw_data_dir}")

            for csv_file in tqdm(csv_files, desc=f"{data_type}/{timeframe}"):
                try:
                    ticker = csv_file.stem
                    df = pd.read_csv(csv_file, parse_dates=['Date'])
                    df.sort_values('Date', inplace=True)

                    # Handle missing values
                    df.fillna(method='ffill', inplace=True)
                    df.dropna(inplace=True)

                    # Verify adjusted close prices
                    if 'Adj Close' not in df.columns:
                        df['Adj Close'] = df['Close']
                    else:
                        # Adjust Close should already account for splits/dividends
                        pass  # Additional verification can be added if necessary

                    # Calculate Technical Indicators
                    df.set_index('Date', inplace=True)

                    # Moving Averages
                    df['SMA_20'] = df['Adj Close'].rolling(window=20).mean()
                    df['SMA_50'] = df['Adj Close'].rolling(window=50).mean()
                    df['EMA_20'] = df['Adj Close'].ewm(span=20, adjust=False).mean()

                    # Relative Strength Index (RSI)
                    df['RSI_14'] = ta.rsi(df['Adj Close'], length=14)

                    # Moving Average Convergence Divergence (MACD)
                    macd = ta.macd(df['Adj Close'])
                    df['MACD'] = macd['MACD_12_26_9']
                    df['MACD_signal'] = macd['MACDs_12_26_9']

                    # Bollinger Bands
                    bb = ta.bbands(df['Adj Close'], length=20, std=2)
                    df['BB_upper'] = bb['BBU_20_2.0']
                    df['BB_middle'] = bb['BBM_20_2.0']
                    df['BB_lower'] = bb['BBL_20_2.0']

                    # Average True Range (ATR)
                    df['ATR_14'] = ta.atr(df['High'], df['Low'], df['Close'], length=14)

                    # Reset index to save Date as a column
                    df.reset_index(inplace=True)

                    # Save processed data
                    output_file = processed_data_dir / f"{ticker}.csv"
                    df.to_csv(output_file, index=False)
                    logger.info(f"Processed data saved for {ticker} at {output_file}")

                except Exception as e:
                    logger.error(f"Error processing {csv_file}: {e}")

# Function to Validate Data
def validate_data():
    """
    Perform data validation checks to ensure data integrity.
    """
    logger.info("Validating processed data.")
    timeframes = ['1d', '1wk', '1mo']
    data_types = ['in_sample', 'out_sample']

    for data_type in data_types:
        for timeframe in timeframes:
            processed_data_dir = TECHNICAL_DATA_PATH / data_type / timeframe
            csv_files = list(processed_data_dir.glob('*.csv'))

            for csv_file in tqdm(csv_files, desc=f"Validating {data_type}/{timeframe}"):
                try:
                    df = pd.read_csv(csv_file, parse_dates=['Date'])
                    ticker = csv_file.stem

                    # Check for anomalies
                    if df.isnull().values.any():
                        logger.warning(f"Missing values detected in {ticker} data.")
                    if df.duplicated(subset=['Date']).any():
                        logger.warning(f"Duplicate dates found in {ticker} data.")
                    # Additional validation checks can be added here

                except Exception as e:
                    logger.error(f"Error validating {csv_file}: {e}")

# Main Function to Execute All Steps
def main():
    """
    Main function to execute data collection, processing, and validation.
    """
    # Step 1: Fetch Macroeconomic Data
    fetch_macro_data()

    # Step 2: Process Price Data and Calculate Technical Indicators
    process_price_data()

    # Step 3: Validate Processed Data
    validate_data()

# Execute the Main Function
if __name__ == "__main__":
    main()
"""
Based on the provided log, your data processing pipeline encountered several issues that could impact the integrity and completeness of your processed datasets. Here's a detailed analysis of the potential errors and warnings observed in the log:

---

## 1. Critical Errors During Data Processing

### a. Error Processing DUSL.csv

- Log Entry:


  2024-09-19 08:49:40,830:ERROR:Error processing data/raw_data/in_sample/1mo/DUSL.csv: unsupported operand type(s) for -: 'float' and 'NoneType'


- Explanation:
  - Issue: The error message unsupported operand type(s) for -: 'float' and 'NoneType' indicates that the code attempted to perform a subtraction operation between a float and a NoneType object.
  - Possible Cause: This typically occurs when a value expected to be a numerical (float) is missing or undefined (None). In the context of processing DUSL.csv, it's likely that one of the columns or data points intended for numerical operations contains missing or None values.
  - Impact: The processing of DUSL.csv failed, meaning that any technical indicators or processed data derived from this file are unavailable or incomplete. This could affect downstream analyses or models that rely on this specific dataset.

### b. Error Processing XLC.csv

- Log Entry:

  2024-09-19 08:49:40,907:ERROR:Error processing data/raw_data/in_sample/1mo/XLC.csv: 'NoneType' object is not subscriptable


- Explanation:
  - Issue: The error 'NoneType' object is not subscriptable suggests that the code attempted to access elements (using indexing) of a variable that is None.
  - Possible Cause: During the processing of XLC.csv, a function or operation expected to return a data structure (like a list, tuple, or dictionary) returned None instead. When the code tried to access elements of this None object (e.g., none_object[0]), it resulted in the error.
  - Impact: Similar to the previous error, the processing of XLC.csv was interrupted, leading to missing or incomplete processed data for this file. This omission can have repercussions on any analyses or models that incorporate XLC.csv data.

---

## 2. Warnings About Missing Values in Validated Data

After processing, the pipeline performs a validation step where it checks for missing values in the processed datasets. The log records numerous warnings indicating the presence of missing values across various data files.

### a. General Pattern of Warnings

- Log Entries:

  WARNING:Missing values detected in FNGU data.
  WARNING:Missing values detected in TQQQ data.
  WARNING:Missing values detected in SDP data.
  ...


- Explanation:
  - Issue: The consistent WARNING:Missing values detected in &lt;Symbol&gt; data. messages indicate that several datasets contain missing (NaN or None) values.
  - Possible Causes:
    - Data Quality Issues: The raw CSV files might have incomplete records, missing entries, or formatting issues that led to missing data during processing.
    - Processing Errors: Earlier processing steps, such as feature calculations or data transformations, might have inadvertently introduced NaN or None values if certain conditions weren't handled properly (e.g., division by zero, invalid operations).
    - Incomplete Data Retrieval: If data fetching from external sources (like FRED for macroeconomic data) was interrupted or incomplete, it could result in missing entries in the processed datasets.
  
- Impact:
  - Analytical Accuracy: Missing values can skew statistical analyses, machine learning model training, and other data-driven processes, leading to inaccurate or unreliable results.
  - Downstream Processes: Subsequent steps that rely on complete datasets (e.g., data visualization, reporting, further data transformations) might fail or produce incomplete outputs.

---

## 3. Summary of Affected Datasets and Scope

- Datasets with Critical Errors:
  - in_sample/1mo/DUSL.csv
  - in_sample/1mo/XLC.csv

- Datasets with Missing Values:
  - In-Sample Data (1d, 1wk, 1mo): Multiple symbols including FNGU, TQQQ, SDP, SOXL, FAS, SIJ, XLV, ERX, SOXS, XLE, UYM, UPW, RETL, XRT, XTL, XLU, UPRO, FAZ, ERY, XLI, XLB, VIX, CURE, SCC, FNGD, XLK, XLY, UVXY, XLC
  - Out-of-Sample Data (1d, 1wk, 1mo): Similar range of symbols showing missing values.

---

## 4. Recommendations to Address the Issues

### a. Addressing Critical Errors

1. Debug the Processing Code:
   - For DUSL.csv: Investigate the specific processing function or formula that involves subtraction operations. Ensure that all operands are validated for None or NaN values before performing arithmetic operations.
   - For XLC.csv: Examine where the NoneType error originates. Ensure that all data structures expected to contain data are properly initialized and populated before subscripting.

2. Implement Robust Error Handling:
   - Incorporate try-except blocks around critical operations to gracefully handle unexpected None or NaN values.
   - Log more detailed error messages or stack traces to pinpoint the exact location and cause of failures.

3. Data Validation Prior to Processing:
   - Perform preliminary checks on raw data to ensure completeness and correctness before initiating processing routines.
   - Fill or impute missing values where appropriate, or exclude incomplete records based on the use case.

### b. Resolving Missing Values in Validated Data

1. Data Cleaning:
   - Imputation: Use statistical methods (e.g., mean, median, mode) or model-based techniques to estimate and fill missing values.
   - Removal: Exclude records with missing values if imputation isn't feasible or could introduce significant bias.

2. Enhance Data Processing Pipelines:
   - Ensure that all data transformations handle missing data gracefully. For instance, using functions that can operate on NaN values without breaking (e.g., pandas' functions which inherently handle NaN).
   - Validate intermediate outputs to catch and address missing data early in the pipeline.

3. Improve Data Quality at Source:
   - If missing values stem from external data sources (like FRED), consider adding checks to verify data completeness after retrieval.
   - Automate alerts or flags when fetched data has unexpected gaps or anomalies.

4. Documentation and Logging Enhancements:
   - Clearly document how missing data is handled within the pipeline to ensure transparency.
   - Enhance logging to include counts or proportions of missing values for each dataset, aiding in comprehensive monitoring and troubleshooting.

---

## 5. Potential Long-Term Solutions

- Automated Testing: Implement unit and integration tests to verify that data processing functions handle edge cases, including missing or malformed data.
  
- Monitoring and Alerts: Set up monitoring tools that can alert the team when critical errors or high levels of missing data are detected during processing.

- Data Integrity Checks: Regularly audit processed data to ensure consistency, completeness, and correctness, adapting the pipeline as data sources or requirements evolve.

---

By addressing the identified errors and warnings, you can enhance the reliability and quality of your data processing pipeline, ensuring more accurate and dependable outcomes for your analyses and models.