# Import required packages
!pip install yfinance pandas numpy beautifulsoup4 requests tqdm deap matplotlib lxml

# Import necessary libraries
import yfinance as yf
import pandas as pd
import numpy as np
import random
from bs4 import BeautifulSoup
import requests
from tqdm import tqdm
import sys
import warnings
import multiprocessing
import matplotlib.pyplot as plt

from deap import base, creator, tools, algorithms

# Suppress warnings for cleaner output
warnings.filterwarnings("ignore")

# Set random seed for reproducibility
RANDOM_SEED = 42
random.seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)

# Function to fetch S&P 500 tickers from Wikipedia
def get_sp500_tickers():
    url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
    response = requests.get(url)
    if response.status_code != 200:
        raise Exception("Failed to fetch S&P 500 tickers from Wikipedia.")
    soup = BeautifulSoup(response.text, "lxml")
    table = soup.find("table", {"id": "constituents"})
    tickers = []
    for row in table.findAll("tr")[1:]:
        ticker = row.findAll("td")[0].text.strip()
        # Replace '.' with '-' for tickers like BRK.B
        ticker = ticker.replace(".", "-")
        tickers.append(ticker)
    return tickers

# Function to calculate metrics
def max_drawdown(series):
    roll_max = series.cummax()
    drawdown = (series - roll_max) / roll_max
    return drawdown.min()

def sortino_ratio(returns, target=0):
    downside = returns[returns < target]
    if len(downside) == 0:
        return 0  # No downside risk; assign a neutral value
    expected_return = returns.mean() * 252
    downside_risk = np.sqrt((downside ** 2).mean()) * np.sqrt(252)
    return (expected_return - target) / downside_risk if downside_risk != 0 else 0

def CAGR(returns):
    cumulative_returns = (1 + returns).prod()
    periods = len(returns) / 252  # Assumes 252 trading days per year
    return cumulative_returns ** (1 / periods) - 1

# Function to generate portfolio weights within constraints
def generate_weights(n, lower=0.10, upper=0.20, max_attempts=1000):
    for _ in range(max_attempts):
        weights = np.random.uniform(lower, upper, n)
        weights /= weights.sum()
        if all(lower <= w <= upper for w in weights):
            return weights
    raise ValueError(
        "Unable to generate weights within constraints after multiple attempts."
    )

# Function to label economic cycles
def label_economic_cycles(dates):
    recession_periods = [
        ("2001-03-01", "2001-11-30"),
        ("2007-12-01", "2009-06-30"),
        ("2020-02-01", "2020-04-30"),
    ]
    labels = []
    for date in dates:
        # Remove timezone information if present
        if date.tzinfo is not None:
            date = date.replace(tzinfo=None)
        label = "Growth"
        for start, end in recession_periods:
            if pd.to_datetime(start) <= date <= pd.to_datetime(end):
                label = "Recession"
                break
        labels.append(label)
    return labels

# Parameters
START_DATE = "2000-01-01"
END_DATE = "2024-10-30"
NUM_ASSETS = 8  # Number of stocks in the portfolio
LOWER_WEIGHT = 0.10
UPPER_WEIGHT = 0.20
RISK_FREE_RATE = 0.017  # Approximate risk-free rate (e.g., 10-year US Treasury yield)

# Step 1: Get S&P 500 tickers
print("Fetching S&P 500 tickers...")
tickers = get_sp500_tickers()
print(f"Number of tickers fetched: {len(tickers)}")

# Step 2: Download historical data with dynamic start date
print("Downloading historical data...")
failed_tickers = []
successful_tickers = []
data = pd.DataFrame()

ticker_first_dates = {}

for ticker in tqdm(tickers, desc="Downloading tickers"):
    try:
        df = yf.download(
            ticker, start=START_DATE, end=END_DATE, progress=False
        )["Adj Close"]
        if df.empty:
            failed_tickers.append(ticker)
        else:
            data[ticker] = df
            successful_tickers.append(ticker)
            ticker_first_dates[ticker] = df.first_valid_index()
    except Exception as e:
        failed_tickers.append(ticker)

print(f"\nTotal tickers with successful data download: {len(successful_tickers)}")
print(f"Total tickers failed to download: {len(failed_tickers)}")
if failed_tickers:
    print(f"Failed tickers: {failed_tickers}")

# Determine the earliest start date among all successful tickers
if ticker_first_dates:
    earliest_date = min(ticker_first_dates.values())
    print(f"Earliest available data starts from: {earliest_date.date()}")

    # Adjust the dataset to start from the earliest_date
    data = data.loc[earliest_date:]

    # Forward fill to propagate last valid observation
    data.fillna(method="ffill", inplace=True)
    # Backward fill to handle any remaining NaNs at the start
    data.fillna(method="bfill", inplace=True)
    print(
        f"Data shape after adjusting start date and dropping incomplete data: {data.shape}"
    )
else:
    print("No data available after processing tickers.")
    sys.exit()

# If there are not enough tickers, exit
if len(data.columns) < NUM_ASSETS:
    print("Not enough tickers with complete data to form a portfolio.")
    sys.exit()

# Step 3: Calculate daily returns
returns = data.pct_change().dropna()

# Step 4: Label economic cycles
print("Labeling economic cycles...")
return_dates = returns.index
labels = label_economic_cycles(return_dates)
labels = pd.Series(labels, index=return_dates)  # Convert to Series with matching index

# Step 5: Split data into Training (70%), Validation (20%), Testing (10%)
split_date_1 = int(len(returns) * 0.70)
split_date_2 = int(len(returns) * 0.90)  # Adjust split for larger validation

train_returns = returns.iloc[:split_date_1]  # No 'Economic_Cycle' column
train_labels = labels.iloc[:split_date_1]  # Use iloc to ensure alignment

validation_returns = returns.iloc[split_date_1:split_date_2]
validation_labels = labels.iloc[split_date_1:split_date_2]

test_returns = returns.iloc[split_date_2:]
test_labels = labels.iloc[split_date_2:]