# Initialize population
print("Initializing population...")
population = toolbox.population(n=POPULATION_SIZE)

# Initialize statistics to keep track
stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("avg", np.mean)
stats.register("std", np.std)
stats.register("min", np.min)
stats.register("max", np.max)

# Initialize Hall of Fame to store the best individual
hof = tools.HallOfFame(HALL_OF_FAME_SIZE)

# Begin the evolution
print("\nStarting Genetic Algorithm Evolution...\n")
for gen in tqdm(range(1, MAX_GENERATIONS + 1), desc="Generations"):
    # Select the next generation individuals
    offspring = toolbox.select(population, len(population))
    offspring = list(map(toolbox.clone, offspring))
    
    # Apply crossover on the offspring
    for child1, child2 in zip(offspring[::2], offspring[1::2]):
        if random.random() < P_CROSSOVER:
            toolbox.mate(child1, child2)
            del child1.fitness.values
            del child2.fitness.values
    
    # Apply mutation on the offspring
    for mutant in offspring:
        if random.random() < P_MUTATION:
            toolbox.mutate(mutant)
            del mutant.fitness.values
    
    # Evaluate the individuals with an invalid fitness
    invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
    fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
    for ind, fit in zip(invalid_ind, fitnesses):
        ind.fitness.values = fit
    
    # Replace population with offspring
    population[:] = offspring
    
    # Update the hall of fame with the generated individuals
    hof.update(population)
    
    # Gather all the fitnesses in one list
    fits = [ind.fitness.values[0] for ind in population]
    
print("\nEvolution Complete!\n")

# Retrieve and display the best portfolio found
if len(hof) > 0:
    best_ind = hof[0]
    selected = best_ind[:NUM_ASSETS]
    weights = np.array(best_ind[NUM_ASSETS:])
    
    # Training Metrics
    portfolio_returns_train = train_returns[selected].dot(weights)
    portfolio_cum_return_train = (1 + portfolio_returns_train).prod() - 1
    portfolio_volatility_train = portfolio_returns_train.std() * np.sqrt(252)
    sharpe_ratio_train = (portfolio_returns_train.mean() * 252 - RISK_FREE_RATE) / portfolio_volatility_train if portfolio_volatility_train != 0 else 0
    sortino_growth_train = sortino_ratio(portfolio_returns_train[train_labels == 'Growth'])
    sortino_recession_train = sortino_ratio(portfolio_returns_train[train_labels == 'Recession'])
    annualized_return_train = portfolio_returns_train.mean() * 252
    portfolio_cum_returns_growth_train = (1 + portfolio_returns_train[train_labels == 'Growth']).cumprod()
    portfolio_cum_returns_recession_train = (1 + portfolio_returns_train[train_labels == 'Recession']).cumprod()
    portfolio_max_drawdown_growth_train = max_drawdown(portfolio_cum_returns_growth_train)
    portfolio_max_drawdown_recession_train = max_drawdown(portfolio_cum_returns_recession_train)
    
    # Validation Metrics
    try:
        portfolio_returns_val = validation_returns[selected].dot(weights)
        portfolio_cum_return_val = (1 + portfolio_returns_val).prod() - 1
        portfolio_volatility_val = portfolio_returns_val.std() * np.sqrt(252)
        sharpe_ratio_val = (portfolio_returns_val.mean() * 252 - RISK_FREE_RATE) / portfolio_volatility_val if portfolio_volatility_val != 0 else 0
        sortino_growth_val = sortino_ratio(portfolio_returns_val[validation_labels == 'Growth'])
        sortino_recession_val = sortino_ratio(portfolio_returns_val[validation_labels == 'Recession'])
        annualized_return_val = portfolio_returns_val.mean() * 252
        portfolio_cum_returns_growth_val = (1 + portfolio_returns_val[validation_labels == 'Growth']).cumprod()
        portfolio_cum_returns_recession_val = (1 + portfolio_returns_val[validation_labels == 'Recession']).cumprod()
        portfolio_max_drawdown_growth_val = max_drawdown(portfolio_cum_returns_growth_val)
        portfolio_max_drawdown_recession_val = max_drawdown(portfolio_cum_returns_recession_val)
    except:
        portfolio_returns_val = 0
        portfolio_cum_return_val = 0
        portfolio_volatility_val = 0
        sharpe_ratio_val = 0
        sortino_growth_val = 0
        sortino_recession_val = 0
        annualized_return_val = 0
        portfolio_cum_returns_growth_val = 0
        portfolio_cum_returns_recession_val = 0
        portfolio_max_drawdown_growth_val = 0
        portfolio_max_drawdown_recession_val = 0
    
    # Testing Metrics
    try:
        portfolio_returns_test = test_returns[selected].dot(weights)
        portfolio_cum_return_test = (1 + portfolio_returns_test).prod() - 1
        portfolio_volatility_test = portfolio_returns_test.std() * np.sqrt(252)
        sharpe_ratio_test = (portfolio_returns_test.mean() * 252 - RISK_FREE_RATE) / portfolio_volatility_test if portfolio_volatility_test != 0 else 0
        sortino_test_growth = sortino_ratio(portfolio_returns_test[test_labels == 'Growth'])
        sortino_test_recession = sortino_ratio(portfolio_returns_test[test_labels == 'Recession'])
        annualized_return_test = portfolio_returns_test.mean() * 252
        portfolio_cum_returns_growth_test = (1 + portfolio_returns_test[test_labels == 'Growth']).cumprod()
        portfolio_cum_returns_recession_test = (1 + portfolio_returns_test[test_labels == 'Recession']).cumprod()
        portfolio_max_drawdown_growth_test = max_drawdown(portfolio_cum_returns_growth_test)
        portfolio_max_drawdown_recession_test = max_drawdown(portfolio_cum_returns_recession_test)
    except:
        portfolio_returns_test = 0
        portfolio_cum_return_test = 0
        portfolio_volatility_test = 0
        sharpe_ratio_test = 0
        sortino_test_growth = 0
        sortino_test_recession = 0
        annualized_return_test = 0
        portfolio_cum_returns_growth_test = 0
        portfolio_cum_returns_recession_test = 0
        portfolio_max_drawdown_growth_test = 0
        portfolio_max_drawdown_recession_test = 0
    
    # Composite Fitness Score on Training and Validation Sets
    fitness_train = (
        WEIGHT_SHARPE * sharpe_ratio_train +
        WEIGHT_SORTINO_GROWTH * sortino_growth_train +
        WEIGHT_SORTINO_RECESSION * sortino_recession_train +
        WEIGHT_VOLATILITY * (-portfolio_volatility_train) +
        WEIGHT_ANNUALIZED_RETURN * annualized_return_train +
        WEIGHT_MAX_DRAWDOWN * (portfolio_max_drawdown_growth_train + portfolio_max_drawdown_recession_train) / 2
    )
    
    fitness_val = (
        WEIGHT_SHARPE * sharpe_ratio_val +
        WEIGHT_SORTINO_GROWTH * sortino_growth_val +
        WEIGHT_SORTINO_RECESSION * sortino_recession_val +
        WEIGHT_VOLATILITY * (-portfolio_volatility_val) +
        WEIGHT_ANNUALIZED_RETURN * annualized_return_val +
        WEIGHT_MAX_DRAWDOWN * (portfolio_max_drawdown_growth_val + portfolio_max_drawdown_recession_val) / 2
    )
    
    fitness = (fitness_train + fitness_val) / 2
    
    portfolio_df = pd.DataFrame({
        'Ticker': selected,
        'Allocation': [f"{w*100:.2f}%" for w in weights]
    })
    
    print("Optimal Portfolio Found:")
    display(portfolio_df)
    print(f"--- Training Set Performance ---")
    print(f"Cumulative Return: {portfolio_cum_return_train*100:.2f}%")
    print(f"Annualized Volatility: {portfolio_volatility_train*100:.2f}%")
    print(f"Sharpe Ratio: {sharpe_ratio_train:.2f}")
    print(f"Sortino Ratio (Growth): {sortino_growth_train:.2f}")
    print(f"Sortino Ratio (Recession): {sortino_recession_train:.2f}")
    print(f"Annualized Return: {annualized_return_train*100:.2f}%")
    print(f"Maximum Drawdown (Growth): {portfolio_max_drawdown_growth_train:.2%}")
    print(f"Maximum Drawdown (Recession): {portfolio_max_drawdown_recession_train:.2%}")
    
    print(f"--- Validation Set Performance ---")
    print(f"Cumulative Return: {portfolio_cum_return_val*100:.2f}%")
    print(f"Annualized Volatility: {portfolio_volatility_val*100:.2f}%")
    print(f"Sharpe Ratio: {sharpe_ratio_val:.2f}")
    print(f"Sortino Ratio (Growth): {sortino_growth_val:.2f}")
    print(f"Sortino Ratio (Recession): {sortino_recession_val:.2f}")
    print(f"Annualized Return: {annualized_return_val*100:.2f}%")
    print(f"Maximum Drawdown (Growth): {portfolio_max_drawdown_growth_val:.2%}")
    print(f"Maximum Drawdown (Recession): {portfolio_max_drawdown_recession_val:.2%}")
    
    print(f"--- Testing Set Performance ---")
    print(f"Cumulative Return: {portfolio_cum_return_test*100:.2f}%")
    print(f"Annualized Volatility: {portfolio_volatility_test*100:.2f}%")
    print(f"Sharpe Ratio: {sharpe_ratio_test:.2f}")
    print(f"Sortino Ratio (Growth): {sortino_test_growth:.2f}")
    print(f"Sortino Ratio (Recession): {sortino_test_recession:.2f}")
    print(f"Annualized Return: {annualized_return_test*100:.2f}%")
    print(f"Maximum Drawdown (Growth): {portfolio_max_drawdown_growth_test:.2%}")
    print(f"Maximum Drawdown (Recession): {portfolio_max_drawdown_recession_test:.2%}")
    
    # Assess overfitting
    print("\n--- Overfitting Assessment ---")
    # Define thresholds for acceptable performance differences
    threshold_return = 0.05  # 5% difference in return
    threshold_volatility = 0.05  # 5% difference in volatility

    train_val_return_diff = portfolio_cum_return_train - portfolio_cum_return_val
    train_test_return_diff = portfolio_cum_return_train - portfolio_cum_return_test if 'portfolio_cum_return_test' in locals() else 0
    train_val_vol_diff = portfolio_volatility_train - portfolio_volatility_val
    train_test_vol_diff = portfolio_volatility_train - portfolio_volatility_test if 'portfolio_volatility_test' in locals() else 0

    overfitting_return = train_test_return_diff > threshold_return
    overfitting_vol = train_test_vol_diff < -threshold_volatility

    if overfitting_return and overfitting_vol:
        print("Warning: Overfitting detected! The portfolio performs significantly better on training and validation data compared to testing data.")
    else:
        print("No significant overfitting detected. The portfolio generalizes well to unseen data.")
    
    # Plot cumulative returns for visual assessment
    plt.figure(figsize=(14, 7))
    
    # Training Cumulative Returns
    plt.subplot(1, 2, 1)
    if not portfolio_cum_returns_growth_train.empty:
        plt.plot(portfolio_cum_returns_growth_train.index, portfolio_cum_returns_growth_train, label='Training Growth')
    if not portfolio_cum_returns_recession_train.empty:
        plt.plot(portfolio_cum_returns_recession_train.index, portfolio_cum_returns_recession_train, label='Training Recession')
    if not portfolio_cum_returns_growth_val.empty:
        plt.plot(portfolio_cum_returns_growth_val.index, portfolio_cum_returns_growth_val, label='Validation Growth')
    if not portfolio_cum_returns_recession_val.empty:
        plt.plot(portfolio_cum_returns_recession_val.index, portfolio_cum_returns_recession_val, label='Validation Recession')
    if 'portfolio_cum_returns_growth_test' in locals() and not portfolio_cum_returns_growth_test.empty:
        plt.plot(portfolio_cum_returns_growth_test.index, portfolio_cum_returns_growth_test, label='Testing Growth')
    if 'portfolio_cum_returns_recession_test' in locals() and not portfolio_cum_returns_recession_test.empty:
        plt.plot(portfolio_cum_returns_recession_test.index, portfolio_cum_returns_recession_test, label='Testing Recession')
    plt.title('Cumulative Returns')
    plt.xlabel('Date')
    plt.ylabel('Cumulative Return')
    plt.legend()
    
    # Drawdowns
    plt.subplot(1, 2, 2)
    if not portfolio_cum_returns_growth_train.empty:
        plt.plot(portfolio_cum_returns_growth_train.index, (portfolio_cum_returns_growth_train / portfolio_cum_returns_growth_train.cummax()) - 1, label='Training Growth Drawdown')
    if not portfolio_cum_returns_recession_train.empty:
        plt.plot(portfolio_cum_returns_recession_train.index, (portfolio_cum_returns_recession_train / portfolio_cum_returns_recession_train.cummax()) - 1, label='Training Recession Drawdown')
    if not portfolio_cum_returns_growth_val.empty:
        plt.plot(portfolio_cum_returns_growth_val.index, (portfolio_cum_returns_growth_val / portfolio_cum_returns_growth_val.cummax()) - 1, label='Validation Growth Drawdown')
    if not portfolio_cum_returns_recession_val.empty:
        plt.plot(portfolio_cum_returns_recession_val.index, (portfolio_cum_returns_recession_val / portfolio_cum_returns_recession_val.cummax()) - 1, label='Validation Recession Drawdown')
    if 'portfolio_cum_returns_growth_test' in locals() and not portfolio_cum_returns_growth_test.empty:
        plt.plot(portfolio_cum_returns_growth_test.index, (portfolio_cum_returns_growth_test / portfolio_cum_returns_growth_test.cummax()) - 1, label='Testing Growth Drawdown')
    if 'portfolio_cum_returns_recession_test' in locals() and not portfolio_cum_returns_recession_test.empty:
        plt.plot(portfolio_cum_returns_recession_test.index, (portfolio_cum_returns_recession_test / portfolio_cum_returns_recession_test.cummax()) - 1, label='Testing Recession Drawdown')
    plt.title('Drawdowns')
    plt.xlabel('Date')
    plt.ylabel('Drawdown')
    plt.legend()
    
    plt.tight_layout()
    plt.show()

correct the code so that it doesnt use cumulative return, use CAGR instead