Hello, I have a strategy backtesting script that is currently setup to output results of series of experiments in a form of heatmap to optimize the parameters. Now I need to turn it into a more traditional backtesting framework that outputs the normal thigs like portfolio size by time, equity price by time, signals etc. Here is my current code: import numpy as np
import pandas as pd
import vectorbt as vbt
import ccxt
from datetime import datetime, timedelta
from numba import jit, njit, prange
import warnings
from concurrent.futures import ThreadPoolExecutor
from functools import partial
import time
from threading import Lock
import matplotlib.pyplot as plt
import seaborn as sns
import os
from itertools import product

warnings.filterwarnings('ignore')


@njit
def calculate_volume_profile(prices, volumes, bucket_count):
    """Calculate volume profile metrics using Numba optimization"""
    if len(prices) == 0 or len(volumes) == 0:
        return np.zeros(bucket_count), np.zeros(bucket_count)
    
    min_price = np.min(prices)
    max_price = np.max(prices)
    
    # Prevent division by zero
    if min_price == max_price:
        price_range = 1.0
    else:
        price_range = max_price - min_price
    
    bucket_size = price_range / bucket_count
    
    volume_profile = np.zeros(bucket_count)
    price_buckets = np.zeros(bucket_count)
    
    for i in range(len(prices)):
        if bucket_size > 0:
            bucket_idx = int((prices[i] - min_price) / bucket_size)
            if 0 <= bucket_idx < bucket_count:
                volume_profile[bucket_idx] += volumes[i]
                price_buckets[bucket_idx] = min_price + (bucket_idx + 0.5) * bucket_size
                
    return volume_profile, price_buckets


@njit
def identify_volume_nodes(volume_profile, price_buckets, threshold, min_distance):
    """Identify high volume nodes using Numba optimization with more lenient conditions"""
    if len(volume_profile) == 0 or len(price_buckets) == 0:
        return np.zeros((0, 2), dtype=np.float64)
    
    total_volume = np.sum(volume_profile)
    if total_volume == 0:
        return np.zeros((0, 2), dtype=np.float64)
        
    normalized_volume = volume_profile / total_volume
    
    # Pre-allocate maximum possible nodes
    max_nodes = len(volume_profile)
    temp_nodes = np.zeros((max_nodes, 2), dtype=np.float64)
    node_count = 0
    
    # Find peaks more leniently
    for i in range(1, len(volume_profile) - 1):
        is_peak = normalized_volume[i] > threshold
        
        # Check if it's a local maximum or close to being one
        if is_peak:
            if normalized_volume[i] >= normalized_volume[i-1] * 0.95 and normalized_volume[i] >= normalized_volume[i+1] * 0.95:
                temp_nodes[node_count, 0] = price_buckets[i]
                temp_nodes[node_count, 1] = normalized_volume[i]
                node_count += 1
    
    if node_count == 0:
        return np.zeros((0, 2), dtype=np.float64)
    
    # Create array with actual nodes
    nodes = temp_nodes[:node_count]
    
    # Sort nodes by price
    sorted_indices = np.argsort(nodes[:, 0])
    nodes = nodes[sorted_indices]
    
    # Filter nodes by minimum distance more leniently
    filtered_nodes = np.zeros((node_count, 2), dtype=np.float64)
    filtered_count = 0
    
    filtered_nodes[filtered_count] = nodes[0]
    filtered_count += 1
    
    for i in range(1, node_count):
        if abs(nodes[i, 0] - filtered_nodes[filtered_count-1, 0]) / filtered_nodes[filtered_count-1, 0] > min_distance:
            filtered_nodes[filtered_count] = nodes[i]
            filtered_count += 1
            
    return filtered_nodes[:filtered_count]


@njit(parallel=True)
def process_volume_nodes_numba(close_prices, volumes, window_size, price_bucket_count, 
                               volume_threshold, node_distance, volume_nodes_prices, 
                               volume_nodes_volumes, num_volume_nodes):
    n = len(close_prices)
    
    for i in prange(window_size, n):
        # Define the window for volume profile
        window_close = close_prices[i - window_size:i]
        window_volume = volumes[i - window_size:i]
        
        # Calculate volume profile
        profile, buckets = calculate_volume_profile(window_close, window_volume, price_bucket_count)
        
        # Identify volume nodes
        nodes = identify_volume_nodes(profile, buckets, volume_threshold, node_distance)
        
        num_nodes = nodes.shape[0]
        if num_nodes > price_bucket_count:
            num_nodes = price_bucket_count  # Cap to maximum nodes
        
        num_volume_nodes[i] = num_nodes
        
        for j in range(num_nodes):
            volume_nodes_prices[i, j] = nodes[j, 0]
            volume_nodes_volumes[i, j] = nodes[j, 1]
            

class VolumeProfileStrategy:
    def __init__(self, 
                 symbol,
                 exchange_id='binance',
                 price_bucket_count=100,  # Number of price buckets for volume profile
                 profile_period='1d',     # Period for calculating volume profile
                 volume_threshold=0.0001,     # Threshold for identifying significant volume nodes
                 node_distance=0.00001,       # Minimum distance between volume nodes
                 entry_confirmation_bars=3, # Bars needed for entry confirmation
                 exit_confirmation_bars=2,  # Bars needed for exit confirmation
                 risk_per_trade=0.02,
                 interval='5m',
                 max_workers=5):
        
        self.symbol = symbol
        self.price_bucket_count = price_bucket_count
        self.profile_period = profile_period
        self.volume_threshold = volume_threshold
        self.node_distance = node_distance
        self.entry_confirmation_bars = entry_confirmation_bars
        self.exit_confirmation_bars = exit_confirmation_bars
        self.risk_per_trade = risk_per_trade
        self.interval = interval
        self.max_workers = max_workers
        
        # Create exchange connection pool
        self.exchange_pool = [
            getattr(ccxt, exchange_id)({
                'enableRateLimit': True,
                'options': {'defaultType': 'future'}
            }) for _ in range(max_workers)
        ]
        self.pool_lock = Lock()
        self.pool_index = 0
        
        # Pre-compile Numba functions by running them with dummy data
        self._compile_numba_functions()

    def _get_exchange(self):
        """Get next available exchange instance from the pool"""
        with self.pool_lock:
            exchange = self.exchange_pool[self.pool_index]
            self.pool_index = (self.pool_index + 1) % len(self.exchange_pool)
            return exchange

    def _compile_numba_functions(self):
        """Pre-compile Numba functions with dummy data"""
        # Use larger dummy dataset for more robust testing
        dummy_prices = np.array([100.0, 101.0, 102.0, 103.0, 104.0], dtype=np.float64)
        dummy_volumes = np.array([1.0, 2.0, 3.0, 2.0, 1.0], dtype=np.float64)
        
        # Test volume profile calculation
        profile, buckets = calculate_volume_profile(dummy_prices, dummy_volumes, 5)
        
        # Test volume nodes identification
        _ = identify_volume_nodes(profile, buckets, 0.2, 0.01)


    def fetch_raw_data(self, period):
        """Fetch raw OHLCV data and cache it."""
        cache_filename = f"raw_ohlcv_{self.symbol}_{self.interval}_{period}.pkl"

        if os.path.exists(cache_filename):
            print(f"Loading raw data from cache: {cache_filename}")
            df = pd.read_pickle(cache_filename)
            print(f"Total fetched OHLCV records: {len(df)}")
            return df

        exchange = self._get_exchange()
        end_time = int(time.time() * 1000)
        start_time = end_time - (int(period[:-1]) * 24 * 60 * 60 * 1000)

        # Fetch historical data
        all_ohlcv = []
        while start_time < end_time:
            print(f"Fetching data from {pd.to_datetime(start_time, unit='ms')}...")
            try:
                ohlcv = exchange.fetch_ohlcv(
                    symbol=self.symbol,
                    timeframe=self.interval,
                    since=start_time,
                    limit=1500
                )

                if not ohlcv:
                    break

                all_ohlcv.extend(ohlcv)
                start_time = ohlcv[-1][0] + 1

            except Exception as e:
                print(f"Error fetching data: {str(e)}")
                time.sleep(1)
                continue

        # Create DataFrame
        df = pd.DataFrame(all_ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
        df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
        df.set_index('timestamp', inplace=True)

        print(f"Saving raw data to cache: {cache_filename}")
        df.to_pickle(cache_filename)

        return df


    def process_volume_nodes(self, raw_data):
        """Calculate volume nodes based on current strategy parameters."""
        df = raw_data.copy()

        # Convert relevant columns to NumPy arrays
        close_prices = df['close'].values.astype(np.float64)
        volumes = df['volume'].values.astype(np.float64)
        
        # Calculate window size based on profile_period and interval
        # Assuming profile_period is in days and interval is in minutes
        # Adjust this calculation as needed based on the exact format
        if self.profile_period.endswith('d') and self.interval.endswith('m'):
            days = int(self.profile_period[:-1])
            minutes = int(self.interval[:-1])
            window_size = days * 24 * 60 // minutes
        else:
            raise NotImplementedError("Only daily profile_period and minute intervals are implemented.")
        
        n = len(close_prices)
        
        # Initialize fixed-size arrays for volume nodes
        max_nodes = self.price_bucket_count  # Assuming maximum nodes equal to bucket count
        volume_nodes_prices = np.zeros((n, max_nodes), dtype=np.float64)
        volume_nodes_volumes = np.zeros((n, max_nodes), dtype=np.float64)
        num_volume_nodes = np.zeros(n, dtype=np.int32)
        
        #print(f"Calculating volume profiles for {n} rows with current parameters...")
        
        # Call the Numba-jitted function
        process_volume_nodes_numba(
            close_prices,
            volumes,
            window_size,
            self.price_bucket_count,
            self.volume_threshold,
            self.node_distance,
            volume_nodes_prices,
            volume_nodes_volumes,
            num_volume_nodes
        )
        
        # Assign results back to the DataFrame
        df['volume_nodes_prices'] = list(volume_nodes_prices)
        df['volume_nodes_volumes'] = list(volume_nodes_volumes)
        df['num_volume_nodes'] = num_volume_nodes

        #print(f"Volume nodes calculated and assigned to data.")
        
        return df

    def plot_data(self, df):
        """Plots the OHLCV and volume profile data."""
        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))
        
        # Plot OHLCV
        ax1.plot(df.index, df['close'], label='Close Price')
        ax1.set_title(f"{self.symbol} Price")
        ax1.set_ylabel("Price")
        ax1.legend()
        
        # Plot Volume Profile
        if 'volume_nodes_prices' in df.columns and 'volume_nodes_volumes' in df.columns:
            latest_prices = df['volume_nodes_prices'].iloc[-1]
            latest_volumes = df['volume_nodes_volumes'].iloc[-1]
            valid = latest_volumes > 0
            ax2.barh(latest_prices[valid], latest_volumes[valid], alpha=0.3)
            ax2.set_title("Volume Profile")
            ax2.set_xlabel("Volume")
            ax2.set_ylabel("Price")
        
        plt.tight_layout()
        plt.show()

    @staticmethod
    @njit(parallel=True)
    def generate_signals_numba(
        close_prices,
        volume_nodes_prices,
        num_volume_nodes,
        entry_conf_bars,
        exit_conf_bars,
        node_distance
    ):
        n = len(close_prices)
        long_entries = np.zeros(n, dtype=np.bool_)
        short_entries = np.zeros(n, dtype=np.bool_)
        exits = np.zeros(n, dtype=np.bool_)

        for i in prange(n):
            if num_volume_nodes[i] == 0:
                continue  # No volume nodes to process

            current_price = close_prices[i]
            nodes = volume_nodes_prices[i, :num_volume_nodes[i]]

            # Initialize nearest nodes
            nearest_above = np.inf
            nearest_below = -np.inf

            # Find nearest above and below nodes
            for j in range(num_volume_nodes[i]):
                price = nodes[j]
                if price > current_price and price < nearest_above:
                    nearest_above = price
                elif price < current_price and price > nearest_below:
                    nearest_below = price

            # Long Entry Condition
            if nearest_below != -np.inf:
                price_diff = np.abs(current_price - nearest_below) / current_price
                if price_diff < node_distance:
                    # Check confirmation bars
                    if i >= entry_conf_bars:
                        positive_bars = 0
                        for k in range(i - entry_conf_bars, i):
                            if close_prices[k] > close_prices[k - 1]:
                                positive_bars += 1
                        if positive_bars >= entry_conf_bars * 0.5:
                            long_entries[i] = True

            # Short Entry Condition
            if nearest_above != np.inf:
                price_diff = np.abs(current_price - nearest_above) / current_price
                if price_diff < node_distance:
                    # Check confirmation bars
                    if i >= entry_conf_bars:
                        negative_bars = 0
                        for k in range(i - entry_conf_bars, i):
                            if close_prices[k] < close_prices[k - 1]:
                                negative_bars += 1
                        if negative_bars >= entry_conf_bars * 0.5:
                            short_entries[i] = True

            # Exit Conditions
            if i >= exit_conf_bars:
                # Check if there's an open position to exit
                if long_entries[i]:
                    if (i + exit_conf_bars) < n:
                        exits[i + exit_conf_bars] = True
                elif short_entries[i]:
                    if (i + exit_conf_bars) < n:
                        exits[i + exit_conf_bars] = True

        return long_entries, short_entries, exits

    def generate_signals(self, data):
        close_prices = data['close'].values.astype(np.float64)
        volume_nodes_prices = np.vstack(data['volume_nodes_prices'].values).astype(np.float64)
        # No need for volume_nodes_volumes in signal generation
        num_volume_nodes = data['num_volume_nodes'].values.astype(np.int32)

        long_entries, short_entries, exits = self.generate_signals_numba(
            close_prices,
            volume_nodes_prices,
            num_volume_nodes,
            self.entry_confirmation_bars,
            self.exit_confirmation_bars,
            self.node_distance
        )

        signals = pd.DataFrame({
            'long_entries': long_entries,
            'short_entries': short_entries,
            'exits': exits
        }, index=data.index)

        return signals

    def backtest(self, data, initial_capital, fee_rate):
        """Run backtest using vectorbt with additional debugging"""
        # Define the numeric columns used in backtesting
        numeric_cols = ['close', 'volume']  # Add other numeric columns if necessary

        # Check for NaN values in numeric columns only
        if data[numeric_cols].isna().any().any():
            print("Data contains NaN values in numeric columns. Please clean the data before backtesting.")
            raise ValueError("Data contains NaN values.")

        # Check for infinite values in numeric columns only
        if np.isinf(data[numeric_cols].values).any():
            print("Data contains infinite values in numeric columns. Please clean the data before backtesting.")
            raise ValueError("Data contains infinite values.")

        signals = self.generate_signals(data)

        # Calculate position sizes and stop-losses
        entry_prices = pd.Series(index=data.index, dtype=float)
        entry_prices[signals['long_entries']] = data.loc[signals['long_entries'], 'close']
        entry_prices[signals['short_entries']] = data.loc[signals['short_entries'], 'close']

        #stop_prices = pd.Series(index=data.index, dtype=float)
        #stop_prices[signals['long_entries']] = entry_prices[signals['long_entries']] * (1 - self.risk_per_trade)
        #stop_prices[signals['short_entries']] = entry_prices[signals['short_entries']] * (1 + self.risk_per_trade)

        # Calculate position size based on initial capital and first close price
        size = initial_capital / data['close'].iloc[0]

        try:
            portfolio = vbt.Portfolio.from_signals(
                close=data['close'],
                entries=signals['long_entries'] | signals['short_entries'],
                exits=signals['exits'],
                short_entries=signals['short_entries'],
                sl_stop=self.risk_per_trade,
                init_cash=initial_capital,
                fees=fee_rate,
                size=size,
                freq=self.interval
            )

            trades = portfolio.trades.records

            metrics = {
                'Total Return (%)': portfolio.total_return() * 100,
                'Sharpe Ratio': portfolio.sharpe_ratio(),
                'Max Drawdown (%)': portfolio.max_drawdown() * 100,
                'Number of Trades': len(trades) if trades is not None else 0,
                'Win Rate (%)': (np.sum(trades['pnl'] > 0) / len(trades)) * 100 if trades is not None and len(trades) > 0 else 0,
                'Profit Factor': np.sum(trades[trades['pnl'] > 0]['pnl']) / abs(np.sum(trades[trades['pnl'] < 0]['pnl'])) if trades is not None and len(trades) > 0 and abs(np.sum(trades[trades['pnl'] < 0]['pnl'])) > 0 else 0
            }

            return metrics, portfolio, data

        except Exception as e:
            print(f"\nError in vectorbt processing: {str(e)}")
            raise

    def verify_volume_nodes(self, data):
        """Verify that volume nodes are being calculated correctly"""
        #print("\nVerifying volume nodes calculation...")
        
        # Check a specific time window
        sample_window = data.iloc[-100:]  # Last 100 rows
        
        # Calculate volume profile for this window
        prices = sample_window['close'].values.astype(np.float64)
        volumes = sample_window['volume'].values.astype(np.float64)
        
        profile, buckets = calculate_volume_profile(
            prices,
            volumes,
            self.price_bucket_count
        )
        
        nodes = identify_volume_nodes(
            profile,
            buckets,
            self.volume_threshold,
            self.node_distance
        )

        '''
        print(f"\nSample window statistics:")
        print(f"Price range: {prices.min():.2f} - {prices.max():.2f}")
        print(f"Volume range: {volumes.min():.2f} - {volumes.max():.2f}")
        print(f"Number of volume nodes found: {len(nodes)}")
        
        if len(nodes) > 0:
            print("\nVolume nodes found:")
            for node in nodes:
                print(f"Price level: {node[0]:.2f}, Relative volume: {node[1]:.4f}")
        else:
            print("No volume nodes found - this may indicate an issue with the parameters")
        ''' 


def progress_bar(current, total):
    """Prints a simple progress bar."""
    percent_complete = int(100 * current / total)
    if percent_complete % 1 == 0: # Only update for whole percentage changes
        bar_length = 20
        filled_length = int(bar_length * current / total)
        bar = '=' * filled_length + '-' * (bar_length - filled_length)
        print(f'\r[{bar}] {percent_complete}%', end='', flush=True)


def optimize_parameters(strategy, param_grid, initial_capital, fee_rate, period):
    results = []
    print("Fetching raw data...")
    raw_data = strategy.fetch_raw_data(period)  # Fetch raw OHLCV data here
    print("Raw data fetched successfully.")

    param_combinations = [dict(zip(param_grid.keys(), values)) 
                         for values in product(*param_grid.values())]
    
    total_combinations = len(param_combinations)
    print(f"Testing {total_combinations} parameter combinations...")
    
    for i, params in enumerate(param_combinations, 1):
        progress_bar(i, total_combinations)
        #print(f"\nTesting combination {i}/{total_combinations}: {params}")
        
        # Update strategy parameters
        for param, value in params.items():
            setattr(strategy, param, value)
        
        # Recalculate volume nodes with updated parameters
        data = strategy.process_volume_nodes(raw_data)
        strategy.verify_volume_nodes(data)  # Optional: for debugging
        
        # Run backtest
        try:
            metrics, _, _ = strategy.backtest(data, initial_capital, fee_rate)
            #print(f"Metrics for combination {i}: {metrics}")
            results.append({**params, **metrics})
        except Exception as e:
            #print(f"Error testing combination {params}: {str(e)}")
            results.append({**params, 'Error': str(e)})

    results_df = pd.DataFrame(results)
    print("\nAvailable columns in results:")
    print(results_df.columns.tolist())  # Print list of columns
    
    if results_df.empty:
        print("No successful backtests. `results` DataFrame is empty.")
    else:
        print("Backtests completed successfully.")
    
    return results_df


if __name__ == "__main__":
    # Initialize strategy
    strategy = VolumeProfileStrategy(
        symbol='SOL/USDT',
        exchange_id='binance'
    )
    
    # Define parameter grid
    param_grid = {
        'price_bucket_count': [50],
        'volume_threshold': [0.080789],
        'node_distance': [0.00037],
        'entry_confirmation_bars': [5],
        'exit_confirmation_bars': [4],
        'risk_per_trade': [0.02]
    }
        
    # Optimize parameters
    results = optimize_parameters(
        strategy=strategy,
        param_grid=param_grid,
        initial_capital=10000,
        fee_rate=0.001,
        period='365d'
    )

    
    optimize_for = 'Sharpe Ratio'
    
    print("\nTop 50 parameter combinations by Sharpe ratio:")
    if optimize_for in results.columns:
        pd.set_option('display.max_columns', None)
        sorted_results = results.sort_values(optimize_for, ascending=False).head(200)
        sorted_results.to_csv('sorted_results.csv', index=False)  # Save to CSV without index
        print(sorted_results) 
    else:
        print("Profit Factor not found in results.")

    

    optimizing_a = 'volume_threshold'
    optimizing_b = 'price_bucket_count'
    
    # Create the heatmap with corrected scaling
    if optimize_for in results.columns:
    
        heatmap_data = results.pivot_table(values=optimize_for, index=optimizing_a, columns=optimizing_b)
        heatmap_data_clean = heatmap_data.replace([np.inf, -np.inf], np.nan)
        finite_max = heatmap_data_clean.max().max()
        finite_min = heatmap_data_clean.min().min()
        plt.figure(figsize=(10, 8))
        sns.heatmap(heatmap_data, annot=False, cmap='viridis', fmt=".2f", 
                    vmin=finite_min, vmax=finite_max)
        plt.title(optimize_for + ' Heatmap')
        plt.ylabel(optimizing_a)
        plt.xlabel(optimizing_b)
        plt.show()
    else:
        print("No Sharpe Ratio data available for heatmap.")