below model gpt2 visualization 
import os
import logging
from typing import Any, Dict, Optional, Union
import torch
import torch.nn as nn
import plotly.graph_objs as go
from plotly.subplots import make_subplots


# Configure logging for debugging and error tracking
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


class ModelVisualizer:
    """
    A versatile visualization tool for various machine learning models (NLP, Computer Vision, etc.).
    Supports 1D to 4D tensor visualizations, saving plots as images and HTML files.
    """

    def __init__(self, save_directory: str) -> None:
        """
        Initializes the ModelVisualizer with a directory to save visualizations.

        Args:
            save_directory (str): The directory path where plots will be saved.
        """
        self.save_directory = save_directory
        self._ensure_directory_exists()

    def _ensure_directory_exists(self) -> None:
        """
        Ensures that the save directory exists; creates it if it does not.
        """
        try:
            os.makedirs(self.save_directory, exist_ok=True)
            logger.info(f"Save directory is set to: {self.save_directory}")
        except OSError as e:
            logger.error(f"Failed to create directory {self.save_directory}: {e}")
            raise

    def visualize(
        self,
        data: Union[torch.Tensor, Any],
        key_input: Optional[Dict[str, Any]] = None,
        plot_type: Optional[str] = None
    ) -> None:
        """
        Determines the tensor dimensionality and calls the appropriate plotting method.

        Args:
            data (Union[torch.Tensor, Any]): The data to visualize.
            key_input (Optional[Dict[str, Any]]): Optional key input for dynamic naming.
            plot_type (Optional[str]): Optional plot type description.
        """
        try:
            if isinstance(data, torch.Tensor):
                tensor = data
            else:
                tensor = self._convert_to_tensor(data)

            dim = tensor.dim()
            logger.info(f"Data has {dim} dimensions.")

            if dim == 1:
                self._plot_1d(tensor, key_input, plot_type)
            elif dim == 2:
                self._plot_2d(tensor, key_input, plot_type)
            elif dim == 3:
                self._plot_3d(tensor, key_input, plot_type)
            elif dim == 4:
                self._plot_4d(tensor, key_input, plot_type)
            else:
                logger.warning(f"Tensors with {dim} dimensions are not supported for visualization.")
        except Exception as e:
            logger.error(f"Visualization failed: {e}")
            raise

    def _convert_to_tensor(self, data: Any) -> torch.Tensor:
        """
        Converts input data to a torch.Tensor.

        Args:
            data (Any): The data to convert.

        Returns:
            torch.Tensor: The converted tensor.
        """
        try:
            tensor = torch.tensor(data)
            logger.info("Data successfully converted to torch.Tensor.")
            return tensor
        except Exception as e:
            logger.error(f"Failed to convert data to tensor: {e}")
            raise

    def _plot_1d(
        self,
        tensor: torch.Tensor,
        key_input: Optional[Dict[str, Any]],
        plot_type: Optional[str]
    ) -> None:
        """Generates and saves a 1D scatter plot."""
        try:
            x = torch.arange(tensor.size(0)).numpy()
            y = tensor.numpy()

            fig = go.Figure(data=go.Scatter(x=x, y=y, mode='markers', name='1D Scatter'))
            title = self._generate_title(key_input, plot_type or "1D Scatter Plot")
            fig.update_layout(
                title=title,
                xaxis_title='Index',
                yaxis_title='Value',
                template='plotly_white'
            )

            self._save_plot(fig, "1D_Scatter_Plot")
            logger.info("1D scatter plot generated and saved successfully.")
        except Exception as e:
            logger.error(f"Error plotting 1D tensor: {e}")

    def _plot_2d(
        self,
        tensor: torch.Tensor,
        key_input: Optional[Dict[str, Any]],
        plot_type: Optional[str]
    ) -> None:
        """Generates and saves a 2D scatter plot."""
        try:
            if tensor.size(1) < 2:
                raise ValueError("2D tensor must have at least two features for plotting.")

            x = tensor[:, 0].numpy()
            y = tensor[:, 1].numpy()

            fig = go.Figure(data=go.Scatter(x=x, y=y, mode='markers', name='2D Scatter'))
            title = self._generate_title(key_input, plot_type or "2D Scatter Plot")
            fig.update_layout(
                title=title,
                xaxis_title='Feature 1',
                yaxis_title='Feature 2',
                template='plotly_white'
            )

            self._save_plot(fig, "2D_Scatter_Plot")
            logger.info("2D scatter plot generated and saved successfully.")
        except Exception as e:
            logger.error(f"Error plotting 2D tensor: {e}")

    def _plot_3d(
        self,
        tensor: torch.Tensor,
        key_input: Optional[Dict[str, Any]],
        plot_type: Optional[str]
    ) -> None:
        """Generates and saves a 3D scatter plot."""
        try:
            if tensor.size(1) < 3:
                raise ValueError("3D tensor must have at least three features for plotting.")

            x = tensor[:, 0].numpy()
            y = tensor[:, 1].numpy()
            z = tensor[:, 2].numpy()

            fig = go.Figure(data=go.Scatter3d(
                x=x, y=y, z=z,
                mode='markers',
                marker=dict(size=5, color='blue', opacity=0.8),
                name='3D Scatter'
            ))
            title = self._generate_title(key_input, plot_type or "3D Scatter Plot")
            fig.update_layout(
                title=title,
                scene=dict(
                    xaxis_title='Feature 1',
                    yaxis_title='Feature 2',
                    zaxis_title='Feature 3'
                ),
                template='plotly_white'
            )

            self._save_plot(fig, "3D_Scatter_Plot")
            logger.info("3D scatter plot generated and saved successfully.")
        except Exception as e:
            logger.error(f"Error plotting 3D tensor: {e}")

    def _plot_4d(
        self,
        tensor: torch.Tensor,
        key_input: Optional[Dict[str, Any]],
        plot_type: Optional[str]
    ) -> None:
        """Generates and saves a 4D scatter plot with color as the fourth dimension."""
        try:
            if tensor.size(1) < 4:
                raise ValueError("4D tensor must have at least four features for plotting.")

            x = tensor[:, 0].numpy()
            y = tensor[:, 1].numpy()
            z = tensor[:, 2].numpy()
            color_feature = tensor[:, 3].numpy()

            fig = go.Figure(data=go.Scatter3d(
                x=x, y=y, z=z,
                mode='markers',
                marker=dict(
                    size=5,
                    color=color_feature,
                    colorscale='Viridis',
                    opacity=0.8,
                    colorbar=dict(title='Feature 4')
                ),
                name='4D Scatter'
            ))
            title = self._generate_title(key_input, plot_type or "4D Scatter Plot")
            fig.update_layout(
                title=title,
                scene=dict(
                    xaxis_title='Feature 1',
                    yaxis_title='Feature 2',
                    zaxis_title='Feature 3'
                ),
                template='plotly_white'
            )

            self._save_plot(fig, "4D_Scatter_Plot")
            logger.info("4D scatter plot generated and saved successfully.")
        except Exception as e:
            logger.error(f"Error plotting 4D tensor: {e}")

    def _generate_title(
        self,
        key_input: Optional[Dict[str, Any]],
        plot_type: str
    ) -> str:
        """
        Generates a descriptive title based on the key input.

        Args:
            key_input (Optional[Dict[str, Any]]): The original key input.
            plot_type (str): The type of plot.

        Returns:
            str: A descriptive title.
        """
        if key_input:
            flat_keys = self._flatten_keys(key_input)
            key_str = "_".join(flat_keys)
            title = f"{plot_type} - Keys: {key_str}"
        else:
            title = plot_type
        return title

    def _flatten_keys(
        self,
        d: Dict[str, Any],
        parent_key: str = '',
        sep: str = '_'
    ) -> list:
        """
        Flattens nested dictionary keys.

        Args:
            d (Dict[str, Any]): The dictionary to flatten.
            parent_key (str, optional): The base key string. Defaults to ''.
            sep (str, optional): Separator between keys. Defaults to '_'.

        Returns:
            list: A list of flattened key strings.
        """
        items = []
        for k, v in d.items():
            new_key = f"{parent_key}{sep}{k}" if parent_key else k
            if isinstance(v, dict):
                items.extend(self._flatten_keys(v, new_key, sep=sep))
            else:
                items.append(new_key)
        return items

    def _save_plot(self, fig: go.Figure, filename: str) -> None:
        """
        Saves the Plotly figure as both HTML and PNG formats.

        Args:
            fig (go.Figure): The Plotly figure to save.
            filename (str): The base filename for saving.
        """
        try:
            html_path = os.path.join(self.save_directory, f"{filename}.html")
            png_path = os.path.join(self.save_directory, f"{filename}.png")

            fig.write_html(html_path)
            fig.write_image(png_path)
            logger.info(f"Plot saved as HTML: {html_path} and PNG: {png_path}")
        except Exception as e:
            logger.error(f"Failed to save plot {filename}: {e}")
            raise


class DynamicModelVisualizationModel(nn.Module):
    """
    A PyTorch model that generates tensors of varying dimensions based on input keys
    and visualizes them using the ModelVisualizer.
    """

    def __init__(self, visualizer: ModelVisualizer) -> None:
        """
        Initializes the DynamicModelVisualizationModel with a ModelVisualizer.

        Args:
            visualizer (ModelVisualizer): An instance of ModelVisualizer.
        """
        super(DynamicModelVisualizationModel, self).__init__()
        self.visualizer = visualizer

    def forward(self, key_input: Dict[str, Any]) -> torch.Tensor:
        """
        Processes the recursive key input to generate a tensor and visualize it.

        Args:
            key_input (Dict[str, Any]): A nested dictionary input.

        Returns:
            torch.Tensor: A tensor with dimensions based on the key input.
        """
        try:
            tensor = self._generate_tensor(key_input)
            self.visualizer.visualize(tensor, key_input, "Dynamic Tensor Visualization")
            return tensor
        except Exception as e:
            logger.error(f"Error in forward pass: {e}")
            raise

    def _generate_tensor(self, key_input: Dict[str, Any], depth: int = 1) -> torch.Tensor:
        """
        Recursively processes the key input to generate a tensor with varying dimensions.

        Args:
            key_input (Dict[str, Any]): A nested dictionary input.
            depth (int, optional): Current depth of recursion. Defaults to 1.

        Returns:
            torch.Tensor: A tensor with dimensions based on recursion depth.
        """
        if not isinstance(key_input, dict):
            raise ValueError("Key input must be a dictionary.")

        if not key_input:
            raise ValueError("Key input dictionary is empty.")

        # Base case: when the value is not a dictionary
        for key, value in key_input.items():
            if isinstance(value, dict):
                return self._generate_tensor(value, depth + 1)
            else:
                # Create a tensor with dimension based on depth
                size = 10  # arbitrary size for each dimension
                dims = min(depth, 4)  # Limit to 4D
                shape = tuple([size] * dims)
                logger.info(f"Generating a {dims}D tensor with shape {shape} based on key '{key}'.")
                return torch.randn(shape)

        # If all values are dictionaries, increase depth
        size = 10
        dims = min(depth, 4)
        shape = tuple([size] * dims)
        logger.info(f"Generating a {dims}D tensor with shape {shape} based on nested keys.")
        return torch.randn(shape)

def main():
    """
    Example usage of the ModelVisualizer and DynamicModelVisualizationModel.
    """
    from transformers import AutoModelForCausalLM
    # Load the GPT-2 model
    model = AutoModelForCausalLM.from_pretrained("gpt2")
    
    # Initialize the visualizer with a desired save directory
    visualizer = ModelVisualizer(save_directory="./model_visualizations")

    # Initialize the dynamic model with the visualizer
    dynamic_model = DynamicModelVisualizationModel(visualizer=visualizer)

    # Define the key inputs for visualization
    key_inputs = [
        {'layer': ['transformer', 'wte'], 'type': 'embedding'},
        {'layer': ['transformer', 'wpe'], 'type': 'embedding'},
        {'layer': ['transformer', 'drop'], 'type': 'dropout'},
        {'layer': ['transformer', 'h'], 'type': 'module'},
        {'layer': ['transformer', 'h', '0', 'ln_1'], 'type': 'layer_norm'},
        {'layer': ['transformer', 'h', '0', 'attn'], 'type': 'module'},
        {'layer': ['transformer', 'h', '0', 'attn', 'c_attn'], 'type': 'linear'},
        {'layer': ['transformer', 'h', '0', 'attn', 'c_proj'], 'type': 'linear'},
        {'layer': ['transformer', 'h', '0', 'ln_2'], 'type': 'layer_norm'},
        {'layer': ['transformer', 'h', '0', 'mlp'], 'type': 'module'},
        {'layer': ['transformer', 'h', '0', 'mlp', 'c_fc'], 'type': 'linear'},
        {'layer': ['transformer', 'h', '0', 'mlp', 'c_proj'], 'type': 'linear'},
        {'layer': ['transformer', 'h', '0', 'mlp', 'dropout'], 'type': 'dropout'},
        {'layer': ['transformer', 'h', '1', 'ln_1'], 'type': 'layer_norm'},
        {'layer': ['transformer', 'h', '1', 'attn'], 'type': 'module'},
        {'layer': ['transformer', 'h', '1', 'attn', 'c_attn'], 'type': 'linear'},
        {'layer': ['transformer', 'h', '1', 'attn', 'c_proj'], 'type': 'linear'},
        {'layer': ['transformer', 'h', '1', 'ln_2'], 'type': 'layer_norm'},
        {'layer': ['transformer', 'h', '1', 'mlp'], 'type': 'module'},
        {'layer': ['transformer', 'h', '1', 'mlp', 'c_fc'], 'type': 'linear'},
        {'layer': ['transformer', 'h', '1', 'mlp', 'c_proj'], 'type': 'linear'},
        {'layer': ['transformer', 'h', '1', 'mlp', 'dropout'], 'type': 'dropout'},
        {'layer': ['transformer', 'ln_f'], 'type': 'layer_norm'},
        {'layer': ['lm_head'], 'type': 'linear'},
        # Continue similarly for blocks 2 to 11
    ]

    # Generate and visualize tensors based on key inputs
    for key_input in key_inputs:
        tensor = dynamic_model.forward(key_input)
        # The tensor visualization is handled within the model's forward method

    logger.info("All visualizations have been generated.")

if __name__ == "__main__":
    main()