write end to correct code single .py file 
g/DAPO/compare.py
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 1: Total Reward = 4.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 2: Total Reward = 2.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 3: Total Reward = 4.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 4: Total Reward = 2.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 5: Total Reward = 2.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 6: Total Reward = 2.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 7: Total Reward = 3.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 8: Total Reward = 5.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 9: Total Reward = 3.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 10: Total Reward = 2.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
INFO:__main__:Episode 11: Total Reward = 2.0
WARNING:__main__:Not enough samples in the replay buffer to perform an update.
ERROR:__main__:An error occurred during training: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward.
Traceback (most recent call last):
  File "e:\LLMS\Fine-tuning\DAPO\compare.py", line 497, in <module>
    train_dapo(episodes=500)
  File "e:\LLMS\Fine-tuning\DAPO\compare.py", line 486, in train_dapo
    agent.update()
  File "e:\LLMS\Fine-tuning\DAPO\compare.py", line 407, in update
    policy_loss.backward()
  File "C:\Users\hemanthk.LAP53-FJS.000\AppData\Local\Programs\Python\Python310\lib\site-packages\torch\_tensor.py", line 525, in backward
    torch.autograd.backward(
  File "C:\Users\hemanthk.LAP53-FJS.000\AppData\Local\Programs\Python\Python310\lib\site-packages\torch\autograd\__init__.py", line 267, in backward
    _engine_run_backward(
  File "C:\Users\hemanthk.LAP53-FJS.000\AppData\Local\Programs\Python\Python310\lib\site-packages\torch\autograd\graph.py", line 744, in _engine_run_backward
    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
RuntimeError: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward. 
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from typing import List, Tuple, NamedTuple, Any, Optional
from collections import namedtuple
import math
import random
import logging

# Configure logging for debugging and error messages
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Define a named tuple for storing transitions in the replay buffer
Transition = namedtuple('Transition', ('state', 'action', 'reward', 'next_state', 'done'))

class SimpleMDP:
    """
    A simple MDP for demonstration purposes.
    States: 0, 1, 2
    Actions: 0, 1
    Transition dynamics and rewards are predefined.
    """

    def __init__(self):
        self.states = [0, 1, 2]
        self.actions = [0, 1]
        self.transition_prob = {
            0: {0: (0.8, 0), 1: (0.2, 1)},
            1: {0: (0.7, 0), 1: (0.3, 2)},
            2: {0: (1.0, 2), 1: (1.0, 2)},
        }
        self.rewards = {
            0: {0: 0, 1: 1},
            1: {0: 0, 1: 1},
            2: {0: 0, 1: 0},
        }
        self.reset()

    def reset(self) -> int:
        """Resets the environment to the initial state."""
        self.current_state = 0
        return self.current_state

    def step(self, action: int) -> Tuple[int, float, bool]:
        """
        Executes the given action and returns the next state, reward, and done flag.
        Args:
            action (int): The action to take.
        Returns:
            Tuple[int, float, bool]: Next state, reward, and done flag.
        Raises:
            ValueError: If the action is invalid.
        """
        if action not in self.actions:
            raise ValueError(f"Invalid action: {action}")

        prob, next_state = self.transition_prob[self.current_state][action]
        reward = self.rewards[self.current_state][action]
        done = next_state == 2  # State 2 is terminal

        # For simplicity, deterministic transitions based on highest probability
        self.current_state = next_state

        return next_state, reward, done

    def get_state_space(self) -> List[int]:
        """Returns the list of states."""
        return self.states

    def get_action_space(self) -> List[int]:
        """Returns the list of actions."""
        return self.actions

class ReplayBuffer:
    """
    A simple replay buffer for storing transitions.
    """

    def __init__(self, capacity: int) -> None:
        """
        Initializes the replay buffer.
        Args:
            capacity (int): Maximum number of transitions to store.
        """
        self.capacity = capacity
        self.buffer: List[Transition] = []
        self.position = 0

    def push(self, *args: Any) -> None:
        """
        Saves a transition.
        """
        if len(self.buffer) < self.capacity:
            self.buffer.append(None)  # Expand the list
        self.buffer[self.position] = Transition(*args)
        self.position = (self.position + 1) % self.capacity

    def sample(self, batch_size: int) -> List[Transition]:
        """
        Samples a batch of transitions.
        Args:
            batch_size (int): Number of transitions to sample.
        Returns:
            List[Transition]: A list of sampled transitions.
        Raises:
            ValueError: If the batch_size is greater than the number of stored transitions.
        """
        if batch_size > len(self.buffer):
            raise ValueError(f"Cannot sample {batch_size} transitions from buffer with {len(self.buffer)} transitions.")
        return random.sample(self.buffer, batch_size)

    def __len__(self) -> int:
        """Returns the current size of internal memory."""
        return len(self.buffer)

def bregman_divergence(p: torch.Tensor, q: torch.Tensor, phi: str = 'kl') -> torch.Tensor:
    """
    Computes the Bregman divergence between two probability distributions.
    Currently supports KL divergence.

    Args:
        p (torch.Tensor): First probability distribution.
        q (torch.Tensor): Second probability distribution.
        phi (str): Type of divergence. Default is 'kl'.

    Returns:
        torch.Tensor: The computed divergence.
    Raises:
        ValueError: If an unsupported divergence type is specified.
    """
    if phi == 'kl':
        # Adding a small epsilon to avoid log(0)
        epsilon = 1e-10
        p = p.clamp(min=epsilon)
        q = q.clamp(min=epsilon)
        return (p * torch.log(p / q)).sum(dim=-1)
    else:
        raise ValueError(f"Unsupported divergence type: {phi}")

def projection_softmax(f_theta_s: torch.Tensor) -> torch.Tensor:
    """
    Applies the softmax function to project logits into a probability distribution.

    Args:
        f_theta_s (torch.Tensor): Logits to be projected.

    Returns:
        torch.Tensor: Projected probability distribution.
    """
    return F.softmax(f_theta_s, dim=-1)

class PolicyNetwork(nn.Module):
    """
    Neural network representing the stochastic policy π_θ.
    Outputs logits which are projected to probabilities via softmax.
    """

    def __init__(self, state_dim: int, action_dim: int, hidden_dim: int = 128) -> None:
        """
        Initializes the policy network.
        Args:
            state_dim (int): Dimension of the state space.
            action_dim (int): Number of possible actions.
            hidden_dim (int): Number of hidden units. Default is 128.
        """
        super(PolicyNetwork, self).__init__()
        self.fc1 = nn.Linear(state_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.logits = nn.Linear(hidden_dim, action_dim)

    def forward(self, state: torch.Tensor) -> torch.Tensor:
        """
        Forward pass through the network.
        Args:
            state (torch.Tensor): Input state.
        Returns:
            torch.Tensor: Action logits.
        """
        x = F.relu(self.fc1(state))
        x = F.relu(self.fc2(x))
        logits = self.logits(x)
        return logits

    def get_action_prob(self, state: torch.Tensor) -> torch.Tensor:
        """
        Returns the action probabilities by applying softmax to logits.
        Args:
            state (torch.Tensor): Input state.
        Returns:
            torch.Tensor: Action probabilities.
        """
        logits = self.forward(state)
        return projection_softmax(logits)

class QNetwork(nn.Module):
    """
    Neural network for estimating Q-values Q_b(k).
    """

    def __init__(self, state_dim: int, action_dim: int, hidden_dim: int = 128) -> None:
        """
        Initializes the Q-network.
        Args:
            state_dim (int): Dimension of the state space.
            action_dim (int): Number of possible actions.
            hidden_dim (int): Number of hidden units. Default is 128.
        """
        super(QNetwork, self).__init__()
        self.fc1 = nn.Linear(state_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.q_values = nn.Linear(hidden_dim, action_dim)

    def forward(self, state: torch.Tensor) -> torch.Tensor:
        """
        Forward pass through the network.
        Args:
            state (torch.Tensor): Input state.
        Returns:
            torch.Tensor: Estimated Q-values for each action.
        """
        x = F.relu(self.fc1(state))
        x = F.relu(self.fc2(x))
        q = self.q_values(x)
        return q

class DualNetwork(nn.Module):
    """
    Neural network for approximating the dual function f_θ.
    """

    def __init__(self, state_dim: int, action_dim: int, hidden_dim: int = 128) -> None:
        """
        Initializes the dual network.
        Args:
            state_dim (int): Dimension of the state space.
            action_dim (int): Number of possible actions.
            hidden_dim (int): Number of hidden units. Default is 128.
        """
        super(DualNetwork, self).__init__()
        self.fc1 = nn.Linear(state_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.output = nn.Linear(hidden_dim, action_dim)

    def forward(self, state: torch.Tensor) -> torch.Tensor:
        """
        Forward pass through the network.
        Args:
            state (torch.Tensor): Input state.
        Returns:
            torch.Tensor: Approximated dual function.
        """
        x = F.relu(self.fc1(state))
        x = F.relu(self.fc2(x))
        f = self.output(x)
        return f

class DAPO:
    """
    Dual Approximation Policy Optimization (DAPO) algorithm implementation.
    """

    def __init__(
        self,
        state_dim: int,
        action_dim: int,
        gamma: float = 0.99,
        eta_start: float = 0.1,
        eta_decay: float = 0.99,
        buffer_capacity: int = 10000,
        batch_size: int = 64,
        device: Optional[str] = None,
    ) -> None:
        """
        Initializes the DAPO algorithm.
        Args:
            state_dim (int): Dimension of the state space.
            action_dim (int): Number of possible actions.
            gamma (float): Discount factor. Default is 0.99.
            eta_start (float): Initial learning rate for dual updates. Default is 0.1.
            eta_decay (float): Decay rate for eta. Default is 0.99.
            buffer_capacity (int): Replay buffer capacity. Default is 10000.
            batch_size (int): Batch size for updates. Default is 64.
            device (Optional[str]): Device to run computations on. Defaults to CUDA if available.
        Raises:
            ValueError: If invalid parameters are provided.
        """
        self.device = device if device else ('cuda' if torch.cuda.is_available() else 'cpu')
        self.gamma = gamma
        self.eta = eta_start
        self.eta_decay = eta_decay
        self.batch_size = batch_size
        self.state_dim = state_dim  # Store state_dim for use in _state_to_tensor

        # Initialize networks
        self.policy_net = PolicyNetwork(state_dim, action_dim).to(self.device)
        self.q_net = QNetwork(state_dim, action_dim).to(self.device)
        self.dual_net = DualNetwork(state_dim, action_dim).to(self.device)

        # Optimizers
        self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=1e-3)
        self.q_optimizer = optim.Adam(self.q_net.parameters(), lr=1e-3)
        self.dual_optimizer = optim.Adam(self.dual_net.parameters(), lr=1e-3)

        # Replay buffer
        self.replay_buffer = ReplayBuffer(capacity=buffer_capacity)

    def select_action(self, state: int) -> int:
        """
        Selects an action based on the current policy.
        Args:
            state (int): Current state.
        Returns:
            int: Selected action.
        """
        state_tensor = self._state_to_tensor(state)
        with torch.no_grad():
            action_probs = self.policy_net.get_action_prob(state_tensor)
        action = torch.multinomial(action_probs, num_samples=1).item()
        return action

    def _state_to_tensor(self, state: int) -> torch.Tensor:
        """
        Converts state to a one-hot encoded tensor.
        Args:
            state (int): The state index.
        Returns:
            torch.Tensor: One-hot encoded state tensor.
        Raises:
            ValueError: If the state index is out of bounds.
        """
        if state < 0 or state >= self.state_dim:
            raise ValueError(f"State index {state} is out of bounds for state_dim {self.state_dim}.")
        state_tensor = torch.zeros(self.state_dim, dtype=torch.float32, device=self.device)
        state_tensor[state] = 1.0
        return state_tensor.unsqueeze(0)  # Add batch dimension

    def store_transition(self, transition: Transition) -> None:
        """
        Stores a transition in the replay buffer.
        Args:
            transition (Transition): The transition to store.
        """
        self.replay_buffer.push(*transition)

    def update(self) -> None:
        """
        Performs a single update of the policy, Q-network, and dual network.
        Raises:
            ValueError: If there are not enough samples in the replay buffer.
        """
        if len(self.replay_buffer) < self.batch_size:
            logger.warning("Not enough samples in the replay buffer to perform an update.")
            return

        try:
            transitions = self.replay_buffer.sample(self.batch_size)
        except ValueError as e:
            logger.error(f"Sampling error: {e}")
            return

        batch = Transition(*zip(*transitions))

        # Convert batch data to tensors
        try:
            state_batch = torch.cat([self._state_to_tensor(s) for s in batch.state]).squeeze(1).to(self.device)
            action_batch = torch.tensor(batch.action, dtype=torch.long, device=self.device).unsqueeze(1)
            reward_batch = torch.tensor(batch.reward, dtype=torch.float32, device=self.device).unsqueeze(1)
            next_state_batch = torch.cat([self._state_to_tensor(s) for s in batch.next_state]).squeeze(1).to(self.device)
            done_batch = torch.tensor(batch.done, dtype=torch.float32, device=self.device).unsqueeze(1)
        except ValueError as e:
            logger.error(f"Error converting batch data to tensors: {e}")
            return

        # Update Q-network using Bellman update
        current_q = self.q_net(state_batch).gather(1, action_batch)
        with torch.no_grad():
            next_q = self.q_net(next_state_batch).max(1)[0].unsqueeze(1)
            target_q = reward_batch + self.gamma * next_q * (1 - done_batch)
        q_loss = F.mse_loss(current_q, target_q)

        self.q_optimizer.zero_grad()
        q_loss.backward()
        self.q_optimizer.step()

        # Dual network update
        f_theta_s = self.dual_net(state_batch)
        # Compute ∇Φ(π(k)s) - ηk Q_b(k)s
        policy_probs = self.policy_net.get_action_prob(state_batch)
        grad_phi = torch.log(policy_probs + 1e-10)  # For negative entropy φ, ∇Φ(π) = log π
        target = grad_phi - self.eta * self.q_net(state_batch).detach()
        dual_loss = F.mse_loss(f_theta_s, target)

        self.dual_optimizer.zero_grad()
        dual_loss.backward()
        self.dual_optimizer.step()

        # Policy network update using the dual approximation
        # Detach dual_net to prevent backward through dual_net
        dual_output = self.dual_net(state_batch).detach()
        projected = projection_softmax(dual_output)
        divergence = bregman_divergence(projected, policy_probs, phi='kl').mean()
        policy_loss = divergence

        self.policy_optimizer.zero_grad()
        policy_loss.backward()
        self.policy_optimizer.step()

        # Decay eta
        self.eta *= self.eta_decay
        logger.debug(f"Updated eta to {self.eta}")

    def save_models(self, path: str) -> None:
        """
        Saves the models to the specified path.
        Args:
            path (str): Directory path to save the models.
        Raises:
            IOError: If the path does not exist or is not writable.
        """
        try:
            torch.save({
                'policy_net': self.policy_net.state_dict(),
                'q_net': self.q_net.state_dict(),
                'dual_net': self.dual_net.state_dict(),
            }, path)
            logger.info(f"Models saved to {path}")
        except IOError as e:
            logger.error(f"Failed to save models: {e}")
            raise

    def load_models(self, path: str) -> None:
        """
        Loads the models from the specified path.
        Args:
            path (str): Directory path to load the models from.
        Raises:
            IOError: If the path does not exist or is not readable.
            KeyError: If the expected keys are not found in the saved state.
        """
        try:
            checkpoint = torch.load(path, map_location=self.device)
            self.policy_net.load_state_dict(checkpoint['policy_net'])
            self.q_net.load_state_dict(checkpoint['q_net'])
            self.dual_net.load_state_dict(checkpoint['dual_net'])
            logger.info(f"Models loaded from {path}")
        except IOError as e:
            logger.error(f"Failed to load models: {e}")
            raise
        except KeyError as e:
            logger.error(f"Missing key in the checkpoint: {e}")
            raise

def train_dapo(episodes: int = 500) -> None:
    """
    Trains the DAPO agent on the SimpleMDP environment.
    Args:
        episodes (int): Number of episodes to train. Default is 500.
    """
    env = SimpleMDP()
    state_dim = len(env.get_state_space())
    action_dim = len(env.get_action_space())

    agent = DAPO(state_dim=state_dim, action_dim=action_dim)

    for episode in range(1, episodes + 1):
        state = env.reset()
        done = False
        total_reward = 0.0

        while not done:
            try:
                action = agent.select_action(state)
            except ValueError as e:
                logger.error(f"Error selecting action: {e}")
                break

            try:
                next_state, reward, done = env.step(action)
            except ValueError as e:
                logger.error(f"Error during environment step: {e}")
                break

            agent.store_transition(Transition(state, action, reward, next_state, done))
            agent.update()

            state = next_state
            total_reward += reward

        logger.info(f"Episode {episode}: Total Reward = {total_reward}")

    logger.info("Training completed.")

if __name__ == "__main__":
    try:
        train_dapo(episodes=500)
    except Exception as e:
        logger.exception(f"An error occurred during training: {e}")