In my reinforcement learning algorithm implementation, i need to pass the intermediate reward to the agent, wen we get 'reward = -200' or 'reward >= 90'. 

In the below existing code i have implemented the reward function, but the implemented reward is only given to the agent after completing the maximum step size of the iteration loop in that current episode, but i need to pass the intermediate reward(while drastic deviation occurs) with in the episode to agent to guide it more effectively. So, i updated the existing code based on my logic.
The following problem i noticed while running the updated code:
consider my max_timesteps for each episode is 10,
i encounter the intermediate reward at the 5th timesteps of episode iteration,
here i am passing all the append data in to the PPO agent after the 5th timesteps of episode iteration
            episode_rewards.append(reward)
            states.append(state)
            actions.append(action)
            log_probs.append(log_prob.item())            
            values.append(agent.critic(node_features_tensor, edge_index).item())
            masks.append(1.0 - float(done))
            perf_metrics_log.append(perf_metrics)
here i am having doubt that, i supposed to pass only the required datas which is related with the corresponding intermediate reward and the corresponding 5th time step alone, 
but in the updated code, we are sending entire append data which are append upto the 5th timesteps of episode iteration, into the PPO agent,
after that intermediate update we are Reset episodic storage, due to this inconsistency all the previous append data upto the 5th timesteps were reset, at the Final policy update at the end of the episode is getting only the datas which are append after the 5th timesteps(from 6th time steps to the 10th time steps),

but i am having the doubt in the implementation present in the updates code, where i need to pass only the corresponding data present at that present timesteps at which the intermediate reward appears and should i need to pass all the append datas to the Final policy update at the end of the episode(including the data which are already passed to the agent during intermediate reward appears). 

Here what should i need to modify in the training function for properly implementing the intermediate reward in to the PPO agent.

Existing Code:
# Define Circuit Environment
class CircuitEnvironment:
    VERY_CLOSE_REWARD = 40
    LARGE_REWARD = 10
    VERY_HIGH_REWARD = 50
    PENALTY = 20
    SEVERE_PENALTY = 200

    def __init__(self, bounds_low, bounds_high):
        # Initialization code: setup bounds
    
    def reset(self, target_metrics=None):
	#existing implementation
        return node_features_tensor, edge_index, performance_metrics    

    def step(self, raw_actions, previous_metrics, target_metrics):
        raw_actions = np.squeeze(raw_actions)
        performance_metrics, temp_netlist_path = run_simulation(raw_actions)
        node_features_tensor, edge_index = netlist_to_graph(temp_netlist_path)
        self.target_metrics = target_metrics 
        
        # Round off the previous and current performance metrics to integer values for comparison
        rounded_performance_metrics = self.round_to_discrete_values(performance_metrics)
        rounded_previous_metrics = self.round_to_discrete_values(previous_metrics)
        print("performance_metrics", rounded_performance_metrics)
        
        reward = self.calculate_reward(rounded_performance_metrics, rounded_previous_metrics)
        done = self.check_done(rounded_performance_metrics)
        # Update previous state
        previous_metrics = rounded_performance_metrics 
        self.node_features_tensor = node_features_tensor
        self.edge_index = edge_index
        return node_features_tensor, edge_index, reward, done, previous_metrics

# Training Function
def train(env, agent, num_episodes, max_timesteps):
    num_targets = 100
    train_targets = CircuitEnvironment.generate_random_targets(num_targets=num_targets, seed=0)
    test_targets = CircuitEnvironment.generate_random_targets(num_targets=20, seed=1)
    
    # Initialize the episode to start from and log lists
    start_episode = 0    
    rewards_log = []
    losses_log = []
    metrics_log = []
    episode_lengths = []
    
    # Attempt to load the saved state
    saved_state = load_training_state()
        
    for episode in range(start_episode, num_episodes):
        if episode >= num_targets:
            break  # Exit the loop if exhausted the number of targets
        
        target = train_targets[episode % len(train_targets)]
        
        node_features_tensor, edge_index, performance_metrics = env.reset(target)
        state = (node_features_tensor, edge_index)
    
        episode_rewards = []
        states = []
        actions = []
        log_probs = []
        values = []
        masks = []
        perf_metrics_log = []

        for t in range(max_timesteps):            
            action, log_prob, perf_metrics = agent.select_action(state, performance_metrics)
            next_node_features_tensor, next_edge_index, reward, done, previous_metrics = env.step(action, perf_metrics, target)
            
            next_state = (next_node_features_tensor, next_edge_index)
    
            episode_rewards.append(reward)
            states.append(state)
            actions.append(action)
            log_probs.append(log_prob.item())            
            values.append(agent.critic(node_features_tensor, edge_index).item())
            masks.append(1.0 - float(done))
            perf_metrics_log.append(perf_metrics)
                
            performance_metrics = previous_metrics
            state = next_state            
  
            if done:
                break
    
        next_value = agent.critic(state[0], state[1]).item() if not done else 0         
        returns = agent.compute_gae(next_value, episode_rewards, masks, values)
    
        advantages = (torch.tensor(returns) - torch.tensor(values)).detach().numpy()
        returns = torch.tensor(returns).detach().numpy()
        average_loss_actor, average_loss_critic = agent.update_policy(states, actions, log_probs, returns, advantages)
    
        agent.save_checkpoint(episode)        

        save_training_state(agent, episode, rewards_log, losses_log, metrics_log, train_targets, test_targets)
    
    agent.save_checkpoint('final') 

My updated Code(training function):
# Training Function
def train(env, agent, num_episodes, max_timesteps):
    num_targets = 100
    train_targets = CircuitEnvironment.generate_random_targets(num_targets=num_targets, seed=0)
    test_targets = CircuitEnvironment.generate_random_targets(num_targets=20, seed=1)
    
    # Initialize the episode to start from and log lists
    start_episode = 0    
    rewards_log = []
    losses_log = []
    metrics_log = []
    episode_lengths = []
    
    # Attempt to load the saved state
    saved_state = load_training_state()
    if saved_state:
        start_episode = saved_state['episode'] + 1  # Start from the next episode
        agent.actor.load_state_dict(saved_state['actor_state_dict'])
        agent.critic.load_state_dict(saved_state['critic_state_dict'])
        agent.optimizer_actor.load_state_dict(saved_state['optimizer_actor_state_dict'])
        agent.optimizer_critic.load_state_dict(saved_state['optimizer_critic_state_dict'])
        rewards_log = saved_state['rewards_log']
        losses_log = saved_state['losses_log']
        metrics_log = saved_state['metrics_log']
        train_targets = saved_state['train_targets']
        test_targets = saved_state['test_targets']
        print(f"Resuming from episode {start_episode}")
        
    for episode in range(start_episode, num_episodes):
        if episode >= num_targets:
            break  # Exit the loop if exhausted the number of targets
        
        target = train_targets[episode % len(train_targets)]
        
        node_features_tensor, edge_index, performance_metrics = env.reset(target)
        state = (node_features_tensor, edge_index)
    
        episode_rewards = []
        states = []
        actions = []
        log_probs = []
        values = []
        masks = []
        perf_metrics_log = []

        for t in range(max_timesteps):            
            action, log_prob, perf_metrics = agent.select_action(state, performance_metrics)
            next_node_features_tensor, next_edge_index, reward, done, previous_metrics = env.step(action, perf_metrics, target)
            
            next_state = (next_node_features_tensor, next_edge_index)
    
            episode_rewards.append(reward)
            states.append(state)
            actions.append(action)
            log_probs.append(log_prob.item())            
            values.append(agent.critic(node_features_tensor, edge_index).item())
            masks.append(1.0 - float(done))
            perf_metrics_log.append(perf_metrics)
                
            performance_metrics = previous_metrics
            state = next_state            

            # Check for intermediate reward conditions
            if reward <= -200 or reward >= 90:
                next_value = agent.critic(state[0], state[1]).item() if not done else 0
                returns = agent.compute_gae(next_value, episode_rewards, masks, values)
                advantages = (torch.tensor(returns) - torch.tensor(values)).detach().numpy()
                returns = torch.tensor(returns).detach().numpy()
                average_loss_actor, average_loss_critic = agent.update_policy(states, actions, log_probs, returns, advantages)
                
                # Reset episodic storage after intermediate update
                episode_rewards = []
                states = []
                actions = []
                log_probs = []
                values = []
                masks = []

            if done:
                break

        # Final policy update at the end of the episode
        next_value = agent.critic(state[0], state[1]).item() if not done else 0         
        returns = agent.compute_gae(next_value, episode_rewards, masks, values)
    
        advantages = (torch.tensor(returns) - torch.tensor(values)).detach().numpy()
        returns = torch.tensor(returns).detach().numpy()
        average_loss_actor, average_loss_critic = agent.update_policy(states, actions, log_probs, returns, advantages)                        
            
        episode_length = len(episode_rewards)
        total_reward = sum(episode_rewards)       
   
        episode_lengths.append(episode_length)
    
        agent.save_checkpoint(episode)

        save_training_state(agent, episode, rewards_log, losses_log, metrics_log, train_targets, test_targets)
  
    agent.save_checkpoint('final')        
   
    logging.info("Training Completed.")