Hi, I've implemented a Evader Node and a DQN model . I want you to go through the code and find all the mistakes I made . I'll mention some of the mistakes I made: Given the destination point, the car which starts at an intial point should make it's way by selecting the appropriate actions to make it all the way to final point. But instead it's going in the other direction . I don't really understand how my agent is performing. So, first remove the run function and update the scan_callback function to include max no of episodes and max steps for each episode. If the car takes too long to reach destination or even max steps reached , put a condition in place to reset the car to its intial position by sending a command to gazebo . Second, In add_experience step, what values do I need to pass since state and next_state are two different things , I'm passing state (109 laser scan points to both of those). So, I don't think it's a right step to do. Correct me If I'm wrong. The agent has to learn something from the data. For observation space , I changed 360 to 109 since I'm getting 109 scan point for every call . Correct this If I did wrong. and move Make all the necessary changes to the code. The training should pretty much look like this : def q_train(agent, env, episodes,epsilon_decay = 0.99):
    rewards = []
    epsilons = []

    for i in range(episodes):
        state = en.resevt()    # we have to reset this by communicating with gazebo 
        episode_reward = 0
        done = False
        while not done:
            action = agent.get_action(state)
            next_state, reward, done, _ = env.step(action)
            agent.update(state, action, reward, next_state)
            
            episode_reward += reward
            state = next_state
        
        agent.epsilon *= epsilon_decay
        epsilons.append(agent.epsilon)
            
        rewards.append(episode_reward)
  .    My actual implementation is as follows: import numpy as np
import random
import math

import rospy
import tf
import copy
import time

from sensor_msgs.msg import LaserScan
from ackermann_msgs.msg import AckermannDrive
from nav_msgs.msg import Odometry
from DQN import DQNAgentPytorch

FORWARD = 0
LEFT = 1
RIGHT = 2
REVERSE = 3
ACTION_SPACE = [FORWARD, LEFT, RIGHT, REVERSE]

class EvaderNode:
    def __init__(self, agent):

        rospy.init_node('evader_node', anonymous=True)

        rospy.Subscriber('car_1/scan', LaserScan, self.scan_callback)
        rospy.Subscriber('car_1/base/odom', Odometry, self.odom_callback)

        self.drive_pub = rospy.Publisher('car_1/command', AckermannDrive, queue_size=10)

        self.init_location = None
        self.current_location = None

        self.max_distance = 30
        self.obstacle_penalty = -250
        self.destination_reward = 500
        self.distance_threshold = 0.5
        self.collision_threshold = 0.2

        self.agent = agent
        self.action = 0

        self.is_training = True
        self.init_odom_received = False
        self.latest_odom_msg = None
        self.reached_destination = False
        self.final_destination = (10, 10)

        self.reward = 0
        self.max_episodes = 10
        self.max_steps_per_episode = 100

    def process_scan(self, scan):
        state = np.array(scan.ranges)[::10]
        return state

    def check_collision(self, scan):
        for i in range(len(scan.ranges)):
            if scan.ranges[i] < self.collision_threshold:
                return True
        return False

    def action_to_drive_msg(self, action):
        drive_msg = AckermannDrive()
        if action == FORWARD:
            drive_msg.speed = 2.0
            drive_msg.steering_angle = 0.0
        elif action == LEFT:
            drive_msg.speed = 2.0
            drive_msg.steering_angle = float(random.uniform(math.pi / 4, math.pi / 2))
        elif action == RIGHT:
            drive_msg.speed = 2.0
            drive_msg.steering_angle = float(random.uniform(-math.pi / 2, -math.pi / 4))
        
        return  drive_msg
    
    def reverse_car(self, reverse_distance):
        reverse_drive_msg = AckermannDrive()
        reverse_drive_msg.speed = -2.0
        reverse_drive_msg.steering_angle = 0.0
        reverse_duration = reverse_distance / (-reverse_drive_msg.speed)

        start_time = time.time()
        while time.time() - start_time < reverse_duration:
            self.drive_pub.publish(reverse_drive_msg)
            rospy.sleep(0.1)
        
        reverse_drive_msg.speed = 0.0
        self.drive_pub.publish(reverse_drive_msg)
        # rospy.sleep(1)

    def calculate_reward(self, collision_detected, done):
        reward = 0
        if done:
            reward += self.destination_reward

        if collision_detected:        
            reward += self.obstacle_penalty
        
        current_dist_diff = np.linalg.norm(np.array(self.final_destination) - np.array(self.current_location)[:2])
        init_dist_diff = np.linalg.norm(np.array(self.final_destination) - np.array(self.init_location))

        if current_dist_diff < init_dist_diff:
            reward += 100
        else:
            reward -= 100
        return reward
        
    def update_reward(self, collision_detected, done):
        self.reward += self.calculate_reward(collision_detected, done)

    def scan_callback(self, scan):
        state = self.process_scan(scan)
        action = self.agent.choose_action(state, test=not self.is_training)

        collision_detected = self.check_collision(scan)
        
        if collision_detected and not self.reached_destination:
            print('Collision Detected')
            self.reverse_car(0.2) # Set the reverse distance here
            action = self.agent.choose_action(state, test=not self.is_training)
            
        drive_msg = self.action_to_drive_msg(action)
        
        
        done = False

        if self.is_training:
            print(self.current_location)
            if np.linalg.norm(np.array(self.final_destination) - np.array(self.current_location)[:2]) < self.distance_threshold:
                done = True
                self.reached_destination = True
            print("Reward Before",self.reward)
            self.update_reward(collision_detected, done)
            # print(state, len(state))
            print("Reward After",self.reward)
            self.agent.add_experience(state, action, self.reward, state, done)
            self.agent.train(32) 

            if done:
                self.reached_destination = True
                rospy.sleep(1)
        else:
            self.agent.load_weights('model_weights.pth')
            action = self.agent.choose_action(state, test=True)

        self.drive_pub.publish(drive_msg)

    def update_current_location(self, odom_msg):
        position = odom_msg.pose.pose.position
        orientation = odom_msg.pose.pose.orientation
        euler_angles = tf.transformations.euler_from_quaternion(
            (orientation.x, orientation.y, orientation.z, orientation.w))

        self.current_location = ( position.x, position.y, euler_angles[2])  # (x, y, yaw)
        
        print("Current Location of the Car:", self.current_location[:2])

    def odom_callback(self, odom_msg):

        if not self.init_odom_received:
            self.init_location = (odom_msg.pose.pose.position.x,
                                  odom_msg.pose.pose.position.y)
            print(f"Intial Location of the Car: {self.init_location}")
            self.init_odom_received = True

        self.update_current_location(odom_msg)
        self.latest_odom_msg = odom_msg

    def run(self):
        self.current_episode = 0
        episode_step = 0
        while not rospy.is_shutdown() and self.current_episode < self.max_episodes:
            rospy.spin()

            if self.reached_destination:
                self.current_episode += 1
                self.reward = 0
                self.reached_destination = False
                
                if self.current_episode % 10 == 0:
                    print(f"Completed Episode {self.current_episode}")
                
                rospy.sleep(1)

if __name__ == '__main__':
    try:
        action_space = ACTION_SPACE
        observation_space = np.zeros((109,))  # Based on the number of data points in the lidar scan

        agent = DQNAgentPytorch(action_space, observation_space)
        node = EvaderNode(agent)

        while not node.init_odom_received:
            rospy.sleep(0.1)
        
        node.run()

    except rospy.ROSInterruptException: 
        pass

    plot_rewards(rewards)