How to handle having multiple clashes .Here is my code for handling single clashes by two element


import gymnasium as gym
import tensorflow as tf
import numpy as np

# Define constants
num_states = 4  # Number of features defining the state
num_actions = 6  # Number of possible resolution actions
upper_bound = -10  # Maximum value for resolution actions
lower_bound = 10  # Minimum value for resolution actions


# Define the Ornstein-Uhlenbeck noise process
class OUActionNoise:
    def __init__(self, mean, std_deviation, theta=0.15, dt=1e-2, x_initial=None):
        self.theta = theta
        self.mean = mean
        self.std_dev = std_deviation
        self.dt = dt
        self.x_initial = x_initial
        self.reset()

    def __call__(self):
        x = (
            self.x_prev
            + self.theta * (self.mean - self.x_prev) * self.dt
            + self.std_dev * np.sqrt(self.dt) * np.random.normal(size=self.mean.shape)
        )
        self.x_prev = x
        return x

    def reset(self):
        if self.x_initial is not None:
            self.x_prev = self.x_initial
        else:
            self.x_prev = np.zeros_like(self.mean)


# Define the Buffer class for experience replay
class Buffer:
    def __init__(self, buffer_capacity=100000, batch_size=64):
        self.buffer_capacity = buffer_capacity
        self.batch_size = batch_size
        self.buffer_counter = 0
        self.state_buffer = np.zeros((self.buffer_capacity, num_states))
        self.action_buffer = np.zeros((self.buffer_capacity, num_actions))
        self.reward_buffer = np.zeros((self.buffer_capacity, 1))
        self.next_state_buffer = np.zeros((self.buffer_capacity, num_states))

    def record(self, obs_tuple):
        print('obs_tuple================================',obs_tuple)
        index = self.buffer_counter % self.buffer_capacity
        print('====buffer_counter , buffer_capicity, index========',self.buffer_counter, self.buffer_capacity, index)
        self.state_buffer[index] = obs_tuple[0]
        self.action_buffer[index] = obs_tuple[1]
        self.reward_buffer[index] = obs_tuple[2]
        self.next_state_buffer[index] = obs_tuple[3]
        self.buffer_counter += 1

    @tf.function
    def update(self, state_batch, action_batch, reward_batch, next_state_batch):
        with tf.GradientTape() as tape:
            target_actions = target_actor(next_state_batch, training=True)
            y = reward_batch + gamma * target_critic(
                [next_state_batch, target_actions], training=True
            )
            critic_value = critic_model([state_batch, action_batch], training=True)
            critic_loss = tf.math.reduce_mean(tf.math.square(y - critic_value))

        critic_grad = tape.gradient(critic_loss, critic_model.trainable_variables)
        critic_optimizer.apply_gradients(
            zip(critic_grad, critic_model.trainable_variables)
        )

        with tf.GradientTape() as tape:
            actions = actor_model(state_batch, training=True)
            critic_value = critic_model([state_batch, actions], training=True)
            actor_loss = -tf.math.reduce_mean(critic_value)

        actor_grad = tape.gradient(actor_loss, actor_model.trainable_variables)
        actor_optimizer.apply_gradients(
            zip(actor_grad, actor_model.trainable_variables)
        )

    def learn(self):
        record_range = min(self.buffer_counter, self.buffer_capacity)
        batch_indices = np.random.choice(record_range, self.batch_size)
        state_batch = tf.convert_to_tensor(self.state_buffer[batch_indices])
        action_batch = tf.convert_to_tensor(self.action_buffer[batch_indices])
        reward_batch = tf.convert_to_tensor(self.reward_buffer[batch_indices])
        reward_batch = tf.cast(reward_batch, dtype=tf.float32)
        next_state_batch = tf.convert_to_tensor(self.next_state_buffer[batch_indices])

        self.update(state_batch, action_batch, reward_batch, next_state_batch)


# Define the update_target function for updating target networks
@tf.function
def update_target(target_weights, weights, tau):
    for (a, b) in zip(target_weights, weights):
        a.assign(b * tau + a * (1 - tau))




# Define the actor and critic neural networks
def get_actor():
    last_init = tf.random_uniform_initializer(minval=-0.003, maxval=0.003)
    inputs = tf.keras.layers.Input(shape=(num_states,))
    out = tf.keras.layers.Dense(256, activation="relu")(inputs)
    out = tf.keras.layers.Dense(256, activation="relu")(out)
    outputs = tf.keras.layers.Dense(6, activation="tanh", kernel_initializer=last_init)(out)
    outputs = outputs * upper_bound
    model = tf.keras.Model(inputs, outputs)
    return model

def get_critic():
    state_input = tf.keras.layers.Input(shape=(num_states,))
    state_out = tf.keras.layers.Dense(16, activation="relu")(state_input)
    state_out = tf.keras.layers.Dense(32, activation="relu")(state_out)
    action_input = tf.keras.layers.Input(shape=(num_actions,))
    action_out = tf.keras.layers.Dense(32, activation="relu")(action_input)
    concat = tf.keras.layers.Concatenate()([state_out, action_out])
    out = tf.keras.layers.Dense(256, activation="relu")(concat)
    out = tf.keras.layers.Dense(256, activation="relu")(out)
    outputs = tf.keras.layers.Dense(1)(out)
    model = tf.keras.Model([state_input, action_input], outputs)
    return model


# Define the policy function
def policy(state, noise_object):
    sampled_actions = tf.squeeze(actor_model(state))
    noise = noise_object()
    sampled_actions = sampled_actions.numpy() + noise
    legal_action = np.clip(sampled_actions, lower_bound, upper_bound)
    #return [np.squeeze(legal_action)]
    return np.squeeze(legal_action)


# Define DDPG hyperparameters
std_dev = 0.2
ou_noise = OUActionNoise(mean=np.zeros(1), std_deviation=float(std_dev) * np.ones(1))
actor_model = get_actor()
critic_model = get_critic()
target_actor = get_actor()
target_critic = get_critic()
target_actor.set_weights(actor_model.get_weights())
target_critic.set_weights(critic_model.get_weights())
critic_lr = 0.002
actor_lr = 0.001
critic_optimizer = tf.keras.optimizers.Adam(critic_lr)
actor_optimizer = tf.keras.optimizers.Adam(actor_lr)
total_episodes = 100 # 100
gamma = 0.99
tau = 0.005
buffer = Buffer(50000, 64)


class ConstructionEnvironment(gym.Env):
    def __init__(self, num_states, num_actions, lower_bound, upper_bound, max_episode_steps):
        super(ConstructionEnvironment, self).__init__()

        # Define action and observation spaces
        self.action_space = gym.spaces.Box(low=lower_bound, high=upper_bound, shape=(num_actions,), dtype=np.float32)
        self.observation_space = gym.spaces.Box(low=-np.inf, high=np.inf, shape=(num_states,), dtype=np.float32)

        # Initialize the state
        self.state = np.zeros(num_states)

        # Episode-related variables
        self.episode_steps = 0
        self.max_episode_steps = max_episode_steps
        self.max_distance = np.sqrt(num_states)  # Maximum Euclidean distance for normalization

    def reset(self):
        # Reset the state and episode steps
        self.state = np.random.uniform(low=-1, high=1, size=self.observation_space.shape)
        self.episode_steps = 0
        return self.state

    def step(self, action):
        # Implement the actual step function based on your application
        # Update the state, calculate reward, and check if the episode is done
        self.episode_steps += 1

        # Update state (simple example: add random noise to each state element)
        self.state += np.random.normal(0, 0.1, size=self.observation_space.shape)

        # Placeholder for reward calculation - maximize the negative Euclidean distance from the origin
        reward = -np.linalg.norm(self.state) / self.max_distance
        reward = self.calculate_reward(action)  # Replace with your reward calculation

        done = (self.episode_steps >= self.max_episode_steps) or (reward == 10)

        return self.state, reward, done, {}
    
    def calculate_reward(self,action):
        reward = -0.5
        # Example: Simple reward calculation based on the current state
        # Replace this with your actual reward calculation logic
        print('action=======inside Revit===========',action)
        # data = np.array([-1.0, -1.0, 1.0, 1.0, 0.71324176, -1.0])
        data  = action

        # Extract positions of element1 and element2
        element1_position = data[:3]
        element2_position = data[3:]

        # Calculate Euclidean distance
        euclidean_distance = np.linalg.norm(element1_position - element2_position)

        # Print the result
        print("Euclidean Distance:", euclidean_distance)
        euclidian_d.append(euclidean_distance)
        if 1.4 <= euclidean_distance <= 1.6:
            reward = 10
        # return -np.sum(np.square(self.state))
        return reward
    

# Example of how to use the custom environment with specified parameters
num_states = 4
num_actions = 6
lower_bound = -1
upper_bound = 1
max_episode_steps = 200

env = ConstructionEnvironment(num_states=num_states, num_actions=num_actions, lower_bound=lower_bound, upper_bound=upper_bound, max_episode_steps=max_episode_steps)
env.max_episode_steps


# Train the DDPG model
reward_per_episode = []
def train_ddpg():
    # env = ConstructionEnvironment()
    for ep in range(total_episodes):
        state = env.reset()
        episodic_reward = 0

        for i in range(env.max_episode_steps):
            tf_state = tf.expand_dims(tf.convert_to_tensor(state), 0)
            action = policy(tf_state, ou_noise)
            print('action==================&&&&&&&&&&&&====',type(action), action)
            next_state, reward, done, _ = env.step(action)
            # reward_data.append(reward)
            print('next_state, reward, done, _====================',next_state, reward, done, _)
            buffer.record((state, action, reward, next_state))
            episodic_reward += reward
            buffer.learn()
            update_target(target_actor.variables, actor_model.variables, tau)
            update_target(target_critic.variables, critic_model.variables, tau)

            if done:
                break

            state = next_state

        print("Episode * {} * Avg Reward ==> {}".format(ep, episodic_reward))
        reward_per_episode.append(episodic_reward)

# Run the DDPG training
train_ddpg()