import gym
import numpy as np
import matplotlib.pyplot as plt

class GridWorld(gym.Env):
    def __init__(self):
        self.grid = np.array([
            [-1, -1, -1, 0],
            [-1, 0, 0, 0],
            [-1, 0, 0, 0],
            [-1, -1, -1, 10]
        ])

        self.observation_space = gym.spaces.Discrete(16)
        self.action_space = gym.spaces.Discrete(4)
        self.reward_range = (-5, 10)
        self.agent_pos = (0, 0)
        self.rewards = []

    def step(self, action):
        # Get the current state and action
        current_state = self.agent_pos
        action = action

        # Calculate the next state
        if action == 0:  # up
            next_state = (current_state[0] - 1, current_state[1])
        elif action == 1:  # down
            next_state = (current_state[0] + 1, current_state[1])
        elif action == 2:  # left
            next_state = (current_state[0], current_state[1] - 1)
        elif action == 3:  # right
            next_state = (current_state[0], current_state[1] + 1)

        # Check if the next state is valid
        if next_state[0] < 0 or next_state[0] >= self.grid.shape[0] or next_state[1] < 0 or next_state[1] >= self.grid.shape[1]:
            next_state = current_state

        # Get the reward
        reward = self.grid[next_state[0]][next_state[1]]

        # Check if the episode is done
        done = reward == 10

        # Update the agent's position
        self.agent_pos = next_state

        # Return the next state, reward, and done
        return next_state, reward, done


    def reset(self):
        # Reset the agent's position to the start
        self.agent_pos = (0, 0)
        return self.agent_pos
        
    def render(self):
        plt.figure(figsize=(5, 5))
        # Add grid lines for each cell
        plt.grid(True, which='both', color='black', linewidth=5)
        plt.imshow(self.grid, cmap='viridis')
        plt.xticks([])
        plt.yticks([])
        # Mark start and end positions with text labels
        plt.text(0, 0, 'START', horizontalalignment='center', verticalalignment='center', fontsize=14, color='white')
        plt.text(3, 3, 'END', horizontalalignment='center', verticalalignment='center', fontsize=14, color='white')
        # Mark agent position with a red circle
        plt.scatter([self.agent_pos[1]], [self.agent_pos[0]], c='red', s=100)
        plt.show()


    def plot_rewards(self):
        plt.plot(self.rewards)
        plt.title("Rewards per Episode")
        plt.xlabel("Episode")
        plt.ylabel("Reward")
        plt.show()       ------------------------------------------------environment--------------------------------------------------------------------------------------------------------------------------------                                                                                                                                                                                                                                                                                               
 import gym
import numpy as np

# Initialize environment, Q-table, and hyperparameters
env = GridWorld()
n_states = env.observation_space.n
n_actions = env.action_space.n
Q = np.zeros((n_states, n_actions))

# SARSA hyperparameters
episodes = 500
alpha = 0.1
gamma = 0.99
epsilon = 1.0
epsilon_decay = 0.995
epsilon_min = 0.01
max_timesteps = 100

# SARSA Algorithm
total_rewards = []
for episode in range(episodes):
    state_tuple = env.reset()
    state = state_tuple[0] * 4 + state_tuple[1]  # Convert the state tuple to single integer
    action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1)))

    total_reward = 0
    for t in range(max_timesteps):
        next_state_tuple, reward, done = env.step(action)
        next_state = next_state_tuple[0] * 4 + next_state_tuple[1]  # Convert the next_state tuple to single integer
        next_action = np.argmax(Q[next_state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1)))

        # Update Q-value
        target = reward + gamma * Q[next_state, next_action]
        Q[state, action] += alpha * (target - Q[state, action])

        state, action = next_state, next_action
        total_reward += reward

        if done:
            break

    epsilon = max(epsilon_min, epsilon * epsilon_decay)
    total_rewards.append(total_reward)
--------------------I have implemented saarsa like this -------------#Plot the SARSA algorithm results:

plt.plot(total_rewards)
plt.xlabel('Episode')
plt.ylabel('Total Reward per Episode')
plt.title('SARSA: Total Reward per Episode')
plt.show()----------------------------------------------------------------------------------------------------------------------------I also need the graph for epsilon delay ----------------------------------------• • Applying SARSA to solve the environment defined in Part 1. Plots should
include epsilon decay and total reward per episode.
--give code to Run your environment for at least 10
episodes, where the agent chooses only greedy actions from the learned
policy. Plot should include the total reward per episode.