import numpy as np
from scipy.optimize import minimize

class MaxEntIRL:
    def __init__(self, num_states, num_actions, discount_factor=0.99, learning_rate=0.01, num_iterations=100):
        self.num_states = num_states
        self.num_actions = num_actions
        self.discount_factor = discount_factor
        self.learning_rate = learning_rate
        self.num_iterations = num_iterations

        # Initialize transition probabilities
        self.P = np.zeros((num_states, num_actions, num_states))

        # Initialize feature expectations
        self.feature_expectations = np.zeros(num_states)

        # Initialize rewards
        self.rewards = np.zeros(num_states)

    def set_transition_probabilities(self, transitions):
        for (s, a, s_next) in transitions:
            self.P[s, a, s_next] += 1
        # Normalize
        self.P = self.P / (np.sum(self.P, axis=2, keepdims=True) + 1e-8)  # Add small epsilon to avoid division by zero

    def compute_state_visitation_freq(self, policy):
        T = 1000  # Time horizon
        mu = np.zeros((T, self.num_states))
        mu[0] = np.ones(self.num_states) / self.num_states  # Initial state distribution

        for t in range(1, T):
            for s in range(self.num_states):
                for a in range(self.num_actions):
                    mu[t, s] += np.sum(mu[t-1] * self.P[:, a, s] * policy[:, a])

        return np.sum(mu, axis=0)

    def compute_expected_svf(self, policy):
        svf = self.compute_state_visitation_freq(policy)
        return np.dot(svf, np.eye(self.num_states))

    def compute_gradient(self, rewards, expert_feature_expectations, policy):
        expected_svf = self.compute_expected_svf(policy)
        gradient = expert_feature_expectations - expected_svf
        return gradient

    def optimize_rewards(self, expert_feature_expectations):
        def objective(rewards):
            policy = self.compute_policy(rewards)
            expected_svf = self.compute_expected_svf(policy)
            return -np.dot(expert_feature_expectations, rewards) + np.dot(expected_svf, rewards)

        def gradient(rewards):
            policy = self.compute_policy(rewards)
            return self.compute_gradient(rewards, expert_feature_expectations, policy)

        initial_rewards = np.random.rand(self.num_states)
        result = minimize(objective, initial_rewards, method='L-BFGS-B', jac=gradient)
        return result.x

    def compute_policy(self, rewards):
        Q = np.zeros((self.num_states, self.num_actions))
        for _ in range(100):  # Value iteration
            for s in range(self.num_states):
                for a in range(self.num_actions):
                    Q[s, a] = rewards[s] + self.discount_factor * np.sum(self.P[s, a] * np.max(Q, axis=1))

        return np.exp(Q) / np.sum(np.exp(Q), axis=1, keepdims=True)

    def fit(self, expert_trajectories):
        # Compute expert feature expectations
        expert_feature_expectations = np.zeros(self.num_states)
        for trajectory in expert_trajectories:
            for state in trajectory:
                expert_feature_expectations[state] += 1
        expert_feature_expectations /= len(expert_trajectories)

        # Optimize rewards
        self.rewards = self.optimize_rewards(expert_feature_expectations)

        return self.rewards

    def get_optimal_policy(self):
        return self.compute_policy(self.rewards)

def load_data(file_path):
    data = np.loadtxt(file_path, dtype=int, delimiter=',', skiprows=1)[:, 1:]
    return data

def process_data(data):
    transitions = [(s, a, s_next) for s, a, s_next in data]
    expert_trajectories = []
    current_trajectory = [data[0, 0]]

    for i in range(1, len(data)):
        if data[i, 0] == data[i-1, 2]:  # If current state is the same as previous next state
            current_trajectory.append(data[i, 0])
        else:
            expert_trajectories.append(current_trajectory)
            current_trajectory = [data[i, 0]]

    expert_trajectories.append(current_trajectory)
    return transitions, expert_trajectories

Kodu kontrol ederek detaylı olarak açıklar mısın?