import numpy as np
import pandas as pd
from scipy.optimize import minimize
from scipy.special import logsumexp

class MaxEntIRL:
    def __init__(self, num_states, num_actions, discount_factor=0.99, learning_rate=0.01, num_iterations=100):
        self.num_states = num_states
        self.num_actions = num_actions
        self.discount_factor = discount_factor
        self.learning_rate = learning_rate
        self.num_iterations = num_iterations

        # Initialize transition probabilities
        self.P = np.zeros((num_states, num_actions, num_states))

        # Initialize rewards
        self.rewards = np.zeros(num_states)

    def set_transition_probabilities(self, transitions):
        for (s, a, s_next) in transitions:
            self.P[s, a, s_next] += 1
        # Add a small value to avoid division by zero
        self.P += 1e-5
        # Normalize
        self.P = self.P / np.sum(self.P, axis=2, keepdims=True)

    def compute_state_visitation_freq(self, policy):
        T = 100  # Time horizon
        mu = np.zeros((T, self.num_states))
        mu[0] = np.ones(self.num_states) / self.num_states  # Initial state distribution

        for t in range(1, T):
            for s in range(self.num_states):
                mu[t, s] = sum([
                    mu[t-1, s_prev] * sum([
                        policy[s_prev, a] * self.P[s_prev, a, s]
                        for a in range(self.num_actions)
                    ]) for s_prev in range(self.num_states)
                ])

        return np.sum(mu, axis=0)

    def compute_expected_svf(self, policy):
        svf = self.compute_state_visitation_freq(policy)
        return svf  # Expected state visitation frequencies

    def compute_gradient(self, rewards, expert_feature_expectations, policy):
        expected_svf = self.compute_expected_svf(policy)
        gradient = expert_feature_expectations - expected_svf
        return gradient

    def optimize_rewards(self, expert_feature_expectations):
        def objective(rewards):
            policy = self.compute_policy(rewards)
            expected_svf = self.compute_expected_svf(policy)
            return -np.dot(expert_feature_expectations, rewards) + np.dot(expected_svf, rewards)

        def gradient(rewards):
            policy = self.compute_policy(rewards)
            return -self.compute_gradient(rewards, expert_feature_expectations, policy)

        initial_rewards = np.random.rand(self.num_states)
        result = minimize(objective, initial_rewards, method='L-BFGS-B', jac=gradient)
        return result.x

    def compute_policy(self, rewards):
        # Initialize value function
        V = np.zeros(self.num_states)
        # Soft value iteration
        for _ in range(self.num_iterations):
            V_prev = V.copy()
            Q = np.zeros((self.num_states, self.num_actions))
            for s in range(self.num_states):
                for a in range(self.num_actions):
                    # Compute Q-function using the soft Bellman update
                    Q[s, a] = rewards[s] + self.discount_factor * np.sum(self.P[s, a] * V_prev)
            # Update value function using log-sum-exp for numerical stability
            V = logsumexp(Q, axis=1)
        # Compute policy
        policy = np.exp(Q - V[:, None])  # Subtract V[s] for normalization
        # Ensure policy is normalized
        policy = policy / np.sum(policy, axis=1, keepdims=True)
        return policy

    def fit(self, expert_trajectories):
        # Compute expert feature expectations
        expert_feature_expectations = np.zeros(self.num_states)
        total_steps = 0
        for trajectory in expert_trajectories:
            for state in trajectory:
                expert_feature_expectations[state] += 1
                total_steps += 1
        expert_feature_expectations /= total_steps  # Normalize over total number of steps

        # Optimize rewards
        self.rewards = self.optimize_rewards(expert_feature_expectations)

        return self.rewards

    def get_optimal_policy(self):
        return self.compute_policy(self.rewards)

# Read your CSV data
data = pd.read_csv('/content/drive/MyDrive/BIST_10/MDP_IRL/MLB/AKBNK.csv')

# Set the number of states and actions according to your data
num_states = 15
num_actions = 3

# Prepare transitions from data
transitions = list(zip(data['State'], data['Action'], data['Next_State']))

# Create the MaxEnt IRL model
model = MaxEntIRL(num_states, num_actions)

# Set the transition probabilities
model.set_transition_probabilities(transitions)

# Prepare expert trajectories
trajectory_length = 20
num_trajectories = len(data) // trajectory_length

expert_trajectories = []
for i in range(num_trajectories):
    start_idx = i * trajectory_length
    end_idx = start_idx + trajectory_length
    trajectory = data['State'][start_idx:end_idx].tolist()
    expert_trajectories.append(trajectory)

# Fit the model
learned_rewards = model.fit(expert_trajectories)

# Print the learned rewards
print("Öğrenilen Ödül Fonksiyonu:")
for state, reward in enumerate(learned_rewards):
    print(f"Durum {state}: Ödül {reward:.4f}")

# Get the optimal policy
optimal_policy = model.get_optimal_policy()

# Print the optimal policy
print("\nOptimal Politika:")
action_names = ['Sat', 'Nötr', 'Satın Al']
for state in range(num_states):
    action = np.argmax(optimal_policy[state])
    print(f"Durum {state}: Eylem {action} ({action_names[action]})")

Kodu detaylı bir şekilde kontrol edip MaxEnt IRL'e olan uygunluğuna bakabilir misin? Herhangi bir değişiklik gerekiyorsa hazırlayıp son halini paylaş.