import numpy as np
import pandas as pd

def main():
    # Load your CSV data
    data = pd.read_csv('/content/drive/MyDrive/BIST_10/MDP_IRL/MLB/AKBNK.csv')  # Replace 'your_data.csv' with your actual file name

    # Number of states and actions
    n_states = 15  # From your data (states are 0 to 14)
    n_actions = 3  # From your data (actions are 0, 1, 2)

    # Convert data to numpy array
    state_action_nextstate = data[['State', 'Action', 'Next_State']].to_numpy()

    # Process data into trajectories
    trajectory_length = 20
    n_observations = state_action_nextstate.shape[0]
    n_full_trajectories = n_observations // trajectory_length

    trajectories = []
    for i in range(n_full_trajectories):
        traj = state_action_nextstate[i*trajectory_length : (i+1)*trajectory_length]
        trajectories.append(traj)

    # Add remaining data as the last trajectory
    if n_observations % trajectory_length != 0:
        last_traj = state_action_nextstate[n_full_trajectories * trajectory_length:]
        # Pad the last trajectory to length 20
        last_step = last_traj[-1]
        padding_length = trajectory_length - last_traj.shape[0]
        padding = np.tile(last_step, (padding_length, 1))
        last_traj_padded = np.vstack((last_traj, padding))
        trajectories.append(last_traj_padded)

    # Convert list of trajectories to numpy array
    trajectories = np.array(trajectories)

    # Build transition probability tensor
    transition_counts = np.zeros((n_states, n_actions, n_states))

    for s, a, s_next in state_action_nextstate:
        transition_counts[int(s), int(a), int(s_next)] += 1

    # Add smoothing to avoid zero probabilities
    smoothing = 1e-5
    transition_counts += smoothing

    transition_probability = np.zeros((n_states, n_actions, n_states))
    for s in range(n_states):
        for a in range(n_actions):
            total = np.sum(transition_counts[s, a, :])
            transition_probability[s, a, :] = transition_counts[s, a, :] / total

    # Create feature matrix (one-hot encoding)
    feature_matrix = np.eye(n_states)

    # Set discount factor
    discount = 0.9

    # Set algorithm parameters
    epochs = 100
    learning_rate = 0.01

    # Run MaxEnt IRL
    reward = irl(feature_matrix, n_actions, discount, transition_probability,
                 trajectories, epochs, learning_rate)

    print("Learned reward function:")
    print(reward)

def irl(feature_matrix, n_actions, discount, transition_probability,
        trajectories, epochs, learning_rate):
    """
    Find the reward function for the given trajectories.
    """
    n_states, d_states = feature_matrix.shape

    # Initialize weights.
    alpha = np.random.uniform(size=(d_states,))

    # Calculate the feature expectations \tilde{phi}.
    feature_expectations = find_feature_expectations(feature_matrix,
                                                     trajectories)

    # Gradient descent on alpha.
    for i in range(epochs):
        r = feature_matrix.dot(alpha)
        expected_svf = find_expected_svf(n_states, r, n_actions, discount,
                                         transition_probability, trajectories)
        grad = feature_expectations - feature_matrix.T.dot(expected_svf)

        alpha += learning_rate * grad

    return feature_matrix.dot(alpha).reshape((n_states,))

def find_feature_expectations(feature_matrix, trajectories):
    """
    Find the feature expectations for the given trajectories.
    """
    T = trajectories.shape[0]  # Number of trajectories
    feature_expectations = np.zeros(feature_matrix.shape[1])

    for traj in trajectories:
        for state, action, next_state in traj:
            feature_expectations += feature_matrix[int(state)]

    feature_expectations /= T * trajectories.shape[1]  # Normalize over all time steps

    return feature_expectations

def find_expected_svf(n_states, r, n_actions, discount,
                      transition_probability, trajectories):
    """
    Find the expected state visitation frequencies.
    """
    # Compute policy using soft value iteration
    policy = compute_policy(n_states, n_actions, transition_probability, r, discount)

    # Initial state probabilities
    T = trajectories.shape[0]
    start_state_counts = np.zeros(n_states)
    for traj in trajectories:
        start_state = int(traj[0, 0])
        start_state_counts[start_state] += 1
    p_start_state = start_state_counts / T

    # Compute expected state visitation frequencies
    trajectory_length = trajectories.shape[1]
    expected_svf = np.zeros((n_states, trajectory_length))
    expected_svf[:, 0] = p_start_state

    for t in range(1, trajectory_length):
        expected_svf[:, t] = 0
        for s in range(n_states):
            for a in range(n_actions):
                for s_next in range(n_states):
                    expected_svf[s_next, t] += expected_svf[s, t-1] * policy[s, a] * transition_probability[s, a, s_next]

    return expected_svf.sum(axis=1)

def compute_policy(n_states, n_actions, transition_probability, reward, discount):
    """
    Compute the policy using soft value iteration.
    """
    max_iters = 1000
    threshold = 1e-5
    V = np.zeros(n_states)

    for i in range(max_iters):
        V_prev = V.copy()
        Q = np.zeros((n_states, n_actions))
        for s in range(n_states):
            for a in range(n_actions):
                Q[s, a] = np.sum(transition_probability[s, a, :] * (reward + discount * V_prev))
        V = np.log(np.sum(np.exp(Q), axis=1))  # Soft value iteration

        if np.max(np.abs(V - V_prev)) < threshold:
            break

    # Compute policy
    policy = np.zeros((n_states, n_actions))
    for s in range(n_states):
        for a in range(n_actions):
            Q_sa = np.sum(transition_probability[s, a, :] * (reward + discount * V))
            policy[s, a] = np.exp(Q_sa - V[s])  # Subtract V[s] for normalization

    # Normalize policy
    policy = policy / np.sum(policy, axis=1, keepdims=True)
    return policy

if __name__ == '__main__':
    main()

Kodu kontrol edebilir misin? Eksik, hatalı ya da değişmesi gereken her yeri titizlikle ele almanı istiyorum.