class SalgovFilterAgent:
    def __init__(self, num_features, possible_window_lengths, possible_polyorders, learning_rate=0.1, discount_factor=0.9):
        self.num_features = num_features
        self.possible_window_lengths = possible_window_lengths
        self.possible_polyorders = possible_polyorders
        self.learning_rate = learning_rate
        self.discount_factor = discount_factor
        self.q_table = {}  # (feature_index, state) -> {action: Q-value}
        self.filter_params = [(possible_window_lengths[0], possible_polyorders[0])] * num_features # Initialize with default

    def get_state(self):
        # The state is simply the current filter parameters for all features
        return tuple(self.filter_params)

    def get_possible_actions(self):
        actions = []
        for i in range(self.num_features):
            for window in self.possible_window_lengths:
                for polyorder in self.possible_polyorders:
                    actions.append((i, window, polyorder))
        return actions

    def choose_action(self, epsilon=0.1):
        if np.random.rand() < epsilon:
            # Explore: choose a random action
            feature_index = np.random.choice(self.num_features)
            window = np.random.choice(self.possible_window_lengths)
            polyorder = np.random.choice(self.possible_polyorders)
            return feature_index, window, polyorder
        else:
            # Exploit: choose the action with the highest Q-value for a randomly selected feature
            feature_index = np.random.choice(self.num_features)
            state = self.get_state()
            if (feature_index, state) in self.q_table:
                q_values = self.q_table[(feature_index, state)]
                best_action = None
                best_q = -np.inf
                for action, q in q_values.items():
                    if q > best_q:
                        best_q = q
                        best_action = action
                if best_action:
                    return best_action[0], best_action[1], best_action[2]

            # If no Q-values for this state, choose a random action for that feature
            window = np.random.choice(self.possible_window_lengths)
            polyorder = np.random.choice(self.possible_polyorders)
            return feature_index, window, polyorder

    def update_filter_params(self, feature_index, window, polyorder):
        self.filter_params[feature_index] = (window, polyorder)

    def apply_filters(self, data):
        filtered_data = data.copy()
        for i in range(self.num_features):
            window, polyorder = self.filter_params[i]
            try:
                if window >= 3 and window % 2 == 1 and polyorder < window:
                    filtered_data[:, i] = savgol_filter(data[:, i], window, polyorder)
            except ValueError as e:
                # Handle cases where savgol_filter might fail (e.g., window too large)
                print(f"Error applying filter to feature {i}: {e}")
        return filtered_data

    def update_q_table(self, old_state, action, reward, next_state):
        feature_index = action[0]
        if (feature_index, old_state) not in self.q_table:
            self.q_table[(feature_index, old_state)] = {}
        if action not in self.q_table[(feature_index, old_state)]:
            self.q_table[(feature_index, old_state)][action] = 0

        old_q_value = self.q_table[(feature_index, old_state)][action]
        best_next_q = 0

        if (feature_index, next_state) in self.q_table:
            if self.q_table[(feature_index, next_state)]: # Check if the dictionary is not empty
                best_next_q = max(self.q_table[(feature_index, next_state)].values())

        td_error = reward + self.discount_factor * best_next_q - old_q_value
        self.q_table[(feature_index, old_state)][action] += self.learning_rate * td_error
# Define the possible Salgov filter parameters
possible_window_lengths = [2, 3, 4, 5]
possible_polyorders = [1, 2, 3]




# Training the RL agent
num_rl_episodes = 100
epsilon = 0.1
alpha = 0.1
gamma = 0.9

rl_agent = SalgovFilterAgent(predictions.shape[1], possible_window_lengths, possible_polyorders)

for episode in tqdm(range(num_rl_episodes), desc="RL Episodes"):
    old_filter_params = list(rl_agent.filter_params) # Store the parameters before the action
    old_state = rl_agent.get_state()

    # Agent chooses an action (feature to adjust, new window, new polyorder)
    feature_index_to_adjust, new_window, new_polyorder = rl_agent.choose_action(epsilon)
    action = (feature_index_to_adjust, new_window, new_polyorder)

    # Apply the selected filter parameters to the predictions
    rl_agent.update_filter_params(feature_index_to_adjust, new_window, new_polyorder)
    filtered_predictions = rl_agent.apply_filters(predictions.values)

    # Make predictions using the base model with the filtered features
    predicted_values = make_predictions(base_model, filtered_predictions, in_sequence_length=10, out_sequence_length=1) # Adjust sequence lengths

    # Evaluate the performance (get the reward)
    current_smape_loss = evaluate_test_set(df_validation.values.T, df_validation.values, predicted_values, feature_names)
    reward = -current_smape_loss  # We want to maximize the reward, which is the negative sMAPE

    # Observe the new state
    next_state = rl_agent.get_state()

    # Update the Q-table
    rl_agent.update_q_table(old_state, action, reward, next_state)

    # Optionally, decay epsilon over time to reduce exploration
    epsilon = max(0.01, epsilon * 0.99)

# After training, the rl_agent.filter_params will hold the learned filter parameters
print("Learned Salgov Filter Parameters:")
for i, params in enumerate(rl_agent.filter_params):
    print(f"Feature {i}: Window={params[0]}, Polyorder={params[1]}")

Error:
RL Episodes:   0%|          | 0/100 [00:00<?, ?it/s]
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[56], line 11
      9 # Apply the selected filter parameters to the predictions
     10 rl_agent.update_filter_params(feature_index_to_adjust, new_window, new_polyorder)
---> 11 filtered_predictions = rl_agent.apply_filters(predictions.values)
     13 # Make predictions using the base model with the filtered features
     14 predicted_values = make_predictions(base_model, filtered_predictions, in_sequence_length=10, out_sequence_length=1) # Adjust sequence lengths

Cell In[49], line 59, in SalgovFilterAgent.apply_filters(self, data)
     57 try:
     58     if window >= 3 and window % 2 == 1 and polyorder < window:
---> 59         filtered_data[:, i] = savgol_filter(data[:, i], window, polyorder)
     60 except ValueError as e:
     61     # Handle cases where savgol_filter might fail (e.g., window too large)
     62     print(f"Error applying filter to feature {i}: {e}")

NameError: name 'savgol_filter' is not defined