Update README.md

README.md

@@ -43,64 +43,26 @@ Reinforcement Learning Tips and Tricks: https://stable-baselines3.readthedocs.io
 
 A Github Training Framework : https://github.com/DLR-RM/rl-baselines3-zoo
 
-Poe (MrProgrammer Bot):
-I tried to follow what this was saying but I had a hard time understanding.
+Poe (GPT-4): Showed me how to use Optuna to do automated hyperparameter optimization, but I was still understanding how it worked and couldn't get it to run properly.
 ```
-import gym
+import optuna
 from stable_baselines3 import A2C
-from stable_baselines3.common.envs import DummyVecEnv
-from stable_baselines3.common.evaluation import evaluate_policy
-from stable_baselines3.common.callbacks import EvalCallback
-from stable_baselines3.common.env_checker import check_env
-from stable_baselines3.common.vec_env import VecNormalize
-```
-
-### Next, load and prepare your environment:
-```
-env = gym.make('your_environment_name')  # Replace with the name of your environment
-env = DummyVecEnv([lambda: env])
-env = VecNormalize(env, norm_obs=True, norm_reward=False, clip_obs=10.)
-```
-
-### Now, define a function to train and evaluate your A2C agent:
-```
-def train_and_evaluate(hyperparameters):
-    model = A2C("MlpPolicy", env, verbose=0, **hyperparameters)
-    
-    eval_env = gym.make('your_evaluation_environment_name')  # Replace with the name of your evaluation environment
-    eval_env = DummyVecEnv([lambda: eval_env])
-    eval_env = VecNormalize(eval_env, norm_obs=True, norm_reward=False, clip_obs=10.)
+from stable_baselines3.common.env_util import make_vec_env
 
-    eval_callback = EvalCallback(eval_env, best_model_save_path='./logs/',
-                                 log_path='./logs/', eval_freq=10000,
-                                 deterministic=True, render=False)
+def optimize_agent(trial):
+    learning_rate = trial.suggest_loguniform('learning_rate', 1e-5, 1)
+    gamma = trial.suggest_uniform('gamma', 0.8, 0.9999)
+    gae_lambda = trial.suggest_uniform('gae_lambda', 0.8, 0.99)
+    n_steps = trial.suggest_int('n_steps', 5, 20)
     
-    model.learn(total_timesteps=int(1e5), callback=eval_callback)
+    model = A2C('MlpPolicy', env, verbose=0, learning_rate=learning_rate, gamma=gamma, gae_lambda=gae_lambda, n_steps=n_steps)
+    model.learn(total_timesteps=5000)
+    rewards = sum(model.rollout_buffer.rewards)
     
-    mean_reward, _ = evaluate_policy(model, eval_env, n_eval_episodes=10)
-    
-    return mean_reward
-```
-
-### Now, we can define the hyperparameters grid and start the grid search:
-```
-hyperparameters_grid = {
-    'gamma': [0.99, 0.95],
-    'learning_rate': [0.001, 0.0001],
-    'ent_coef': [0.01, 0.1],
-    # Add other hyperparameters of interest
-}
+    return rewards
 
-best_reward = float('-inf')
-best_hyperparameters = {}
+study = optuna.create_study(direction='maximize')
+study.optimize(optimize_agent, n_trials=100)
 
-for hyperparameters in hyperparameters_grid:
-    mean_reward = train_and_evaluate(hyperparameters)
-    
-    if mean_reward > best_reward:
-        best_reward = mean_reward
-        best_hyperparameters = hyperparameters
-
-print("Best hyperparameters:", best_hyperparameters)
-```
-In this grid search, we specify a range of values for each hyperparameter of interest. The train_and_evaluate function trains the A2C agent with the given hyperparameters and evaluates its performance. We then update the best hyperparameters if the current combination achieves a higher reward.
+print('Best hyperparameters:', study.best_params)
+```