|
--- |
|
library_name: stable-baselines3 |
|
tags: |
|
- LunarLander-v2 |
|
- deep-reinforcement-learning |
|
- reinforcement-learning |
|
- stable-baselines3 |
|
model-index: |
|
- name: PPO |
|
results: |
|
- task: |
|
type: reinforcement-learning |
|
name: reinforcement-learning |
|
dataset: |
|
name: LunarLander-v2 |
|
type: LunarLander-v2 |
|
metrics: |
|
- type: mean_reward |
|
value: 255.78 +/- 22.96 |
|
name: mean_reward |
|
verified: false |
|
--- |
|
|
|
# **PPO** Agent playing **LunarLander-v2** |
|
This is a trained model of a **PPO** agent playing **LunarLander-v2** |
|
using the [stable-baselines3 library](https://github.com/DLR-RM/stable-baselines3). |
|
|
|
## Usage (with Stable-baselines3) |
|
TODO: Add your code |
|
|
|
|
|
```python |
|
!apt install swig cmake |
|
!pip install -r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit1/requirements-unit1.txt |
|
!sudo apt-get update |
|
!apt install python-opengl |
|
!apt install ffmpeg |
|
!apt install xvfb |
|
!pip3 install pyvirtualdisplay |
|
|
|
#might need to restart google colab to run virtual display |
|
#import os |
|
#os.kill(os.getpid(), 9) |
|
|
|
# Virtual display |
|
from pyvirtualdisplay import Display |
|
|
|
virtual_display = Display(visible=0, size=(1400, 900)) |
|
virtual_display.start() |
|
|
|
import gymnasium |
|
|
|
from huggingface_sb3 import load_from_hub, package_to_hub |
|
from huggingface_hub import notebook_login # To log to our Hugging Face account to be able to upload models to the Hub. |
|
|
|
from stable_baselines3 import PPO |
|
from stable_baselines3.common.evaluation import evaluate_policy |
|
from stable_baselines3.common.env_util import make_vec_env |
|
|
|
import gymnasium as gym |
|
|
|
# First, we create our environment called LunarLander-v2 |
|
env = gym.make("LunarLander-v2") |
|
|
|
# Then we reset this environment |
|
observation, info = env.reset() |
|
|
|
for _ in range(20): |
|
# Take a random action |
|
action = env.action_space.sample() |
|
print("Action taken:", action) |
|
|
|
# Do this action in the environment and get |
|
# next_state, reward, terminated, truncated and info |
|
observation, reward, terminated, truncated, info = env.step(action) |
|
|
|
# If the game is terminated (in our case we land, crashed) or truncated (timeout) |
|
if terminated or truncated: |
|
# Reset the environment |
|
print("Environment is reset") |
|
observation, info = env.reset() |
|
|
|
env.close() |
|
|
|
# We create our environment with gym.make("<name_of_the_environment>") |
|
env = gym.make("LunarLander-v2") |
|
env.reset() |
|
print("_____OBSERVATION SPACE_____ \n") |
|
print("Observation Space Shape", env.observation_space.shape) |
|
print("Sample observation", env.observation_space.sample()) # Get a random observation |
|
|
|
print("\n _____ACTION SPACE_____ \n") |
|
print("Action Space Shape", env.action_space.n) |
|
print("Action Space Sample", env.action_space.sample()) # Take a random action |
|
|
|
#Action 0: Do nothing, |
|
#Action 1: Fire left orientation engine, |
|
#Action 2: Fire the main engine, |
|
#Action 3: Fire right orientation engine. |
|
|
|
# Create the environment |
|
env = make_vec_env('LunarLander-v2', n_envs=16) |
|
|
|
# Create environment |
|
env = gym.make('LunarLander-v2') |
|
|
|
# Instantiate the agent - example |
|
#model = PPO('MlpPolicy', env, verbose=1) |
|
# Train the agent |
|
#model.learn(total_timesteps=int(2e5)) |
|
|
|
#faster learning |
|
model = PPO( |
|
policy = 'MlpPolicy', |
|
env = env, |
|
n_steps = 1024, |
|
batch_size = 64, |
|
n_epochs = 4, |
|
gamma = 0.999, |
|
gae_lambda = 0.98, |
|
ent_coef = 0.01, |
|
verbose=1) |
|
|
|
# TODO: Train it for 1,000,000 timesteps |
|
model.learn(total_timesteps = 1000000) |
|
# TODO: Specify file name for model and save the model to file |
|
model_name = "ppo-LunarLander-v2-niftymark" |
|
model.save(model_name) |
|
|
|
# TODO: Evaluate the agent |
|
# Create a new environment for evaluation |
|
eval_env = gym.make("LunarLander-v2") |
|
|
|
# Evaluate the model with 10 evaluation episodes and deterministic=True |
|
mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=10, deterministic=True) |
|
|
|
# Print the results |
|
print(f"mean_reward = {mean_reward:.2f} +/- {std_reward}") |
|
|
|
notebook_login() |
|
!git config --global credential.helper store |
|
|
|
#If you don't want to use a Google Colab or a Jupyter Notebook, you need to use this command instead: huggingface-cli login |
|
|
|
import gymnasium as gym |
|
from stable_baselines3.common.vec_env import DummyVecEnv |
|
from stable_baselines3.common.env_util import make_vec_env |
|
|
|
from huggingface_sb3 import package_to_hub |
|
|
|
## TODO: Define a repo_id |
|
## repo_id is the id of the model repository from the Hugging Face Hub (repo_id = {organization}/{repo_name} for instance ThomasSimonini/ppo-LunarLander-v2 |
|
repo_id = "niftymark/ppo-LunarLander-v2" |
|
|
|
# TODO: Define the name of the environment |
|
env_id = "LunarLander-v2" |
|
|
|
# Create the evaluation env and set the render_mode="rgb_array" |
|
eval_env = DummyVecEnv([lambda: gym.make(env_id, render_mode="rgb_array")]) |
|
|
|
|
|
# TODO: Define the model architecture we used |
|
model_architecture = "PPO" |
|
|
|
## TODO: Define the commit message |
|
commit_message = "first commit with working Lunar Lander - mean reward 259.93" |
|
|
|
# method save, evaluate, generate a model card and record a replay video of your agent before pushing the repo to the hub |
|
package_to_hub(model=model, # Our trained model |
|
model_name=model_name, # The name of our trained model |
|
model_architecture=model_architecture, # The model architecture we used: in our case PPO |
|
env_id=env_id, # Name of the environment |
|
eval_env=eval_env, # Evaluation Environment |
|
repo_id=repo_id, # id of the model repository from the Hugging Face Hub (repo_id = {organization}/{repo_name} for instance ThomasSimonini/ppo-LunarLander-v2 |
|
commit_message=commit_message) |
|
|
|
... |
|
``` |
|
|