Initial commit

README.md

@@ -6,7 +6,7 @@ tags:
 - reinforcement-learning
 - stable-baselines3
 model-index:
-- name: PPO
+- name: DQN
   results:
   - task:
       type: reinforcement-learning
@@ -16,13 +16,13 @@ model-index:
       type: SpaceInvadersNoFrameskip-v4
     metrics:
     - type: mean_reward
-      value: 144.50 +/- 99.26
+      value: 955.50 +/- 413.08
       name: mean_reward
       verified: false
 ---
 
-# **PPO** Agent playing **SpaceInvadersNoFrameskip-v4**
-This is a trained model of a **PPO** agent playing **SpaceInvadersNoFrameskip-v4**
+# **DQN** Agent playing **SpaceInvadersNoFrameskip-v4**
+This is a trained model of a **DQN** agent playing **SpaceInvadersNoFrameskip-v4**
 using the [stable-baselines3 library](https://github.com/DLR-RM/stable-baselines3)
 and the [RL Zoo](https://github.com/DLR-RM/rl-baselines3-zoo).
 
@@ -43,38 +43,42 @@ pip install rl_zoo3
 
 ```
 # Download model and save it into the logs/ folder
-python -m rl_zoo3.load_from_hub --algo ppo --env SpaceInvadersNoFrameskip-v4 -orga RAWsi-18 -f logs/
-python -m rl_zoo3.enjoy --algo ppo --env SpaceInvadersNoFrameskip-v4  -f logs/
+python -m rl_zoo3.load_from_hub --algo dqn --env SpaceInvadersNoFrameskip-v4 -orga RAWsi-18 -f logs/
+python -m rl_zoo3.enjoy --algo dqn --env SpaceInvadersNoFrameskip-v4  -f logs/
 ```
 
 If you installed the RL Zoo3 via pip (`pip install rl_zoo3`), from anywhere you can do:
 ```
-python -m rl_zoo3.load_from_hub --algo ppo --env SpaceInvadersNoFrameskip-v4 -orga RAWsi-18 -f logs/
-python -m rl_zoo3.enjoy --algo ppo --env SpaceInvadersNoFrameskip-v4  -f logs/
+python -m rl_zoo3.load_from_hub --algo dqn --env SpaceInvadersNoFrameskip-v4 -orga RAWsi-18 -f logs/
+python -m rl_zoo3.enjoy --algo dqn --env SpaceInvadersNoFrameskip-v4  -f logs/
 ```
 
 ## Training (with the RL Zoo)
 ```
-python -m rl_zoo3.train --algo ppo --env SpaceInvadersNoFrameskip-v4 -f logs/
+python -m rl_zoo3.train --algo dqn --env SpaceInvadersNoFrameskip-v4 -f logs/
 # Upload the model and generate video (when possible)
-python -m rl_zoo3.push_to_hub --algo ppo --env SpaceInvadersNoFrameskip-v4 -f logs/ -orga RAWsi-18
+python -m rl_zoo3.push_to_hub --algo dqn --env SpaceInvadersNoFrameskip-v4 -f logs/ -orga RAWsi-18
 ```
 
 ## Hyperparameters
 ```python
-OrderedDict([('batch_size', 256),
-             ('clip_range', 0.001),
-             ('ent_coef', 0.01),
+OrderedDict([('batch_size', 32),
+             ('buffer_size', 400000),
              ('env_wrapper',
               ['stable_baselines3.common.atari_wrappers.AtariWrapper']),
+             ('exploration_final_eps', 0.01),
+             ('exploration_fraction', 0.1),
              ('frame_stack', 4),
+             ('gamma', 0.99),
+             ('gradient_steps', 1),
              ('learning_rate', 0.0001),
-             ('n_envs', 8),
-             ('n_epochs', 4),
-             ('n_steps', 128),
-             ('n_timesteps', 100000),
+             ('learning_starts', 200000),
+             ('n_timesteps', 10000000.0),
+             ('optimize_memory_usage', True),
              ('policy', 'CnnPolicy'),
-             ('vf_coef', 0.5),
+             ('replay_buffer_kwargs', {'handle_timeout_termination': False}),
+             ('target_update_interval', 30000),
+             ('train_freq', 4),
              ('normalize', False)])
 ```
 

args.yml

@@ -1,10 +1,10 @@
 !!python/object/apply:collections.OrderedDict
 - - - algo
-    - ppo
+    - dqn
   - - conf_file
-    - ppo.yml
+    - null
   - - device
-    - auto
+    - mps
   - - env
     - SpaceInvadersNoFrameskip-v4
   - - env_kwargs
@@ -18,7 +18,7 @@
   - - hyperparams
     - null
   - - log_folder
-    - logs/
+    - ./logs/
   - - log_interval
     - -1
   - - max_total_trials
@@ -54,7 +54,7 @@
   - - save_replay_buffer
     - false
   - - seed
-    - 2285675408
+    - 2851482261
   - - storage
     - null
   - - study_name
@@ -77,5 +77,5 @@
     - null
   - - wandb_project_name
     - sb3
-  - - wandb_tags
-    - []
+  - - yaml_file
+    - null

config.yml

@@ -1,25 +1,33 @@
 !!python/object/apply:collections.OrderedDict
 - - - batch_size
-    - 256
-  - - clip_range
-    - 0.001
-  - - ent_coef
-    - 0.01
+    - 32
+  - - buffer_size
+    - 400000
   - - env_wrapper
     - - stable_baselines3.common.atari_wrappers.AtariWrapper
+  - - exploration_final_eps
+    - 0.01
+  - - exploration_fraction
+    - 0.1
   - - frame_stack
     - 4
+  - - gamma
+    - 0.99
+  - - gradient_steps
+    - 1
   - - learning_rate
     - 0.0001
-  - - n_envs
-    - 8
-  - - n_epochs
-    - 4
-  - - n_steps
-    - 128
+  - - learning_starts
+    - 200000
   - - n_timesteps
-    - 100000
+    - 10000000.0
+  - - optimize_memory_usage
+    - true
   - - policy
     - CnnPolicy
-  - - vf_coef
-    - 0.5
+  - - replay_buffer_kwargs
+    - handle_timeout_termination: false
+  - - target_update_interval
+    - 30000
+  - - train_freq
+    - 4

dqn-SpaceInvadersNoFrameskip-v4.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:450f283c244ff7b9427931d86c613214f3ecd394ffa306c6e1485c4781f66d24
+size 27220503

dqn-SpaceInvadersNoFrameskip-v4/_stable_baselines3_version

@@ -0,0 +1 @@
+2.4.0a4

dqn-SpaceInvadersNoFrameskip-v4/policy.optimizer.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f32af3e79370e1fc8757d69ebe4a7bf7fdc344c8bf905009fb2d6225c8a842b
+size 13505852

dqn-SpaceInvadersNoFrameskip-v4/policy.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b539097fe1cce9cfac67848f897cad9244bb80684d65010f7c0ece893094560
+size 13505178

dqn-SpaceInvadersNoFrameskip-v4/pytorch_variables.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c35cea3b2e60fb5e7e162d3592df775cd400e575a31c72f359fb9e654ab00c5
+size 864

dqn-SpaceInvadersNoFrameskip-v4/system_info.txt

@@ -0,0 +1,9 @@
+- OS: Linux-6.1.85+-x86_64-with-glibc2.35 # 1 SMP PREEMPT_DYNAMIC Thu Jun 27 21:05:47 UTC 2024
+- Python: 3.10.12
+- Stable-Baselines3: 2.4.0a4
+- PyTorch: 2.3.0+cu121
+- GPU Enabled: False
+- Numpy: 1.25.2
+- Cloudpickle: 2.2.1
+- Gymnasium: 0.29.1
+- OpenAI Gym: 0.25.2

replay.mp4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:395e1fd2d8a8af0742bba9ec795d4c68a907e6f425dc14e12cfdf4006984a159
-size 254024
+oid sha256:bf23b688757adad01a08b1c70cef53088f9be2758949443d55231e5fa373700c
+size 207513

results.json

@@ -1 +1 @@
-{"mean_reward": 144.5, "std_reward": 99.2585008953893, "is_deterministic": false, "n_eval_episodes": 10, "eval_datetime": "2024-07-11T14:08:17.727661"}
+{"mean_reward": 955.5, "std_reward": 413.08261885487263, "is_deterministic": false, "n_eval_episodes": 10, "eval_datetime": "2024-07-11T14:25:17.598243"}

train_eval_metrics.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f35e6afd136de9b1edd90d97013042d5104ef9f5b0ec1dfc7dd40d4aff0a72f
-size 6961
+oid sha256:4533414821c4753535a135ee5bbf143f73a553d68852a70f5f822a7a1d78a8b4
+size 270682