Initial commit

Browse files

Files changed (12) hide show

README.md +10 -4
args.yml +16 -10
config.yml +8 -1
ppo-seals-CartPole-v0.zip +2 -2
ppo-seals-CartPole-v0/_stable_baselines3_version +1 -1
ppo-seals-CartPole-v0/data +23 -23
ppo-seals-CartPole-v0/policy.optimizer.pth +2 -2
ppo-seals-CartPole-v0/policy.pth +1 -1
ppo-seals-CartPole-v0/system_info.txt +2 -2
replay.mp4 +2 -2
results.json +1 -1
train_eval_metrics.zip +2 -2

README.md CHANGED Viewed

@@ -37,15 +37,21 @@ SB3 Contrib: https://github.com/Stable-Baselines-Team/stable-baselines3-contrib
 ```
 # Download model and save it into the logs/ folder
-python -m utils.load_from_hub --algo ppo --env seals/CartPole-v0 -orga ernestumorga -f logs/
 python enjoy.py --algo ppo --env seals/CartPole-v0  -f logs/
 ```
 ## Training (with the RL Zoo)
 ```
 python train.py --algo ppo --env seals/CartPole-v0 -f logs/
 # Upload the model and generate video (when possible)
-python -m utils.push_to_hub --algo ppo --env seals/CartPole-v0 -f logs/ -orga ernestumorga
 ```
 ## Hyperparameters
@@ -63,8 +69,8 @@ OrderedDict([('batch_size', 256),
              ('n_timesteps', 100000.0),
              ('policy', 'MlpPolicy'),
              ('policy_kwargs',
-              'dict(activation_fn=nn.ReLU, net_arch=[dict(pi=[64, 64], vf=[64, '
-              '64])])'),
              ('vf_coef', 0.489343896591493),
              ('normalize', False)])
 ```

 ```
 # Download model and save it into the logs/ folder
+python -m rl_zoo3.load_from_hub --algo ppo --env seals/CartPole-v0 -orga ernestumorga -f logs/
 python enjoy.py --algo ppo --env seals/CartPole-v0  -f logs/
 ```
+If you installed the RL Zoo3 via pip (`pip install rl_zoo3`), from anywhere you can do:
+```
+python -m rl_zoo3.load_from_hub --algo ppo --env seals/CartPole-v0 -orga ernestumorga -f logs/
+rl_zoo3 enjoy --algo ppo --env seals/CartPole-v0  -f logs/
+```
 ## Training (with the RL Zoo)
 ```
 python train.py --algo ppo --env seals/CartPole-v0 -f logs/
 # Upload the model and generate video (when possible)
+python -m rl_zoo3.push_to_hub --algo ppo --env seals/CartPole-v0 -f logs/ -orga ernestumorga
 ```
 ## Hyperparameters
              ('n_timesteps', 100000.0),
              ('policy', 'MlpPolicy'),
              ('policy_kwargs',
+              {'activation_fn': <class 'torch.nn.modules.activation.ReLU'>,
+               'net_arch': [{'pi': [64, 64], 'vf': [64, 64]}]}),
              ('vf_coef', 0.489343896591493),
              ('normalize', False)])
 ```

args.yml CHANGED Viewed

@@ -1,6 +1,8 @@
 !!python/object/apply:collections.OrderedDict
 - - - algo
     - ppo
   - - device
     - cpu
   - - env
@@ -12,13 +14,15 @@
   - - eval_freq
     - 25000
   - - gym_packages
-    - []
   - - hyperparams
     - null
   - - log_folder
-    - seals_experts
   - - log_interval
     - -1
   - - n_eval_envs
     - 1
   - - n_evaluations
@@ -34,11 +38,13 @@
   - - no_optim_plots
     - false
   - - num_threads
-    - 4
   - - optimization_log_path
     - null
   - - optimize_hyperparameters
     - false
   - - pruner
     - median
   - - sampler
@@ -48,17 +54,15 @@
   - - save_replay_buffer
     - false
   - - seed
-    - 3866955406
   - - storage
     - null
   - - study_name
     - null
   - - tensorboard_log
-    - ''
-  - - total_n_trials
-    - null
   - - track
-    - false
   - - trained_agent
     - ''
   - - truncate_last_trajectory
@@ -70,6 +74,8 @@
   - - verbose
     - 1
   - - wandb_entity
-    - null
   - - wandb_project_name
-    - sb3

 !!python/object/apply:collections.OrderedDict
 - - - algo
     - ppo
+  - - conf_file
+    - hyperparams/python/ppo.py
   - - device
     - cpu
   - - env
   - - eval_freq
     - 25000
   - - gym_packages
+    - - seals
   - - hyperparams
     - null
   - - log_folder
+    - logs
   - - log_interval
     - -1
+  - - max_total_trials
+    - null
   - - n_eval_envs
     - 1
   - - n_evaluations
   - - no_optim_plots
     - false
   - - num_threads
+    - 1
   - - optimization_log_path
     - null
   - - optimize_hyperparameters
     - false
+  - - progress
+    - false
   - - pruner
     - median
   - - sampler
   - - save_replay_buffer
     - false
   - - seed
+    - 7
   - - storage
     - null
   - - study_name
     - null
   - - tensorboard_log
+    - runs/seals/CartPole-v0__ppo__7__1670516892
   - - track
+    - true
   - - trained_agent
     - ''
   - - truncate_last_trajectory
   - - verbose
     - 1
   - - wandb_entity
+    - ernestum
   - - wandb_project_name
+    - seals-experts-normalized
+  - - yaml_file
+    - null

config.yml CHANGED Viewed

@@ -24,6 +24,13 @@
   - - policy
     - MlpPolicy
   - - policy_kwargs
-    - dict(activation_fn=nn.ReLU, net_arch=[dict(pi=[64, 64], vf=[64, 64])])
   - - vf_coef
     - 0.489343896591493

   - - policy
     - MlpPolicy
   - - policy_kwargs
+    - activation_fn: !!python/name:torch.nn.modules.activation.ReLU ''
+      net_arch:
+      - pi:
+        - 64
+        - 64
+        vf:
+        - 64
+        - 64
   - - vf_coef
     - 0.489343896591493

ppo-seals-CartPole-v0.zip CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0430bbef17d9463be60c35cc6cc619a3a6dbc7db81e1d9469b484a1d2c5b156
-size 138640

 version https://git-lfs.github.com/spec/v1
+oid sha256:003d7ffa0640edfa2b72507f5934a87a94e51b1b62ad13c3414db0e92a6519e6
+size 141656

ppo-seals-CartPole-v0/_stable_baselines3_version CHANGED Viewed

	@@ -1 +1 @@
1	- 1.4.~~1a0~~


1	+ 1.6.2

ppo-seals-CartPole-v0/data CHANGED Viewed

@@ -4,19 +4,19 @@
         ":serialized:": "gAWVOwAAAAAAAACMIXN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi5wb2xpY2llc5SMEUFjdG9yQ3JpdGljUG9saWN5lJOULg==",
         "__module__": "stable_baselines3.common.policies",
         "__doc__": "\n    Policy class for actor-critic algorithms (has both policy and value prediction).\n    Used by A2C, PPO and the likes.\n\n    :param observation_space: Observation space\n    :param action_space: Action space\n    :param lr_schedule: Learning rate schedule (could be constant)\n    :param net_arch: The specification of the policy and value networks.\n    :param activation_fn: Activation function\n    :param ortho_init: Whether to use or not orthogonal initialization\n    :param use_sde: Whether to use State Dependent Exploration or not\n    :param log_std_init: Initial value for the log standard deviation\n    :param full_std: Whether to use (n_features x n_actions) parameters\n        for the std instead of only (n_features,) when using gSDE\n    :param sde_net_arch: Network architecture for extracting features\n        when using gSDE. If None, the latent features from the policy will be used.\n        Pass an empty list to use the states as features.\n    :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure\n        a positive standard deviation (cf paper). It allows to keep variance\n        above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.\n    :param squash_output: Whether to squash the output using a tanh function,\n        this allows to ensure boundaries when using gSDE.\n    :param features_extractor_class: Features extractor to use.\n    :param features_extractor_kwargs: Keyword arguments\n        to pass to the features extractor.\n    :param normalize_images: Whether to normalize images or not,\n         dividing by 255.0 (True by default)\n    :param optimizer_class: The optimizer to use,\n        ``th.optim.Adam`` by default\n    :param optimizer_kwargs: Additional keyword arguments,\n        excluding the learning rate, to pass to the optimizer\n    ",
-        "__init__": "<function ActorCriticPolicy.__init__ at 0x7fc420aef280>",
-        "_get_constructor_parameters": "<function ActorCriticPolicy._get_constructor_parameters at 0x7fc420aef310>",
-        "reset_noise": "<function ActorCriticPolicy.reset_noise at 0x7fc420aef3a0>",
-        "_build_mlp_extractor": "<function ActorCriticPolicy._build_mlp_extractor at 0x7fc420aef430>",
-        "_build": "<function ActorCriticPolicy._build at 0x7fc420aef4c0>",
-        "forward": "<function ActorCriticPolicy.forward at 0x7fc420aef550>",
-        "_get_action_dist_from_latent": "<function ActorCriticPolicy._get_action_dist_from_latent at 0x7fc420aef5e0>",
-        "_predict": "<function ActorCriticPolicy._predict at 0x7fc420aef670>",
-        "evaluate_actions": "<function ActorCriticPolicy.evaluate_actions at 0x7fc420aef700>",
-        "get_distribution": "<function ActorCriticPolicy.get_distribution at 0x7fc420aef790>",
-        "predict_values": "<function ActorCriticPolicy.predict_values at 0x7fc420aef820>",
         "__abstractmethods__": "frozenset()",
-        "_abc_impl": "<_abc_data object at 0x7fc420b67870>"
     },
     "verbose": 1,
     "policy_kwargs": {
@@ -51,27 +51,27 @@
     },
     "action_space": {
         ":type:": "<class 'gym.spaces.discrete.Discrete'>",
-        ":serialized:": "gAWVLwsAAAAAAACME2d5bS5zcGFjZXMuZGlzY3JldGWUjAhEaXNjcmV0ZZSTlCmBlH2UKIwBbpRLAowGX3NoYXBllCmMBWR0eXBllIwFbnVtcHmUaAeTlIwCaTiUiYiHlFKUKEsDjAE8lE5OTkr/////Sv////9LAHSUYowKX25wX3JhbmRvbZSMFG51bXB5LnJhbmRvbS5fcGlja2xllIwSX19yYW5kb21zdGF0ZV9jdG9ylJOUjAdNVDE5OTM3lIWUUpR9lCiMDWJpdF9nZW5lcmF0b3KUaBOMBXN0YXRllH2UKIwDa2V5lIwSbnVtcHkuY29yZS5udW1lcmljlIwLX2Zyb21idWZmZXKUk5QolsAJAAAAAAAAAAAAgFPCs5yHA7WTcuyrW+jmsvLPtoHa1QbvYaExTaBrtczZE+YYn3SceS/IpRuAxHkBSZ4PQ+Rd4BiTkhNVzNRKKWTSCJW+NNCeRHJ6A/Ctvwpzm3s+6WCxBtp1A0ULbn3WFmrrBDRmg7fz9aUvae9CY0O1XPfCd1LMRkQ3LQiJbtCOrnf7GLaAT3ue+U8y7BLYuY5ehakZyq49di1nK0KAnsXuxx/1IgFdS88kD8wZUmREV5hwnQr1Ehe16VreO8T/Qc86sV+2h90z2FiJqqLNf3s/yZS3bA9DHzGZfRtgkKu3Bx0ZTN7I+466APXPqRreJf4gBqUW/NB248FO9cpD0wFaed9QV63NqpiFOs/RYeEwD8e32dZFRfi9SK5aLvuApJMu2LOfZweypHIkyPjeY5W+tsm2bdPmVoCAOiyi73cf5k0LQeJNWqZU/wuy/f8myghZ9qrjf+2JSJMaB9VNMXWmxuq4Dq0fkhzJr1ML7SgftfSG29O5koFUMozQL58gAzyX96ZMcpWbZ+3/zlaVhGln5egXC8MtIK6xIFCvh/vD/F1jLgYIp14MM597MuPmTpa+OaKek7bql9Cp8/0skhg5QSCvCaijm8wenxrfqLyRxDPCpS+L3isJC5LrjFgWnsdxQXVrJK8uaobJcTIJ5NrYYfA2l5gH27iPI9EqhzFtZJXiE4vXpH18f3kouYV9RowPzOtmYsbmstR/Mx/VY7E0XBmnMidL4dYTUXgxaDbFxWy3y6miL2yw0I2O09vPWV7LqbwMbthlU26lrLfnJDz88B+7y4pFCwvsHgCsMWq6pvroAF4Ms+++JnhzrL0GLrtfJ7667p42Vg78GirmKIRMFH0p6aLRPV4V/fclE3PLBj0InU315M5v7fDFj/IreJeFUhIAy5/BlvLdgwIfBMBWvyjhqGRBo2KmdiDAO9BPhdi6oGmZUCxTDjUyMd8rjeRdntTE+L9sHQUSvUfpRjKcSIjZXcjsMMSwwU0QzIXewD4nZg8EAZ72iHnChWveW7cB5EQRFE+YPvvyNAw4OvORF+DjDtmLUz6jNz0JbkuKbhIQEEVbpDRkoeIGA4HATymZeE4TX/hvhOxJfX5liXD9/Uon8OjJ/hhIrDNUYR2NOV2R1L+VuK/JA9o0izGPiCC9vi7UK1u4re8tvPrBInC6BH2DpMvWSRepdH0frFxGsH/kRB/S77USeMs38a+yorDaa9Wsb9WnuQg10vl087jhvwSZx77fGq52NV1h/UnZAjnqJOjCPBgQrY0wsSeSW6SOSdItAOupiWkVvSzHz+gopd/3FUUPbjdik7Vd6O6ycGOFwp4+wyZqq1MKHww5exJND6De8NB4fGBzsvKoT8O1fAC3Y2Z+3mLYwkCJXULx2zNSO90bQi0L0BhTF1AidNtqRkak72V+V1kos6m4F+kyZiOiwxfcHKWHmpQe3A8HpX6nacx0Zso0WyF/W/sOLo//2g0YD9koIjgKvw//f9Em4CvxNBUMnJYTVQJGysF4QMShnM6byaMNJhRVALf+X0+SgLJkEwLfKdBbKxjA035OEc2YKmUpvooVUUt+9U8d7cKRgNYKKPwCHyYfLJSQt4ZEdThgjeywDxgsGzPax5SklOLroSYn39feofatZDzJXevfPTHyi1ZLwpe6Hwkbqz1FuOnHiShPedbEA9b/HCtBytSZrgUwlwHpQlAiBTSxpN7TFzVZrrhRd6N8W+FeE9SAGCHwXchFR4SCGThPyRGO/XWkMPJ42BLUOmMGepDQgWH646tjoJSE3EXcA5iDS+Nq6Oh521oC2UPAnvxIj1QbVh8IbxlBytRTTjc0c14E9cyhIPlgIoHriuEFSMXSEzOGL1MmL6UCbiXfsRg9Z6OwWCCl3VeGg5bEZ1kjJkvs08k7wtPk4ATAjaTL3QoY2gf106zFbJtL4D5gmLMJ9OuzE2Fn5uaAqqpjXIqqEXxS9jtpsRU9VTHCg68RwXQVIUhuVJgHq8fOigBMrW7Am5+jjo/GNNlcFcp813dFiXy4qHhjGSEjNXp3ln03NZkOgqXQ8SalJlPOvyrAS9wW9EtjQKhcrBSWSsQ8C3o/Mc/sR/CMRB85ZIojR/tiCKtOutxQMIusIOnYHK8g6kPpTQ8J/PfJ8pa3GEoYoA67axQTXsysd2Y6ZDwpz1HkAeISVK2AlCcuQssrQv8dVLAcins/2kjRFp5Vp82HSX9j6Ci9GH5mkdyqV84vWsdwRz4JNXHZoHVZKnrxSdA1HUwRUI/5oWiqnGk9KyiS4Mv9dQIVluJ1+/pAHYEdG9YgLHiNE2zA7aIQbqqlGX6jH1CrHSPL9mnlHdPiKjozwRXu02UQuzlGJn+/PUkU6cPYLLeLc8e7S2qfCZxbdpHioand7wYKqb5bMb8dA3Dwvm6P6iJXSogJ+Q+0z43li8ydYJqZNZlSjsljr/2c5UU33vMhLDLEXIHT8WzrsS45TsKNbhYfTYx6Ds/8W5yOtiOPWc0+fRlAjbQC++FKo5UaMl2eRCxI4U5/heX2HxJGNk50rICjuFsG/8Q+NUuJgS+y3FMhe+sm1e2MdC+ldkBqEn8oRxajECbP4Wizz1tfJliW/1A5fdGTfUxM3HV72bJgnqswmoAumAx6d36KfuZwEEp0/wcrdo+8/unJ5f2mYeqCrOcaxDJrs6SxW1zVaH/YTZl+RNA0NjPLgaqnlveaes/MkpzsVEQDtvKGFrG1cnmGjZVi2azrSDGQ0Y423nEksDC5awYcOJmVYbeA4DkMSNfj+7Dx2SzH/PVPuXLX9aw9K9QF4Ml48zsSrwsVjIa8+gIdffs2pf2wCKcPtFez6vOT0UDuFHknJjMDg0fI3DnyC7jJqO8V4XpmPyarTp3JJRGhmqTHhpZInn70JMfS+RFry5+rLSOM0T+KWV8fYjs9eyCiZijlR4AiADooXm9G8JIzZCLZX2Dty83iyz7gQzSxYO7ULuTT1stvGuJwbBP4LMhLXkbxdhAmBSDiYNOnc3O+yFsO6Ps9UOQD8S4Pbr8hZ4mFjbicpO635SwpmHINYDeuewln3/GHz69LpCjmpnKPeF9ZxXcq6MR4kJUV2j/dQzqjLniNaQmrMkULdI7W1sMXRFcsz9xs1GVwVqmtMVws8HtvXMYNmosCrrgAFX2ghPz7dXCV6vML5YhfNbDAzzG6MHffrslrhMav3vtlt8Fnld4VaH6IhMkowayT1lSVvfvlKHCWwtKaTcOZrR5LZGalJOpFbVIFUOAo+LnY/25bmc3KloyLzgiTudjPsXEGPNPBIvE/5cMEvU4Lrs0N3tCke4abYDXF9f14QrwLlGgJjAJ1NJSJiIeUUpQoSwNoDU5OTkr/////Sv////9LAHSUYk1wAoWUjAFDlHSUUpSMA3Bvc5RNcAJ1jAloYXNfZ2F1c3OUSwCMBWdhdXNzlEcAAAAAAAAAAHVidWIu",
         "n": 2,
         "_shape": [],
         "dtype": "int64",
         "_np_random": "RandomState(MT19937)"
     },
-    "n_envs": 8,
     "num_timesteps": 102400,
     "_total_timesteps": 100000,
     "_num_timesteps_at_start": 0,
-    "seed": 0,
     "action_noise": null,
-    "start_time": 1651240811.8226469,
     "learning_rate": {
         ":type:": "<class 'function'>",
-        ":serialized:": "gAWVmAIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwNX2J1aWx0aW5fdHlwZZSTlIwKTGFtYmRhVHlwZZSFlFKUKGgCjAhDb2RlVHlwZZSFlFKUKEsBSwBLAEsBSwFLE0MEiABTAJROhZQpjAFflIWUjGgvbmFzL3VjYi9tYXhpbWlsaWFuL3JsLWJhc2VsaW5lczMtem9vL3ZlbnYvbGliL3B5dGhvbjMuOC9zaXRlLXBhY2thZ2VzL3N0YWJsZV9iYXNlbGluZXMzL2NvbW1vbi91dGlscy5weZSMBGZ1bmOUS4BDAgABlIwDdmFslIWUKXSUUpR9lCiMC19fcGFja2FnZV9flIwYc3RhYmxlX2Jhc2VsaW5lczMuY29tbW9ulIwIX19uYW1lX1+UjB5zdGFibGVfYmFzZWxpbmVzMy5jb21tb24udXRpbHOUjAhfX2ZpbGVfX5RoDXVOTmgAjBBfbWFrZV9lbXB0eV9jZWxslJOUKVKUhZR0lFKUjBxjbG91ZHBpY2tsZS5jbG91ZHBpY2tsZV9mYXN0lIwSX2Z1bmN0aW9uX3NldHN0YXRllJOUaB99lH2UKGgXaA6MDF9fcXVhbG5hbWVfX5SMGWNvbnN0YW50X2ZuLjxsb2NhbHM+LmZ1bmOUjA9fX2Fubm90YXRpb25zX1+UfZSMDl9fa3dkZWZhdWx0c19flE6MDF9fZGVmYXVsdHNfX5ROjApfX21vZHVsZV9flGgYjAdfX2RvY19flE6MC19fY2xvc3VyZV9flGgAjApfbWFrZV9jZWxslJOURz9UUk/azSgzhZRSlIWUjBdfY2xvdWRwaWNrbGVfc3VibW9kdWxlc5RdlIwLX19nbG9iYWxzX1+UfZR1hpSGUjAu"
     },
-    "tensorboard_log": null,
     "lr_schedule": {
         ":type:": "<class 'function'>",
-        ":serialized:": "gAWVmAIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwNX2J1aWx0aW5fdHlwZZSTlIwKTGFtYmRhVHlwZZSFlFKUKGgCjAhDb2RlVHlwZZSFlFKUKEsBSwBLAEsBSwFLE0MEiABTAJROhZQpjAFflIWUjGgvbmFzL3VjYi9tYXhpbWlsaWFuL3JsLWJhc2VsaW5lczMtem9vL3ZlbnYvbGliL3B5dGhvbjMuOC9zaXRlLXBhY2thZ2VzL3N0YWJsZV9iYXNlbGluZXMzL2NvbW1vbi91dGlscy5weZSMBGZ1bmOUS4BDAgABlIwDdmFslIWUKXSUUpR9lCiMC19fcGFja2FnZV9flIwYc3RhYmxlX2Jhc2VsaW5lczMuY29tbW9ulIwIX19uYW1lX1+UjB5zdGFibGVfYmFzZWxpbmVzMy5jb21tb24udXRpbHOUjAhfX2ZpbGVfX5RoDXVOTmgAjBBfbWFrZV9lbXB0eV9jZWxslJOUKVKUhZR0lFKUjBxjbG91ZHBpY2tsZS5jbG91ZHBpY2tsZV9mYXN0lIwSX2Z1bmN0aW9uX3NldHN0YXRllJOUaB99lH2UKGgXaA6MDF9fcXVhbG5hbWVfX5SMGWNvbnN0YW50X2ZuLjxsb2NhbHM+LmZ1bmOUjA9fX2Fubm90YXRpb25zX1+UfZSMDl9fa3dkZWZhdWx0c19flE6MDF9fZGVmYXVsdHNfX5ROjApfX21vZHVsZV9flGgYjAdfX2RvY19flE6MC19fY2xvc3VyZV9flGgAjApfbWFrZV9jZWxslJOURz9UUk/azSgzhZRSlIWUjBdfY2xvdWRwaWNrbGVfc3VibW9kdWxlc5RdlIwLX19nbG9iYWxzX1+UfZR1hpSGUjAu"
     },
     "_last_obs": null,
     "_last_episode_starts": {
@@ -85,7 +85,7 @@
     "_current_progress_remaining": -0.02400000000000002,
     "ep_info_buffer": {
         ":type:": "<class 'collections.deque'>",
-        ":serialized:": "gAWVRAwAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKUKH2UKIwBcpRHQH9AAAAAAACMAWyUTfQBjAF0lEdAJVd0JWvKU3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCVWtlqagEl1fZQoaAZHQH9AAAAAAABoB030AWgIR0AlVe0G/vfCdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAJVUj1PFefXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCbGpZOi35N1fZQoaAZHQH9AAAAAAABoB030AWgIR0AmxhF3IMjNdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAJsVzIV/MGHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCbE1IiC8OF1fZQoaAZHQH9AAAAAAABoB030AWgIR0Amw/etSydGdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAJsM6BAfMfXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCbCcLBsQ/Z1fZQoaAZHQH9AAAAAAABoB030AWgIR0AmwafBeokzdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAKBRMN+b3GnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCgTujRD1Gt1fZQoaAZHQH9AAAAAAABoB030AWgIR0AoExvegte2dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAKBJ9iMHbAXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCgRoM8YAKh1fZQoaAZHQH9AAAAAAABoB030AWgIR0AoEONHYpUhdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAKBAaNuLrHHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCgPUUfxMFl1fZQoaAZHQH9AAAAAAABoB030AWgIR0ApZaURnOB2dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAKWUONHYpUnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQClkbzbvgFZ1fZQoaAZHQH9AAAAAAABoB030AWgIR0ApY9AX2ugZdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAKWLy1/lQuXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCliNOuaF251fZQoaAZHQH9AAAAAAABoB030AWgIR0ApYWszVMEidX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAKWChvitJWnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCqzKNhmXgN1fZQoaAZHQH9AAAAAAABoB030AWgIR0AqspmVZ9uxdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAKrH8KohpxnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCqxXhfjS5R1fZQoaAZHQH9AAAAAAABoB030AWgIR0AqsIHkcS5BdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAKq/GdZq20HV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCqu/gzguRN1fZQoaAZHQH9AAAAAAABoB030AWgIR0AqrjZL7GeddX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdALAVea8YhuHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCwEzXSSeRR1fZQoaAZHQH9AAAAAAABoB030AWgIR0AsBC9h7VridX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdALAOQ6p5u63V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCwCtA9mpVF1fZQoaAZHQH9AAAAAAABoB030AWgIR0AsAfapPykLdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdALAEtmL9/BnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQCwAZMtbs4V1fZQoaAZHQH9AAAAAAABoB030AWgIR0AuwE6DGtITdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdALr+4TbnHN3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQC6/GbTc6/91fZQoaAZHQH9AAAAAAABoB030AWgIR0AuvntfG+9KdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdALr2eg+Qlr3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQC684NqgyuZ1fZQoaAZHQH9AAAAAAABoB030AWgIR0AuvBfKISDidX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdALrtOuaF23nV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDAW5Fw1ivx1fZQoaAZHQH9AAAAAAABoB030AWgIR0AwFphnanJldX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMBZI+W4Vh3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDAV+YtxuKp1fZQoaAZHQH9AAAAAAABoB030AWgIR0AwFYr8R+SbdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMBUsFt8/lnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDAUx0uDjBF1fZQoaAZHQH9AAAAAAABoB030AWgIR0AwFGKhtcfOdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMMCFsYVIqnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDDAOkLx7Rh1fZQoaAZHQH9AAAAAAABoB030AWgIR0Awv+r2g398dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAML+bd8Aq/nV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDC/LMcIZ651fZQoaAZHQH9AAAAAAABoB030AWgIR0Awvs3Q2MsIdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAML5pBX0Xg3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDC+BEroW591fZQoaAZHQH9AAAAAAABoB030AWgIR0AxcYNAkcCHdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMXE5ZKWcBnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDFw6vJRwZR1fZQoaAZHQH9AAAAAAABoB030AWgIR0AxcJvHcUM5dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMXAtapxWDHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDFvzpX6qKh1fZQoaAZHQH9AAAAAAABoB030AWgIR0Axb2n889wFdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMW8FlkH2RXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDIxpEhJRO11fZQoaAZHQH9AAAAAAABoB030AWgIR0AyMVkMCtA+dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMjEJv5xionV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDIwulGgBcR1fZQoaAZHQH9AAAAAAABoB030AWgIR0AyMEvTPSlWdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMi/s3Q2MsHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDIviBGx2St1fZQoaAZHQH9AAAAAAABoB030AWgIR0AyLyNGViWndX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMuDsdDIBBHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDLgoH9m6Gx1fZQoaAZHQH9AAAAAAABoB030AWgIR0Ay4FEiMYMwdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMuABo24usnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDLfkxREWqN1fZQoaAZHQH9AAAAAAABoB030AWgIR0Ay3zQeFL39dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMt7PY4ACGXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDLeaqjrRjV1fZQoaAZHQH9AAAAAAABoB030AWgIR0Azkd3B55Z9dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAM5GSIP9UCXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDORQrMC9yt1fZQoaAZHQH9AAAAAAABoB030AWgIR0AzkPN3W4EwdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAM5CE12q1gHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDOQJeE7GNt1fZQoaAZHQH9AAAAAAABoB030AWgIR0Azj8EV32VWdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAM49cW0qpcXVlLg=="
     },
     "ep_success_buffer": {
         ":type:": "<class 'collections.deque'>",
@@ -102,9 +102,9 @@
     "n_epochs": 10,
     "clip_range": {
         ":type:": "<class 'function'>",
-        ":serialized:": "gAWVmAIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwNX2J1aWx0aW5fdHlwZZSTlIwKTGFtYmRhVHlwZZSFlFKUKGgCjAhDb2RlVHlwZZSFlFKUKEsBSwBLAEsBSwFLE0MEiABTAJROhZQpjAFflIWUjGgvbmFzL3VjYi9tYXhpbWlsaWFuL3JsLWJhc2VsaW5lczMtem9vL3ZlbnYvbGliL3B5dGhvbjMuOC9zaXRlLXBhY2thZ2VzL3N0YWJsZV9iYXNlbGluZXMzL2NvbW1vbi91dGlscy5weZSMBGZ1bmOUS4BDAgABlIwDdmFslIWUKXSUUpR9lCiMC19fcGFja2FnZV9flIwYc3RhYmxlX2Jhc2VsaW5lczMuY29tbW9ulIwIX19uYW1lX1+UjB5zdGFibGVfYmFzZWxpbmVzMy5jb21tb24udXRpbHOUjAhfX2ZpbGVfX5RoDXVOTmgAjBBfbWFrZV9lbXB0eV9jZWxslJOUKVKUhZR0lFKUjBxjbG91ZHBpY2tsZS5jbG91ZHBpY2tsZV9mYXN0lIwSX2Z1bmN0aW9uX3NldHN0YXRllJOUaB99lH2UKGgXaA6MDF9fcXVhbG5hbWVfX5SMGWNvbnN0YW50X2ZuLjxsb2NhbHM+LmZ1bmOUjA9fX2Fubm90YXRpb25zX1+UfZSMDl9fa3dkZWZhdWx0c19flE6MDF9fZGVmYXVsdHNfX5ROjApfX21vZHVsZV9flGgYjAdfX2RvY19flE6MC19fY2xvc3VyZV9flGgAjApfbWFrZV9jZWxslJOURz/ZmZmZmZmahZRSlIWUjBdfY2xvdWRwaWNrbGVfc3VibW9kdWxlc5RdlIwLX19nbG9iYWxzX1+UfZR1hpSGUjAu"
     },
     "clip_range_vf": null,
-    "target_kl": null,
-    "normalize_advantage": true
 }

         ":serialized:": "gAWVOwAAAAAAAACMIXN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi5wb2xpY2llc5SMEUFjdG9yQ3JpdGljUG9saWN5lJOULg==",
         "__module__": "stable_baselines3.common.policies",
         "__doc__": "\n    Policy class for actor-critic algorithms (has both policy and value prediction).\n    Used by A2C, PPO and the likes.\n\n    :param observation_space: Observation space\n    :param action_space: Action space\n    :param lr_schedule: Learning rate schedule (could be constant)\n    :param net_arch: The specification of the policy and value networks.\n    :param activation_fn: Activation function\n    :param ortho_init: Whether to use or not orthogonal initialization\n    :param use_sde: Whether to use State Dependent Exploration or not\n    :param log_std_init: Initial value for the log standard deviation\n    :param full_std: Whether to use (n_features x n_actions) parameters\n        for the std instead of only (n_features,) when using gSDE\n    :param sde_net_arch: Network architecture for extracting features\n        when using gSDE. If None, the latent features from the policy will be used.\n        Pass an empty list to use the states as features.\n    :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure\n        a positive standard deviation (cf paper). It allows to keep variance\n        above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.\n    :param squash_output: Whether to squash the output using a tanh function,\n        this allows to ensure boundaries when using gSDE.\n    :param features_extractor_class: Features extractor to use.\n    :param features_extractor_kwargs: Keyword arguments\n        to pass to the features extractor.\n    :param normalize_images: Whether to normalize images or not,\n         dividing by 255.0 (True by default)\n    :param optimizer_class: The optimizer to use,\n        ``th.optim.Adam`` by default\n    :param optimizer_kwargs: Additional keyword arguments,\n        excluding the learning rate, to pass to the optimizer\n    ",
+        "__init__": "<function ActorCriticPolicy.__init__ at 0x7f9370da4700>",
+        "_get_constructor_parameters": "<function ActorCriticPolicy._get_constructor_parameters at 0x7f9370da4790>",
+        "reset_noise": "<function ActorCriticPolicy.reset_noise at 0x7f9370da4820>",
+        "_build_mlp_extractor": "<function ActorCriticPolicy._build_mlp_extractor at 0x7f9370da48b0>",
+        "_build": "<function ActorCriticPolicy._build at 0x7f9370da4940>",
+        "forward": "<function ActorCriticPolicy.forward at 0x7f9370da49d0>",
+        "_get_action_dist_from_latent": "<function ActorCriticPolicy._get_action_dist_from_latent at 0x7f9370da4a60>",
+        "_predict": "<function ActorCriticPolicy._predict at 0x7f9370da4af0>",
+        "evaluate_actions": "<function ActorCriticPolicy.evaluate_actions at 0x7f9370da4b80>",
+        "get_distribution": "<function ActorCriticPolicy.get_distribution at 0x7f9370da4c10>",
+        "predict_values": "<function ActorCriticPolicy.predict_values at 0x7f9370da4ca0>",
         "__abstractmethods__": "frozenset()",
+        "_abc_impl": "<_abc_data object at 0x7f9370d9cc30>"
     },
     "verbose": 1,
     "policy_kwargs": {
     },
     "action_space": {
         ":type:": "<class 'gym.spaces.discrete.Discrete'>",
+        ":serialized:": "gAWVLwsAAAAAAACME2d5bS5zcGFjZXMuZGlzY3JldGWUjAhEaXNjcmV0ZZSTlCmBlH2UKIwBbpRLAowGX3NoYXBllCmMBWR0eXBllIwFbnVtcHmUaAeTlIwCaTiUiYiHlFKUKEsDjAE8lE5OTkr/////Sv////9LAHSUYowKX25wX3JhbmRvbZSMFG51bXB5LnJhbmRvbS5fcGlja2xllIwSX19yYW5kb21zdGF0ZV9jdG9ylJOUjAdNVDE5OTM3lIWUUpR9lCiMDWJpdF9nZW5lcmF0b3KUaBOMBXN0YXRllH2UKIwDa2V5lIwSbnVtcHkuY29yZS5udW1lcmljlIwLX2Zyb21idWZmZXKUk5QolsAJAAAAAAAAAAAAgMvWU2I7QWSDdygaT5++KSAATCN90WccEqDtnFV3Y9MyLlpJIkIvbFJbpmpRHJX7SZr4rNIblBqp+ccsvAR6E3PBn61gcJfsxhg3x3PqnSmOqgLqwqIfkWzrz6Tf7NSrm1SSpbmnGNt3g/H87CT3UgGUR8+HrL79sEyK19tohIeSCjwJLlaEiqt73sZ9FJuvWOgJ3QNRV0zMGqSYGeuLv+TZ0EN3+cyUE+2SDHCnC8u+eTF8fJciknY5HW6tbE8ba0T1dJIbNDabvq3Lgzsjaj2J1a7ewVritpsjEYCurZVZpRofQC3nO0jM0C3+YuXA2lZGNYHChc8MyrCvJFQJFlfQ6BPvCeiw5+FaDMxbSonLCcRFZPKJp/lWXhgYkj0swzyhXxNK4EuHqIvN1K+tQ0O3iv/AsSiGk4LYvZONnoPxuDvSFcQq8UdUpcJew8fp98fUiPvPgHKAyO+L/60Vzkqu4hXw9hUKR7cWJQZLV9ChyDHgbsufUNTzdmm58PdxfyD8sb0Fhi30bqDkMwc6vuIXQYXYIgfYVnRcsI8ZmwcYjxg8cQeVZf1hZkd00dKZryQNLb2bz1WRqlWiK1AOV5KEPeonMRrdcwQnEBRqx9ThMDIDdCHWE2moNgX9Lu29fAxYIe//6Duh7KKhp5v80UQ7VD/0ddR8UbUVl4xVW4UneWt9U8AcBpaAgolQEZzIT0q3RzCYbWDwzLzFaawdkttUhKkFiwjDJcESKnKqKDofg5+k4YTJOSO1kagch1NV1JXrfrSfcaUDMG8F27PRTcBF/8idtyoK/uZB9ThefVNaXRCRY/K/zZz7ehTk1GSqe70bQUNaAEdFBYnd+ywk04zhEOrR532pOkyigwWnJdHBJm2VoJlaH9A04Rkw45X+ZhEcpFFBrpzhU/TBs+BgQp/ZZR1WcrWYkGzvojd9ICb8Sm3rD1jGKIuDrtw92SCXLHVCMv5Wm6IbJyVjtfO1ey/FC8aWgnTf7j/rrF5xcXoVpRipjNzX5Pc8VuvLPEPTMynVqU5M40Dr5koIKfSIg6gySM5BKS5t/zKzpo234Ril+u6/qEWjW8BdnXxw/98IhTfAUPlCXWhHyGEOF2hlQA0OG/zY0mmECbWESSFGPFOJzPNOXV/RwcT0jZ8gbecWrVBYrwJfPTHLudSMk+cb17Lwog0aT7oTB0ADU79VLykWKy8iWUGqRcsS0FMHfx4iGcxqht31heDU97MgsQJmlXKdLYP1/I6dXbpgsy/u67kQD9Gc6DjRYf6MdyDVrgymYFM18I6iwSKBIZaf0ZB1AiPK+L9N4eHDVJ1yNvscr5WfwpXg5IhpIJO7E2Rlhbh3xzcWLHo3yj4KLPpwHm2QkjDB3g+Ki/dU2++svSiw4QxzUoXT828rksNjdWXXTOoT6Uc130tGHrmbrfAVnzgUBztr1rkmZmPghFQKId2BR0FkP5MHIsn2tnq7yBXUyyvp+mIJovz5pl0thi5PXl2iiuF6NEbn1rcJ20T5r8gm1lx8gEiHeJss6lg569RDEQPP7gzAjSdS2qGZxEvnyz3QZtpWUXbNsTn+XMObH0/QYw5toa1OjO9wA6j722A5g+vSqeD4H/SOSdR/vl4+m6ZQ8ZjH04RKbGijjsHt0O438qwtwYcmyvBOgEVPUZoJsKm4xIbTANeMWUfW/9en6llQ09Q2GNGYzbPUrXjNPVTzcw7MRMwvfjqL8KSMFxRb41p/8TdxBoLL0Kl8GeA7gzPO76lQg0havYsWjS7Oihv6lY7NdC4fy/ZvTWPeicOus0EdBbVieeskgrHKNAXoP8f/8MT2Kfh5aG69pUbV60nurbRfvbxAQp7QY1LHpt9wdiNBHrmXapt1i9ujL40360NSK5qm9zNaW0EF79jlogPKlKxZI3MMzLpD9a/TxRRQCbc2K+kYySF2kmJWtbyC383WeVDjimpDXpDTPng23MBKrwP+ZU5SynDdFY59LM/DdidhSjcBqa6d0uHGivnMXdui4qFwj1M3fkqCda0ks0pyoNTL9x6ORZZwLL3l6A3sNruF9bVyLi1KSnsE/umc0nRu3EPtaMLzo3ltupslyMhR2aMVjgxE9qykusS4Q9K/wMBItytZsaIg9J9C/SE4w/SSWtq/FT0Tky+uDkGCtR/ESEt0G9ldpIsQTGEO5JXn7MvPOK6elxfmcFxnn5Wegq9zY0lVOaYpuTkeFtSDEJCKDu2AMEjpgXs13GEd7vcDJgl7i3uQaSJWOFaK+l51C6R77PVNOFl/jAHq+Diu+K2PhI6xodlk2ZZNsPYEFyZC+zzkiczrIxx4xx8T0WsCWYZEQMums3oxrEwdQqqbrrPmQSs07U+JfaagHlzslMbmbqiZ/9VXWQjxyfVlGjuM1qxgqeSHyVwvKl6f6DbkpOoYlTnRU9g3C1FENK717sF5/fZbbQMNqsJuaGvpqf57LLRKt/XFPDjHDUB9pbPM1r0TLH5KG74ICd43JrtiaCgNXJxHkYyzKrXQAx/XvilvzBvNLsrDUaoMvlT9ijtcF8dTjTB+U6sP7eAXA0BbuZW+EylZ66meD1auuTfd0MfWNxggoAj3px8RCXRyJkOqxjTgrgUGflH8TjL4mJSsrd9MCQoAqwmN8CrrLff3nlT3YhXUsXXNSI7lqBxXsy2OTNgHlHZmYfLg+67h3KnjOexQhueOod2KqF7EsJNcscbpywzWZmEx66XZre2BVKEbFJnQG0B9GOxbw8T1fGhwRrMuNpprpFRYE7Cir5MNHd7U7JCpO2J9mIKRaeHMdlu8wGONMGAJYjxNKFrJjvc2lFG4yXDmOMxpzYx8jAEFnKRCzlohYowX63r30lQ4KO/+nCK12rJ6Ev2TdjF+bL6tXybdp0eNo+5+MlTDa407XyUrOv6ve6iZTSZvvK9h6M/DFP/JFhv1CmcmlKYYF5oF1chTIAZiG4S+uH/Zn/hw4j9cjKhWCjhViTrooipaq6pyhd1CXpzXXodGrsmlrYX9e47fikDqRsw06oJCUGni5RqsyZWZizGL7GqQlNzsxSRWioTQh0IAEDkhV/Tg3NCJbL6dV7h4idhehjvMc9wki3ApcdPoz3QxY3jHak0bQGkXnRUa+8tsH9zttnNH66piW6P++i1LUHsmpkMLrtOKLDFAW72M8tpJm8I0A7dSD7VO/llyhZN45fvyOyq/h93CJr+tpGqzKWRJZ2n1OekXsEx5jMtdpWGkxn51Xu206pxf9ZmEONv3kabS+LJblwnigtqyvI/nbvWVzBYR4H38BjJgszCdfpqdEJXN+CD/0cWG1uvD4Eq6VRFAt2+kNUgP7ZeKlGgJjAJ1NJSJiIeUUpQoSwNoDU5OTkr/////Sv////9LAHSUYk1wAoWUjAFDlHSUUpSMA3Bvc5RNcAJ1jAloYXNfZ2F1c3OUSwCMBWdhdXNzlEcAAAAAAAAAAHVidWIu",
         "n": 2,
         "_shape": [],
         "dtype": "int64",
         "_np_random": "RandomState(MT19937)"
     },
+    "n_envs": 1,
     "num_timesteps": 102400,
     "_total_timesteps": 100000,
     "_num_timesteps_at_start": 0,
+    "seed": 1,
     "action_noise": null,
+    "start_time": 1670516894814637504,
     "learning_rate": {
         ":type:": "<class 'function'>",
+        ":serialized:": "gAWVhQIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAUsTQwSIAFMAlE6FlCmMAV+UhZSMUy9ob21lL21heGltaWxpYW4vdmVudi9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLgEMCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgMdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHn2UfZQoaBZoDYwMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBeMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP1RST9rNKDOFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
     },
+    "tensorboard_log": "runs/seals/CartPole-v0__ppo__7__1670516892/seals-CartPole-v0",
     "lr_schedule": {
         ":type:": "<class 'function'>",
+        ":serialized:": "gAWVhQIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAUsTQwSIAFMAlE6FlCmMAV+UhZSMUy9ob21lL21heGltaWxpYW4vdmVudi9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLgEMCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgMdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHn2UfZQoaBZoDYwMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBeMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP1RST9rNKDOFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
     },
     "_last_obs": null,
     "_last_episode_starts": {
     "_current_progress_remaining": -0.02400000000000002,
     "ep_info_buffer": {
         ":type:": "<class 'collections.deque'>",
+        ":serialized:": "gAWVRAwAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKUKH2UKIwBcpRHQH9AAAAAAACMAWyUTfQBjAF0lEdAMZfC66J66nV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDGXhvR7Z391fZQoaAZHQH9AAAAAAABoB030AWgIR0Axl0r9VFQVdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMZcAFPi1iXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDLI36yjYZl1fZQoaAZHQH9AAAAAAABoB030AWgIR0AyyJ7sv7FbdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMshHkLhJiHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDLH8XN1QqJ1fZQoaAZHQH9AAAAAAABoB030AWgIR0Ayx7KJVKf4dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAMsd1p0wJxHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDLHOW0JF9d1fZQoaAZHQH9AAAAAAABoB030AWgIR0Ayxu4gA6uGdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAM/rXxvvSdHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDP6mHgxagV1fZQoaAZHQH9AAAAAAABoB030AWgIR0Az+kIX0oSddX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAM/ntWuHN5nV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDP5r1uivgZ1fZQoaAZHQH9AAAAAAABoB030AWgIR0Az+XQtz0YkdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAM/k43m3fAXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDP47nxJ/Xp1fZQoaAZHQH9AAAAAAABoB030AWgIR0A1J6RyOq//dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdANSdfTkQwsXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDUnCN0eU6h1fZQoaAZHQH9AAAAAAABoB030AWgIR0A1JrQPZqVRdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdANSZ3s5XEInV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDUmPaL4vex1fZQoaAZHQH9AAAAAAABoB030AWgIR0A1JgRsdkrgdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdANSW7voePrHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDZcFeOXE611fZQoaAZHQH9AAAAAAABoB030AWgIR0A2W9VWCEpRdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdANlt92HLzPXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDZbKDCgsbx1fZQoaAZHQH9AAAAAAABoB030AWgIR0A2WunuRcNZdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdANlqtknTiKnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDZacVgx8D11fZQoaAZHQH9AAAAAAABoB030AWgIR0A2WiYsunMudX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAN5hNZeRgZ3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDeYCcPOIIp1fZQoaAZHQH9AAAAAAABoB030AWgIR0A3l7Qb+98JdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAN5deIEbHZXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDeXH2h7E511fZQoaAZHQH9AAAAAAABoB030AWgIR0A3luM+/xlQdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAN5anJkoWpXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDeWW8h9srN1fZQoaAZHQH9AAAAAAABoB030AWgIR0A6KNmDlHSXdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAOiiYPXkHU3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDooQTVUdaN1fZQoaAZHQH9AAAAAAABoB030AWgIR0A6J+x4Y77sdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAOiewcHWz4XV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDondadMCcR1fZQoaAZHQH9AAAAAAABoB030AWgIR0A6JzqKP4mDdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAOibwvxpco3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDtZEuxrzoV1fZQoaAZHQH9AAAAAAABoB030AWgIR0A7WM23rleXdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAO1h3u/k/8nV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDtYIcBEKE51fZQoaAZHQH9AAAAAAABoB030AWgIR0A7V+MIeHSGdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAO1emR/3Fk3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDtXaews5GV1fZQoaAZHQH9AAAAAAABoB030AWgIR0A7Vx6v7m+1dX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAPH9Jvo/zKHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDx/BJqZc9p1fZQoaAZHQH9AAAAAAABoB030AWgIR0A8fq3EyckMdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAPH5XyRSxaHV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDx+GQCCBf91fZQoaAZHQH9AAAAAAABoB030AWgIR0A8fdxQzk6tdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAPH2f5DZ13nV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQDx9VJcxCY11fZQoaAZHQH9AAAAAAABoB030AWgIR0A9rQKrq+rVdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAPazB2wFC9nV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQD2sa5wwTM91fZQoaAZHQH9AAAAAAABoB030AWgIR0A9rBY3eenRdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAPavYODrZ8XV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQD2rnHNorWl1fZQoaAZHQH9AAAAAAABoB030AWgIR0A9q2Cdz4lAdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAPasV+I/JNnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQD7iBd2PkrB1fZQoaAZHQH9AAAAAAABoB030AWgIR0A+4cDr7fpEdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAPuFqWTot+XV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQD7hFG5MDfZ1fZQoaAZHQH9AAAAAAABoB030AWgIR0A+4NWluWKNdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAPuCY5T6zmnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQD7gXGff4yp1fZQoaAZHQH9AAAAAAABoB030AWgIR0A+4BGx2SuAdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAQA0sH0K7ZnV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQEANDTBqKxd1fZQoaAZHQH9AAAAAAABoB030AWgIR0BADOOsDGLldX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAQAy66J66a3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQEAMna37UG51fZQoaAZHQH9AAAAAAABoB030AWgIR0BADIFvAGjcdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAQAxlg+hXbXV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQEAMQfZElVt1fZQoaAZHQH9AAAAAAABoB030AWgIR0BArgpKBd2QdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAQK3pD/lyR3V9lChoBkdAf0AAAAAAAGgHTfQBaAhHQECtvybx3FF1fZQoaAZHQH9AAAAAAABoB030AWgIR0BArZX2dupCdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAQK13jdYW+HV9lChoBkdAf0AAAAAAAGgHTfQBaAhHQECtWWhRIjJ1fZQoaAZHQH9AAAAAAABoB030AWgIR0BArTtb9qDcdX2UKGgGR0B/QAAAAAAAaAdN9AFoCEdAQK0VvddmhHVlLg=="
     },
     "ep_success_buffer": {
         ":type:": "<class 'collections.deque'>",
     "n_epochs": 10,
     "clip_range": {
         ":type:": "<class 'function'>",
+        ":serialized:": "gAWVhQIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAUsTQwSIAFMAlE6FlCmMAV+UhZSMUy9ob21lL21heGltaWxpYW4vdmVudi9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLgEMCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgMdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHn2UfZQoaBZoDYwMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBeMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP9mZmZmZmZqFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
     },
     "clip_range_vf": null,
+    "normalize_advantage": true,
+    "target_kl": null
 }

ppo-seals-CartPole-v0/policy.optimizer.pth CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d017cd4add2514e12384da546ec6ed5303da8dd59711d3d8fc5c2b28a4575c3
-size 79453

 version https://git-lfs.github.com/spec/v1
+oid sha256:5f8f95042d63d71329a83889b244855b119ff1fde42470577638bb8a7472455e
+size 82425

ppo-seals-CartPole-v0/policy.pth CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b7f8bd78b461cde84c74f83d80b296a12c5a9206435029ce843acec812a023f
 size 40513

 version https://git-lfs.github.com/spec/v1
+oid sha256:385b42f25c52b68029dd9c47cf3b1b779c3ec4a4441606218bacd6091fa5bb6f
 size 40513

ppo-seals-CartPole-v0/system_info.txt CHANGED Viewed

@@ -1,6 +1,6 @@
-OS: Linux-5.4.0-109-generic-x86_64-with-glibc2.29 #123-Ubuntu SMP Fri Apr 8 09:10:54 UTC 2022
 Python: 3.8.10
-Stable-Baselines3: 1.4.1a0
 PyTorch: 1.11.0+cu102
 GPU Enabled: False
 Numpy: 1.22.3

+OS: Linux-5.4.0-125-generic-x86_64-with-glibc2.29 #141-Ubuntu SMP Wed Aug 10 13:42:03 UTC 2022
 Python: 3.8.10
+Stable-Baselines3: 1.6.2
 PyTorch: 1.11.0+cu102
 GPU Enabled: False
 Numpy: 1.22.3

replay.mp4 CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c1428837404497e463bb6328186283ec09068bce8601f5d59ca3f46bbde225b
-size 55885

 version https://git-lfs.github.com/spec/v1
+oid sha256:4b7051380170d8c8c4a86101d6def7c0b8950e420b0bbd4775b96c1e170313aa
+size 60768

results.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"mean_reward": 500.0, "std_reward": 0.0, "is_deterministic": true, "n_eval_episodes": 10, "eval_datetime": "2022-06-~~28T14~~:06:50.~~696510~~"}


1	+ {"mean_reward": 500.0, "std_reward": 0.0, "is_deterministic": true, "n_eval_episodes": 10, "eval_datetime": "2022-12-29T14:33:46.683447"}

train_eval_metrics.zip CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23e9479340a3b7324125e6ab6b6e913dcd48421d53b468708ab3fbcf8c814931
-size 6634

 version https://git-lfs.github.com/spec/v1
+oid sha256:913814bce3741222e544117345a87cf5fd970bd301ff0985679b10763887659e
+size 6702