Repository: sail-sg/envpool (language: python)

Issue:
[Feature Request] Mujoco integration
https://github.com/openai/gym/tree/master/gym/envs/mujoco

Env List:

- [x] Ant-v4 (#74)
- [x] HalfCheetah-v4 (#75)
- [x] Hopper-v4 (#76)
- [x] Humanoid-v4 (#77)
- [x] HumanoidStandup-v4 (#78)
- [x] InvertedDoublePendulum-v4 (@Benjamin-eecs, #83)
- [x] InvertedPendulum-v4 (#79)
- [x] Pusher-v4 (#82)
- [x] Reacher-v4 (#81)
- [x] Swimmer-v4 (#80)
- [x] Walker2d-v4 (@Benjamin-eecs, #86)
- [x] add other options to align with gym (#93)

Road Map:

- [x] Get comfortable with current codebase, go through https://envpool.readthedocs.io/en/latest/pages/env.html and add a toy environment by yourself locally;
- [x] Download Mujoco and run on your local machine [1] [5], try with different env settings and see the actual behavior;
- [x] Go through their code [1] [2] (I think it's better to go through both openai and deepmind versions, but only use deepmind's solution as reference), understand their ctype APIs and what we can use to bind with EnvPool APIs [3];
- [x] Integrate only one game and let it work;
- [x] Add some unit tests (good to submit the first PR here);
- [x] Integrate other environments (submit another PR) and related tests.

Resources:
1. https://github.com/openai/mujoco-py 
2. https://github.com/deepmind/dm_control/tree/master/dm_control/mujoco 
3. https://github.com/deepmind/mujoco/blob/main/doc/programming.rst
4. It is quite similar with Atari games which we have already integrated: https://github.com/mgbellemare/Arcade-Learning-Environment
5. First install gym and mujoco, then run with
```python
import gym
env = gym.make("Ant-v3")
env.reset()
for _ in range(10):
  env.step(env.action_space.sample())
  env.render()
```
6. https://github.com/ikostrikov/gym_dmc/blob/master/compare.py a checker


Discussion / hints:

Fix patch:
diff --git a/docs/api/mujoco.rst b/docs/api/mujoco.rst
index 9bf55508..82889723 100644
--- a/docs/api/mujoco.rst
+++ b/docs/api/mujoco.rst
@@ -19,8 +19,8 @@ Ant-v4
 <https://github.com/openai/gym/blob/master/gym/envs/mujoco/ant_v4.py>`_
 
 - Observation space: ``(111)``, first 13 elements for ``qpos[2:]``, next 14
-  elements for ``qvel``, other elements for clipped ``cfrc_ext`` (contact
-  force);
+  elements for ``qvel``, other elements for clipped ``cfrc_ext`` (com-based
+  external force on body, a.k.a. contact force);
 - Action space: ``(8)``, with range ``[-1, 1]``.
 
 
@@ -44,3 +44,17 @@ Hopper-v4
 - Observation space: ``(11)``, first 5 elements for ``qpos[1:]``, next 6
   elements for ``qvel``;
 - Action space: ``(3)``, with range ``[-1, 1]``.
+
+
+Humanoid-v4
+-----------
+
+`gym Humanoid-v4 source code
+<https://github.com/openai/gym/blob/master/gym/envs/mujoco/humanoid_v4.py>`_
+
+- Observation space: ``(376)``, first 22 elements for ``qpos[2:]``, next 23
+  elements for ``qvel``, next 140 elements for ``cinert`` (com-based body
+  inertia and mass), next 84 elements for ``cvel`` (com-based velocity [3D
+  rot; 3D tran]), next 23 elements for ``qfrc_actuator`` (actuator force),
+  next 84 elements for ``cfrc_ext`` (com-based external force on body);
+- Action space: ``(17)``, with range ``[-0.4, 0.4]``.
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 761c2140..016d9ad0 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -47,3 +47,4 @@ timestep
 RGB
 py
 Mujoco
+tran
diff --git a/envpool/mujoco/BUILD b/envpool/mujoco/BUILD
index bda8858f..7eb1d4b9 100644
--- a/envpool/mujoco/BUILD
+++ b/envpool/mujoco/BUILD
@@ -55,6 +55,15 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "humanoid",
+    hdrs = ["humanoid.h"],
+    deps = [
+        ":mujoco_env",
+        "//envpool/core:async_envpool",
+    ],
+)
+
 pybind_extension(
     name = "mujoco_envpool",
     srcs = ["mujoco_envpool.cc"],
@@ -62,6 +71,7 @@ pybind_extension(
         ":ant",
         ":half_cheetah",
         ":hopper",
+        ":humanoid",
         "//envpool/core:py_envpool",
     ],
 )
diff --git a/envpool/mujoco/__init__.py b/envpool/mujoco/__init__.py
index 87c187e2..404ef4f3 100644
--- a/envpool/mujoco/__init__.py
+++ b/envpool/mujoco/__init__.py
@@ -22,6 +22,8 @@
   _HalfCheetahEnvSpec,
   _HopperEnvPool,
   _HopperEnvSpec,
+  _HumanoidEnvPool,
+  _HumanoidEnvSpec,
 )
 
 AntEnvSpec, AntDMEnvPool, AntGymEnvPool = py_env(_AntEnvSpec, _AntEnvPool)
@@ -34,6 +36,10 @@
   _HopperEnvSpec, _HopperEnvPool
 )
 
+HumanoidEnvSpec, HumanoidDMEnvPool, HumanoidGymEnvPool = py_env(
+  _HumanoidEnvSpec, _HumanoidEnvPool
+)
+
 __all__ = [
   "AntEnvSpec",
   "AntDMEnvPool",
@@ -44,4 +50,7 @@
   "HopperEnvSpec",
   "HopperDMEnvPool",
   "HopperGymEnvPool",
+  "HumanoidEnvSpec",
+  "HumanoidDMEnvPool",
+  "HumanoidGymEnvPool",
 ]
diff --git a/envpool/mujoco/ant.h b/envpool/mujoco/ant.h
index 97750d4c..a7f84ef9 100644
--- a/envpool/mujoco/ant.h
+++ b/envpool/mujoco/ant.h
@@ -34,10 +34,11 @@ class AntEnvFns {
     return MakeDict(
         "max_episode_steps"_.bind(1000), "reward_threshold"_.bind(6000.0),
         "frame_skip"_.bind(5), "post_constraint"_.bind(true),
-        "ctrl_cost_weight"_.bind(0.5), "contact_cost_weight"_.bind(5e-4),
-        "healthy_reward"_.bind(1.0), "healthy_z_min"_.bind(0.2),
-        "healthy_z_max"_.bind(1.0), "contact_force_min"_.bind(-1.0),
-        "contact_force_max"_.bind(1.0), "reset_noise_scale"_.bind(0.1));
+        "forward_reward_weight"_.bind(1.0), "ctrl_cost_weight"_.bind(0.5),
+        "contact_cost_weight"_.bind(5e-4), "healthy_reward"_.bind(1.0),
+        "healthy_z_min"_.bind(0.2), "healthy_z_max"_.bind(1.0),
+        "contact_force_min"_.bind(-1.0), "contact_force_max"_.bind(1.0),
+        "reset_noise_scale"_.bind(0.1));
   }
   template <typename Config>
   static decltype(auto) StateSpec(const Config& conf) {
@@ -67,7 +68,8 @@ typedef class EnvSpec<AntEnvFns> AntEnvSpec;
 class AntEnv : public Env<AntEnvSpec>, public MujocoEnv {
  protected:
   int max_episode_steps_, elapsed_step_;
-  mjtNum ctrl_cost_weight_, contact_cost_weight_, healthy_reward_;
+  mjtNum ctrl_cost_weight_, contact_cost_weight_;
+  mjtNum forward_reward_weight_, healthy_reward_;
   mjtNum healthy_z_min_, healthy_z_max_;
   mjtNum contact_force_min_, contact_force_max_;
   std::unique_ptr<mjtNum> qpos0_, qvel0_;  // for align check
@@ -84,6 +86,7 @@ class AntEnv : public Env<AntEnvSpec>, public MujocoEnv {
         elapsed_step_(max_episode_steps_ + 1),
         ctrl_cost_weight_(spec.config["ctrl_cost_weight"_]),
         contact_cost_weight_(spec.config["contact_cost_weight"_]),
+        forward_reward_weight_(spec.config["forward_reward_weight"_]),
         healthy_reward_(spec.config["healthy_reward"_]),
         healthy_z_min_(spec.config["healthy_z_min"_]),
         healthy_z_max_(spec.config["healthy_z_max"_]),
@@ -140,7 +143,8 @@ class AntEnv : public Env<AntEnvSpec>, public MujocoEnv {
     }
 
     // reward and done
-    float reward = xv + healthy_reward_ - ctrl_cost - contact_cost;
+    float reward = xv * forward_reward_weight_ + healthy_reward_ - ctrl_cost -
+                   contact_cost;
     ++elapsed_step_;
     done_ = !IsHealthy() || (elapsed_step_ >= max_episode_steps_);
     WriteObs(reward, xv, yv, ctrl_cost, contact_cost, x_after, y_after);
@@ -184,7 +188,7 @@ class AntEnv : public Env<AntEnvSpec>, public MujocoEnv {
       *(obs++) = x;
     }
     // info
-    state["info:reward_forward"_] = xv;
+    state["info:reward_forward"_] = xv * forward_reward_weight_;
     state["info:reward_ctrl"_] = -ctrl_cost;
     state["info:reward_contact"_] = -contact_cost;
     state["info:reward_survive"_] = healthy_reward_;
diff --git a/envpool/mujoco/half_cheetah.h b/envpool/mujoco/half_cheetah.h
index fe4f342f..bb5e54eb 100644
--- a/envpool/mujoco/half_cheetah.h
+++ b/envpool/mujoco/half_cheetah.h
@@ -34,7 +34,8 @@ class HalfCheetahEnvFns {
     return MakeDict(
         "max_episode_steps"_.bind(1000), "reward_threshold"_.bind(4800.0),
         "frame_skip"_.bind(5), "post_constraint"_.bind(true),
-        "ctrl_cost_weight"_.bind(0.1), "reset_noise_scale"_.bind(0.1));
+        "ctrl_cost_weight"_.bind(0.1), "forward_reward_weight"_.bind(1.0),
+        "reset_noise_scale"_.bind(0.1));
   }
   template <typename Config>
   static decltype(auto) StateSpec(const Config& conf) {
@@ -59,7 +60,7 @@ typedef class EnvSpec<HalfCheetahEnvFns> HalfCheetahEnvSpec;
 class HalfCheetahEnv : public Env<HalfCheetahEnvSpec>, public MujocoEnv {
  protected:
   int max_episode_steps_, elapsed_step_;
-  mjtNum ctrl_cost_weight_;
+  mjtNum ctrl_cost_weight_, forward_reward_weight_;
   std::unique_ptr<mjtNum> qpos0_, qvel0_;  // for align check
   std::uniform_real_distribution<> dist_qpos_;
   std::normal_distribution<> dist_qvel_;
@@ -73,6 +74,7 @@ class HalfCheetahEnv : public Env<HalfCheetahEnvSpec>, public MujocoEnv {
         max_episode_steps_(spec.config["max_episode_steps"_]),
         elapsed_step_(max_episode_steps_ + 1),
         ctrl_cost_weight_(spec.config["ctrl_cost_weight"_]),
+        forward_reward_weight_(spec.config["forward_reward_weight"_]),
         qpos0_(new mjtNum[model_->nq]),
         qvel0_(new mjtNum[model_->nv]),
         dist_qpos_(-spec.config["reset_noise_scale"_],
@@ -114,7 +116,7 @@ class HalfCheetahEnv : public Env<HalfCheetahEnvSpec>, public MujocoEnv {
     mjtNum dt = frame_skip_ * model_->opt.timestep;
     mjtNum xv = (x_after - x_before) / dt;
     // reward and done
-    float reward = xv - ctrl_cost;
+    float reward = xv * forward_reward_weight_ - ctrl_cost;
     done_ = (++elapsed_step_ >= max_episode_steps_);
     WriteObs(reward, xv, ctrl_cost, x_after);
   }
@@ -133,7 +135,7 @@ class HalfCheetahEnv : public Env<HalfCheetahEnvSpec>, public MujocoEnv {
       *(obs++) = data_->qvel[i];
     }
     // info
-    state["info:reward_run"_] = xv;
+    state["info:reward_run"_] = xv * forward_reward_weight_;
     state["info:reward_ctrl"_] = -ctrl_cost;
     state["info:x_position"_] = x_after;
     state["info:x_velocity"_] = xv;
diff --git a/envpool/mujoco/hopper.h b/envpool/mujoco/hopper.h
index 37598db6..e0a83783 100644
--- a/envpool/mujoco/hopper.h
+++ b/envpool/mujoco/hopper.h
@@ -34,11 +34,12 @@ class HopperEnvFns {
     return MakeDict(
         "max_episode_steps"_.bind(1000), "reward_threshold"_.bind(6000.0),
         "frame_skip"_.bind(4), "post_constraint"_.bind(true),
-        "ctrl_cost_weight"_.bind(1e-3), "healthy_reward"_.bind(1.0),
-        "velocity_min"_.bind(-10.0), "velocity_max"_.bind(10.0),
-        "healthy_state_min"_.bind(-100.0), "healthy_state_max"_.bind(100.0),
-        "healthy_angle_min"_.bind(-0.2), "healthy_angle_max"_.bind(0.2),
-        "healthy_z_min"_.bind(0.7), "reset_noise_scale"_.bind(5e-3));
+        "ctrl_cost_weight"_.bind(1e-3), "forward_reward_weight"_.bind(1.0),
+        "healthy_reward"_.bind(1.0), "velocity_min"_.bind(-10.0),
+        "velocity_max"_.bind(10.0), "healthy_state_min"_.bind(-100.0),
+        "healthy_state_max"_.bind(100.0), "healthy_angle_min"_.bind(-0.2),
+        "healthy_angle_max"_.bind(0.2), "healthy_z_min"_.bind(0.7),
+        "reset_noise_scale"_.bind(5e-3));
   }
   template <typename Config>
   static decltype(auto) StateSpec(const Config& conf) {
@@ -61,7 +62,8 @@ typedef class EnvSpec<HopperEnvFns> HopperEnvSpec;
 class HopperEnv : public Env<HopperEnvSpec>, public MujocoEnv {
  protected:
   int max_episode_steps_, elapsed_step_;
-  mjtNum ctrl_cost_weight_, healthy_reward_, healthy_z_min_;
+  mjtNum ctrl_cost_weight_, forward_reward_weight_;
+  mjtNum healthy_reward_, healthy_z_min_;
   mjtNum velocity_min_, velocity_max_;
   mjtNum healthy_state_min_, healthy_state_max_;
   mjtNum healthy_angle_min_, healthy_angle_max_;
@@ -77,6 +79,7 @@ class HopperEnv : public Env<HopperEnvSpec>, public MujocoEnv {
         max_episode_steps_(spec.config["max_episode_steps"_]),
         elapsed_step_(max_episode_steps_ + 1),
         ctrl_cost_weight_(spec.config["ctrl_cost_weight"_]),
+        forward_reward_weight_(spec.config["forward_reward_weight"_]),
         healthy_reward_(spec.config["healthy_reward"_]),
         healthy_z_min_(spec.config["healthy_z_min"_]),
         velocity_min_(spec.config["velocity_min"_]),
@@ -125,7 +128,7 @@ class HopperEnv : public Env<HopperEnvSpec>, public MujocoEnv {
     mjtNum dt = frame_skip_ * model_->opt.timestep;
     mjtNum xv = (x_after - x_before) / dt;
     // reward and done
-    float reward = xv + healthy_reward_ - ctrl_cost;
+    float reward = xv * forward_reward_weight_ + healthy_reward_ - ctrl_cost;
     ++elapsed_step_;
     done_ = !IsHealthy() || (elapsed_step_ >= max_episode_steps_);
     WriteObs(reward, xv, x_after);
diff --git a/envpool/mujoco/humanoid.h b/envpool/mujoco/humanoid.h
new file mode 100644
index 00000000..73b09478
--- /dev/null
+++ b/envpool/mujoco/humanoid.h
@@ -0,0 +1,216 @@
+/*
+ * Copyright 2022 Garena Online Private Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ENVPOOL_MUJOCO_HUMANOID_H_
+#define ENVPOOL_MUJOCO_HUMANOID_H_
+
+#include <algorithm>
+#include <limits>
+#include <memory>
+#include <string>
+
+#include "envpool/core/async_envpool.h"
+#include "envpool/core/env.h"
+#include "envpool/mujoco/mujoco_env.h"
+
+namespace mujoco {
+
+class HumanoidEnvFns {
+ public:
+  static decltype(auto) DefaultConfig() {
+    return MakeDict(
+        "max_episode_steps"_.bind(1000), "frame_skip"_.bind(5),
+        "post_constraint"_.bind(true), "forward_reward_weight"_.bind(1.25),
+        "ctrl_cost_weight"_.bind(0.1), "contact_cost_weight"_.bind(5e-7),
+        "contact_cost_max"_.bind(10.0), "healthy_reward"_.bind(5.0),
+        "healthy_z_min"_.bind(1.0), "healthy_z_max"_.bind(2.0),
+        "reset_noise_scale"_.bind(1e-2));
+  }
+  template <typename Config>
+  static decltype(auto) StateSpec(const Config& conf) {
+    mjtNum inf = std::numeric_limits<mjtNum>::infinity();
+    return MakeDict("obs"_.bind(Spec<mjtNum>({376}, {-inf, inf})),
+                    "info:reward_linvel"_.bind(Spec<mjtNum>({-1})),
+                    "info:reward_quadctrl"_.bind(Spec<mjtNum>({-1})),
+                    "info:reward_alive"_.bind(Spec<mjtNum>({-1})),
+                    "info:reward_impact"_.bind(Spec<mjtNum>({-1})),
+                    "info:x_position"_.bind(Spec<mjtNum>({-1})),
+                    "info:y_position"_.bind(Spec<mjtNum>({-1})),
+                    "info:distance_from_origin"_.bind(Spec<mjtNum>({-1})),
+                    "info:x_velocity"_.bind(Spec<mjtNum>({-1})),
+                    "info:y_velocity"_.bind(Spec<mjtNum>({-1})),
+                    // TODO(jiayi): remove these two lines for speed
+                    "info:qpos0"_.bind(Spec<mjtNum>({24})),
+                    "info:qvel0"_.bind(Spec<mjtNum>({23})));
+  }
+  template <typename Config>
+  static decltype(auto) ActionSpec(const Config& conf) {
+    return MakeDict("action"_.bind(Spec<mjtNum>({-1, 17}, {-0.4f, 0.4f})));
+  }
+};
+
+typedef class EnvSpec<HumanoidEnvFns> HumanoidEnvSpec;
+
+class HumanoidEnv : public Env<HumanoidEnvSpec>, public MujocoEnv {
+ protected:
+  int max_episode_steps_, elapsed_step_;
+  mjtNum ctrl_cost_weight_, contact_cost_weight_, contact_cost_max_;
+  mjtNum forward_reward_weight_, healthy_reward_;
+  mjtNum healthy_z_min_, healthy_z_max_;
+  mjtNum mass_x_, mass_y_;
+  std::unique_ptr<mjtNum> qpos0_, qvel0_;  // for align check
+  std::uniform_real_distribution<> dist_;
+  bool done_;
+
+ public:
+  HumanoidEnv(const Spec& spec, int env_id)
+      : Env<HumanoidEnvSpec>(spec, env_id),
+        MujocoEnv(spec.config["base_path"_] + "/mujoco/assets/humanoid.xml",
+                  spec.config["frame_skip"_], spec.config["post_constraint"_]),
+        max_episode_steps_(spec.config["max_episode_steps"_]),
+        elapsed_step_(max_episode_steps_ + 1),
+        ctrl_cost_weight_(spec.config["ctrl_cost_weight"_]),
+        contact_cost_weight_(spec.config["contact_cost_weight"_]),
+        contact_cost_max_(spec.config["contact_cost_max"_]),
+        forward_reward_weight_(spec.config["forward_reward_weight"_]),
+        healthy_reward_(spec.config["healthy_reward"_]),
+        healthy_z_min_(spec.config["healthy_z_min"_]),
+        healthy_z_max_(spec.config["healthy_z_max"_]),
+        mass_x_(0),
+        mass_y_(0),
+        qpos0_(new mjtNum[model_->nq]),
+        qvel0_(new mjtNum[model_->nv]),
+        dist_(-spec.config["reset_noise_scale"_],
+              spec.config["reset_noise_scale"_]),
+        done_(true) {}
+
+  void MujocoResetModel() {
+    for (int i = 0; i < model_->nq; ++i) {
+      data_->qpos[i] = qpos0_.get()[i] = init_qpos_[i] + dist_(gen_);
+    }
+    for (int i = 0; i < model_->nv; ++i) {
+      data_->qvel[i] = qvel0_.get()[i] = init_qvel_[i] + dist_(gen_);
+    }
+  }
+
+  bool IsDone() override { return done_; }
+
+  void Reset() override {
+    done_ = false;
+    elapsed_step_ = 0;
+    MujocoReset();
+    WriteObs(0.0f, 0, 0, 0, 0, 0, 0);
+  }
+
+  void Step(const Action& action) override {
+    // step
+    mjtNum* act = static_cast<mjtNum*>(action["action"_].data());
+    GetMassCenter();
+    mjtNum x_before = mass_x_, y_before = mass_y_;
+    MujocoStep(act);
+    GetMassCenter();
+    mjtNum x_after = mass_x_, y_after = mass_y_;
+
+    // ctrl_cost
+    mjtNum ctrl_cost = 0.0;
+    for (int i = 0; i < model_->nu; ++i) {
+      // ctrl_cost += ctrl_cost_weight_ * act[i] * act[i];
+      ctrl_cost += ctrl_cost_weight_ * data_->ctrl[i] * data_->ctrl[i];
+    }
+    // xv and yv
+    mjtNum dt = frame_skip_ * model_->opt.timestep;
+    mjtNum xv = (x_after - x_before) / dt;
+    mjtNum yv = (y_after - y_before) / dt;
+    // contact cost
+    mjtNum contact_cost = 0.0;
+    for (int i = 0; i < 6 * model_->nbody; ++i) {
+      mjtNum x = data_->cfrc_ext[i];
+      contact_cost += contact_cost_weight_ * x * x;
+    }
+    contact_cost = std::min(contact_cost, contact_cost_max_);
+
+    // reward and done
+    float reward = xv * forward_reward_weight_ + healthy_reward_ - ctrl_cost -
+                   contact_cost;
+    ++elapsed_step_;
+    done_ = !IsHealthy() || (elapsed_step_ >= max_episode_steps_);
+    WriteObs(reward, xv, yv, ctrl_cost, contact_cost, x_after, y_after);
+  }
+
+ private:
+  bool IsHealthy() {
+    return healthy_z_min_ < data_->qpos[2] && data_->qpos[2] < healthy_z_max_;
+  }
+
+  void GetMassCenter() {
+    mjtNum mass_sum = 0.0;
+    mass_x_ = mass_y_ = 0.0;
+    for (int i = 0; i < model_->nbody; ++i) {
+      mjtNum mass = model_->body_mass[i];
+      mass_sum += mass;
+      mass_x_ += mass * data_->xipos[3 * i + 0];
+      mass_y_ += mass * data_->xipos[3 * i + 1];
+    }
+    mass_x_ /= mass_sum;
+    mass_y_ /= mass_sum;
+  }
+
+  void WriteObs(float reward, mjtNum xv, mjtNum yv, mjtNum ctrl_cost,
+                mjtNum contact_cost, mjtNum x_after,
+                mjtNum y_after) {  // NOLINT
+    State state = Allocate();
+    state["reward"_] = reward;
+    // obs
+    mjtNum* obs = static_cast<mjtNum*>(state["obs"_].data());
+    for (int i = 2; i < model_->nq; ++i) {
+      *(obs++) = data_->qpos[i];
+    }
+    for (int i = 0; i < model_->nv; ++i) {
+      *(obs++) = data_->qvel[i];
+    }
+    for (int i = 0; i < 10 * model_->nbody; ++i) {
+      *(obs++) = data_->cinert[i];
+    }
+    for (int i = 0; i < 6 * model_->nbody; ++i) {
+      *(obs++) = data_->cvel[i];
+    }
+    for (int i = 0; i < model_->nv; ++i) {
+      *(obs++) = data_->qfrc_actuator[i];
+    }
+    for (int i = 0; i < 6 * model_->nbody; ++i) {
+      *(obs++) = data_->cfrc_ext[i];
+    }
+    // info
+    state["info:reward_linvel"_] = xv * forward_reward_weight_;
+    state["info:reward_quadctrl"_] = -ctrl_cost;
+    state["info:reward_impact"_] = -contact_cost;
+    state["info:reward_alive"_] = healthy_reward_;
+    state["info:x_position"_] = x_after;
+    state["info:y_position"_] = y_after;
+    state["info:distance_from_origin"_] =
+        std::sqrt(x_after * x_after + y_after * y_after);
+    state["info:x_velocity"_] = xv;
+    state["info:y_velocity"_] = yv;
+    state["info:qpos0"_].Assign(qpos0_.get(), model_->nq);
+    state["info:qvel0"_].Assign(qvel0_.get(), model_->nv);
+  }
+};
+
+typedef AsyncEnvPool<HumanoidEnv> HumanoidEnvPool;
+
+}  // namespace mujoco
+
+#endif  // ENVPOOL_MUJOCO_HUMANOID_H_
diff --git a/envpool/mujoco/mujoco_envpool.cc b/envpool/mujoco/mujoco_envpool.cc
index b0a66a99..ab1d8108 100644
--- a/envpool/mujoco/mujoco_envpool.cc
+++ b/envpool/mujoco/mujoco_envpool.cc
@@ -16,6 +16,7 @@
 #include "envpool/mujoco/ant.h"
 #include "envpool/mujoco/half_cheetah.h"
 #include "envpool/mujoco/hopper.h"
+#include "envpool/mujoco/humanoid.h"
 
 typedef PyEnvSpec<mujoco::AntEnvSpec> AntEnvSpec;
 typedef PyEnvPool<mujoco::AntEnvPool> AntEnvPool;
@@ -26,8 +27,12 @@ typedef PyEnvPool<mujoco::HalfCheetahEnvPool> HalfCheetahEnvPool;
 typedef PyEnvSpec<mujoco::HopperEnvSpec> HopperEnvSpec;
 typedef PyEnvPool<mujoco::HopperEnvPool> HopperEnvPool;
 
+typedef PyEnvSpec<mujoco::HumanoidEnvSpec> HumanoidEnvSpec;
+typedef PyEnvPool<mujoco::HumanoidEnvPool> HumanoidEnvPool;
+
 PYBIND11_MODULE(mujoco_envpool, m) {
   REGISTER(m, AntEnvSpec, AntEnvPool)
   REGISTER(m, HalfCheetahEnvSpec, HalfCheetahEnvPool)
   REGISTER(m, HopperEnvSpec, HopperEnvPool)
+  REGISTER(m, HumanoidEnvSpec, HumanoidEnvPool)
 }
diff --git a/envpool/mujoco/registration.py b/envpool/mujoco/registration.py
index cab7583d..ba6e56b9 100644
--- a/envpool/mujoco/registration.py
+++ b/envpool/mujoco/registration.py
@@ -51,3 +51,13 @@
   reward_threshold=3800.0,
   base_path=base_path,
 )
+
+register(
+  task_id="Humanoid-v4",
+  import_path="envpool.mujoco",
+  spec_cls="HumanoidEnvSpec",
+  dm_cls="HumanoidDMEnvPool",
+  gym_cls="HumanoidGymEnvPool",
+  max_episode_steps=1000,
+  base_path=base_path,
+)


Test patch:
diff --git a/envpool/make_test.py b/envpool/make_test.py
index af1a7648..b13b6d21 100644
--- a/envpool/make_test.py
+++ b/envpool/make_test.py
@@ -14,6 +14,7 @@
 """Test for envpool.make."""
 
 import pprint
+from typing import List
 
 import dm_env
 import gym
@@ -51,16 +52,8 @@ def test_make_vizdoom(self) -> None:
     self.assertIsInstance(env, gym.Env)
     env.reset()
 
-  def test_make_classic_and_toytext(self) -> None:
-    classic = [
-      "CartPole-v0", "CartPole-v1", "Pendulum-v0", "MountainCar-v0",
-      "MountainCarContinuous-v0", "Acrobot-v1"
-    ]
-    toytext = [
-      "Catch-v0", "FrozenLake-v1", "FrozenLake8x8-v1", "Taxi-v3", "NChain-v0",
-      "CliffWalking-v0", "Blackjack-v1"
-    ]
-    for task_id in classic + toytext:
+  def check_step(self, env_list: List[str]) -> None:
+    for task_id in env_list:
       envpool.make_spec(task_id)
       env_gym = envpool.make_gym(task_id)
       env_dm = envpool.make_dm(task_id)
@@ -71,18 +64,38 @@ def test_make_classic_and_toytext(self) -> None:
       env_dm.reset()
       env_gym.reset()
 
+  def test_make_classic(self) -> None:
+    self.check_step(
+      [
+        "CartPole-v0",
+        "CartPole-v1",
+        "Pendulum-v0",
+        "MountainCar-v0",
+        "MountainCarContinuous-v0",
+        "Acrobot-v1",
+      ]
+    )
+
+  def test_make_toytext(self) -> None:
+    self.check_step(
+      [
+        "Catch-v0",
+        "FrozenLake-v1",
+        "FrozenLake8x8-v1",
+        "Taxi-v3",
+        "NChain-v0",
+        "CliffWalking-v0",
+        "Blackjack-v1",
+      ]
+    )
+
   def test_make_mujoco(self) -> None:
-    mujoco = ["Ant-v4", "HalfCheetah-v4", "Hopper-v4"]
-    for task_id in mujoco:
-      envpool.make_spec(task_id)
-      env_gym = envpool.make_gym(task_id)
-      env_dm = envpool.make_dm(task_id)
-      print(env_dm)
-      print(env_gym)
-      self.assertIsInstance(env_gym, gym.Env)
-      self.assertIsInstance(env_dm, dm_env.Environment)
-      env_dm.reset()
-      env_gym.reset()
+    self.check_step([
+      "Ant-v4",
+      "HalfCheetah-v4",
+      "Hopper-v4",
+      "Humanoid-v4",
+    ])
 
 
 if __name__ == "__main__":
diff --git a/envpool/mujoco/mujoco_test.py b/envpool/mujoco/mujoco_test.py
index 5643533f..b43e83f4 100644
--- a/envpool/mujoco/mujoco_test.py
+++ b/envpool/mujoco/mujoco_test.py
@@ -28,6 +28,8 @@
   HalfCheetahGymEnvPool,
   HopperEnvSpec,
   HopperGymEnvPool,
+  HumanoidEnvSpec,
+  HumanoidGymEnvPool,
 )
 
 
@@ -130,6 +132,13 @@ def test_hopper(self) -> None:
     self.run_align_check(env0, env1)
     self.run_deterministic_check(HopperEnvSpec, HopperGymEnvPool)
 
+  def test_humanoid(self) -> None:
+    env0 = mjc_mwe.HumanoidEnv()
+    env1 = HumanoidGymEnvPool(HumanoidEnvSpec(HumanoidEnvSpec.gen_config()))
+    self.run_space_check(env0, env1)
+    self.run_align_check(env0, env1)
+    self.run_deterministic_check(HumanoidEnvSpec, HumanoidGymEnvPool)
+
 
 if __name__ == "__main__":
   absltest.main()