Merge branch 'mujoco-env' into master

README.md

@@ -62,4 +62,27 @@ Or you can also test this by copying a command from a .sh script
 ```
 python main_reflexion.py --env_name CartPole-v0 --init_summarizer cart_init_translator --curr_summarizer cart_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator 
 ```
-If you use openai key, please add "--api_type openai" at the end of the command!
+
+If you use openai key, please add "--api_type openai" at the end of the command!
+
+### Install Mujoco Environment
+1. Download the MuJoCo, recommand [mujoco210](https://github.com/google-deepmind/mujoco/releases/tag/2.1.0), for Linux, it is `mujoco210-linux-x86_64.tar.gz
+`, then
+- make new file `mkdir ~/.mujoco`
+- move the dowload file into the file `cp mujoco210-linux-x86_64.tar.gz ~/.mujoco` and extract it by `tar -zxvf mujoco210-linux-x86_64.tar.gz` 
+- `vim ~/.bashrc` and add the following line into the `.bashrc`:
+`export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/<user>/.mujoco/mujoco210/bin
+`
+
+2. install mujoco_py which allows using MuJoCo from Python 
+```
+sudo apt install libosmesa6-dev libgl1-mesa-glx libglfw3
+sudo apt-get install libglew-dev
+
+pip install mujoco-py==2.1.2.14
+pip install cython==0.29.37
+```
+
+3. install gym[mujoco]
+`pip install gym[mujoco]`
+

envs/__init__.py

@@ -1,4 +1,5 @@
 from .base_env import BaseEnv, SettableStateEnv
+
 from .classic_control import cartpole_translator, cartpole_policies
 from .classic_control import acrobot_translator, acrobot_policies
 from .classic_control import mountaincar_translator, mountaincar_policies
@@ -10,11 +11,13 @@ from .toy_text import blackjack_translator, blackjack_policies
 from .toy_text import taxi_translator, taxi_policies
 from .toy_text import cliffwalking_translator, cliffwalking_policies
 from .toy_text import frozenlake_translator, frozenlake_policies
+
 from .atari import register_environments
 from .atari import Boxing_policies, Boxing_translator, Pong_policies, Pong_translator
-
 register_environments()
 
+from .mujoco import ant_translator, ant_policies
+
 REGISTRY = {}
 REGISTRY["sampling_wrapper"] = SettableStateEnv
 REGISTRY["base_env"] = BaseEnv
@@ -91,4 +94,8 @@ REGISTRY["RepresentedPong_basic_policies"] = [
     Pong_policies.dedicated_4_policy,
     Pong_policies.dedicated_5_policy,
     Pong_policies.dedicated_6_policy,
-]
+]
+
+REGISTRY["ant_init_translator"] = ant_translator.GameDescriber
+REGISTRY["ant_basic_translator"] = ant_translator.BasicStateSequenceTranslator
+REGISTRY["ant_policies"] = [ant_policies.pseudo_random_policy, ant_policies.real_random_policy]

envs/mujoco/ant_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(8)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(8)]

envs/mujoco/ant_translator.py

@@ -0,0 +1,129 @@
+class BasicLevelTranslator:
+    def __init__(self):
+        pass
+
+    def translate(self, state):
+        (
+            torso_z_coordinate,
+            torso_x_orientation,
+            torso_y_orientation,
+            torso_z_orientation,
+            torso_w_orientation,
+            front_left_hip_angle,
+            front_left_link_angle,
+            front_right_hip_angle,
+            front_right_link_angle,
+            back_left_hip_angle,
+            back_left_link_angle,
+            back_right_hip_angle,
+            back_right_link_angle,
+            torso_x_velocity,
+            torso_y_velocity,
+            torso_z_velocity,
+            torso_x_angular_velocity,
+            torso_y_angular_velocity,
+            torso_z_angular_velocity,
+            front_left_hip_angular_velocity,
+            front_left_link_angular_velocity,
+            front_right_hip_angular_velocity,
+            front_right_link_angular_velocity,
+            back_left_hip_angular_velocity,
+            back_left_link_angular_velocity,
+            back_right_hip_angular_velocity,
+            back_right_link_angular_velocity,
+        ) = state[:27]
+
+        res = (
+            f"Torso Z-coordinate: {torso_z_coordinate:.2f}, "
+            f"Torso X-orientation: {torso_x_orientation:.2f}, "
+            f"Torso Y-orientation: {torso_y_orientation:.2f}, "
+            f"Torso Z-orientation: {torso_z_orientation:.2f}, "
+            f"Torso W-orientation: {torso_w_orientation:.2f}, "
+            f"Front Left Hip Angle: {front_left_hip_angle:.2f}, "
+            f"Front Left Link Angle: {front_left_link_angle:.2f}, "
+            f"Front Right Hip Angle: {front_right_hip_angle:.2f}, "
+            f"Front Right Link Angle: {front_right_link_angle:.2f}, "
+            f"Back Left Hip Angle: {back_left_hip_angle:.2f}, "
+            f"Back Left Link Angle: {back_left_link_angle:.2f}, "
+            f"Back Right Hip Angle: {back_right_hip_angle:.2f}, "
+            f"Back Right Link Angle: {back_right_link_angle:.2f}, "
+            f"Torso X Velocity: {torso_x_velocity:.2f}, "
+            f"Torso Y Velocity: {torso_y_velocity:.2f}, "
+            f"Torso Z Velocity: {torso_z_velocity:.2f}, "
+            f"Torso X Angular Velocity: {torso_x_angular_velocity:.2f}, "
+            f"Torso Y Angular Velocity: {torso_y_angular_velocity:.2f}, "
+            f"Torso Z Angular Velocity: {torso_z_angular_velocity:.2f}, "
+            f"Front Left Hip Angular Velocity: {front_left_hip_angular_velocity:.2f}, "
+            f"Front Left Link Angular Velocity: {front_left_link_angular_velocity:.2f}, "
+            f"Front Right Hip Angular Velocity: {front_right_hip_angular_velocity:.2f}, "
+            f"Front Right Link Angular Velocity: {front_right_link_angular_velocity:.2f}, "
+            f"Back Left Hip Angular Velocity: {back_left_hip_angular_velocity:.2f}, "
+            f"Back Left Link Angular Velocity: {back_left_link_angular_velocity:.2f}, "
+            f"Back Right Hip Angular Velocity: {back_right_hip_angular_velocity:.2f}, "
+            f"Back Right Link Angular Velocity: {back_right_link_angular_velocity:.2f}"
+        )
+
+        return res
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+    
+    def describe_goal(self):
+        return "The goal is to coordinate the four legs of the ant robot to move forward."
+
+    def describe_game(self):
+        return (
+            "In the Ant environment, you control a 3D robot called the ant. The ant has a torso with four legs, "
+            "each consisting of two links and connected by hinge joints. Your objective is to apply torques to "
+            "the eight hinge joints to coordinate the four legs and make the ant move forward in the positive x-direction. "
+            "The environment provides observations of the ant's body parts and velocities, including the torso and leg angles, "
+            "orientations, and velocities. The episode ends when the ant becomes unhealthy, which can be due to various conditions."
+        )
+
+    def describe_action(self):
+        return (
+            "Your next move: \n Please provide a list of eight numerical values, each within the range of [-1,1], "
+            "representing the torques to be applied at the eight hinge joints of the ant."
+        )
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = (
+                "Take Action: "
+                "Apply Front Left Hip Torque: {:.2f}, "
+                "Apply Front Left Link Torque: {:.2f}, "
+                "Apply Front Right Hip Torque: {:.2f}, "
+                "Apply Front Right Link Torque: {:.2f}, "
+                "Apply Back Left Hip Torque: {:.2f}, "
+                "Apply Back Left Link Torque: {:.2f}, "
+                "Apply Back Right Hip Torque: {:.2f}, "
+                "Apply Back Right Link Torque: {:.2f}"
+            ).format(
+                info['action'][0], info['action'][1], info['action'][2], info['action'][3],
+                info['action'][4], info['action'][5], info['action'][6], info['action'][7]
+            )
+
+            reward_desc = f"Result: Reward of {info['reward']:.2f}, "
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(f"{state_desc}.\\n {action_desc} \\n {reward_desc} \\n Transit to {next_state_desc}")
+        return descriptions

main_reflexion.py

@@ -221,7 +221,7 @@ if __name__ == "__main__":
         help="The actor used to select action",
     )
     parser.add_argument(
-        "--gpt_version", type=str, default="gpt-35-turbo", help="The version of GPT to use"
+        "--gpt_version", type=str, default="gpt-3.5-turbo", help="The version of GPT to use"
     )
     parser.add_argument(
         "--render", type=str, default="rgb_array", help="The render mode"
@@ -296,6 +296,7 @@ if __name__ == "__main__":
         "--api_type",
         type=str,
         default="azure",
+        choices=["azure", "openai"],
         help="choose api type, now support azure and openai"
     )
     args = parser.parse_args()

shell/test_mujoco_ant.sh

@@ -0,0 +1,6 @@
+# exe
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+python main_reflexion.py --env_name CliffWalking-v0 --init_summarizer cliffwalking_init_translator --curr_summarizer cliffwalking_basic_translator --decider exe_actor --prompt_level 2 --num_trails 1 --distiller guide_generator --prompt_path "envs/toy_text/few_shot_examples/cliffwalking"
+python main_reflexion.py --env_name CliffWalking-v0 --init_summarizer cliffwalking_init_translator --curr_summarizer cliffwalking_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+python main_reflexion.py --env_name CliffWalking-v0 --init_summarizer cliffwalking_init_translator --curr_summarizer cliffwalking_basic_translator --decider exe_actor --prompt_level 4 --num_trails 1 --distiller guide_generator --prompt_path "envs/toy_text/few_shot_examples/cliffwalking"
+python main_reflexion.py --env_name CliffWalking-v0 --init_summarizer cliffwalking_init_translator --curr_summarizer cliffwalking_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator