[tab] data/jasmine_data/ViZDoomPPO/load_model_generate_dataset.py (python)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#   /$$    /$$ /$$           /$$$$$$$                                          /$$$$$$$  /$$$$$$$   /$$$$$$     #
#  | $$   | $$|__/          | $$__  $$                                        | $$__  $$| $$__  $$ /$$__  $$    #
#  | $$   | $$ /$$ /$$$$$$$$| $$  \ $$  /$$$$$$   /$$$$$$  /$$$$$$/$$$$       | $$  \ $$| $$  \ $$| $$  \ $$    #
#  |  $$ / $$/| $$|____ /$$/| $$  | $$ /$$__  $$ /$$__  $$| $$_  $$_  $$      | $$$$$$$/| $$$$$$$/| $$  | $$    #
#   \  $$ $$/ | $$   /$$$$/ | $$  | $$| $$  \ $$| $$  \ $$| $$ \ $$ \ $$      | $$____/ | $$____/ | $$  | $$    #
#    \  $$$/  | $$  /$$__/  | $$  | $$| $$  | $$| $$  | $$| $$ | $$ | $$      | $$      | $$      | $$  | $$    #
#     \  $/   | $$ /$$$$$$$$| $$$$$$$/|  $$$$$$/|  $$$$$$/| $$ | $$ | $$      | $$      | $$      |  $$$$$$/    #
#      \_/    |__/|________/|_______/  \______/  \______/ |__/ |__/ |__/      |__/      |__/       \______/     #
#                                                                                                               #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #                                                                                                
                                                                                                         
# FORK OF LEANDRO KIELIGER'S DOOM PPO TUTORIAL: https://lkieliger.medium.com/deep-reinforcement-learning-in-practice-by-playing-doom-part-1-getting-started-618c99075c77                                                                                                       

# SCRIPT TO RUN PPO AGENT AND GENERATE DATASET FOR DOOM ENVIRONMENT. 

from dataclasses import dataclass
import imageio
from common import envs
import torch
import json
from vizdoom.vizdoom import GameVariable
import os

import numpy as np
from train_ppo_parallel import DoomWithBotsCurriculum, game_instance
from stable_baselines3.common.vec_env import (
    VecTransposeImage,
    DummyVecEnv
)
from loguru import logger
import tyro
from jasmine_data.utils import save_chunks

# To replicate frame_skip in the environment
ACTION_REPEAT = 4

@dataclass
class Args:
    num_episodes_train: int = 1000
    num_episodes_val: int = 100
    num_episodes_test: int = 100
    min_episode_length: int = 100
    max_episode_length: int = 1000
    chunk_size: int = 160
    chunks_per_file: int = 100
    agent_path: str = ""
    seed: int = 0
    output_dir: str = "data/vizdoom_episodes"
    generate_gif: bool = False

args = tyro.cli(Args)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
logger.info(f"Using device: {device}")

def dummy_vec_env_with_bots_curriculum(n_envs=1, **kwargs) -> VecTransposeImage:
    """Wraps a Doom game instance in a vectorized environment with shaped rewards and curriculum."""
    scenario = kwargs.pop("scenario")  # Remove 'scenario' from kwargs
    return VecTransposeImage(
        DummyVecEnv(
            [lambda: DoomWithBotsCurriculum(game_instance(scenario), **kwargs)] * n_envs
        )
    )

def make_gif(agent, eval_env_args):
    """Generate a GIF by running the agent in the environment.

    Args:
        agent: The trained PPO agent.
        file_path (str): Path to save the generated GIF.
        eval_env_args (dict): Arguments for the evaluation environment.
        num_episodes (int): Number of episodes to run.
    
    Returns:
        list: Collected health values for analysis.
    """
    # Set frame_skip to 1 to capture all frames
    eval_env_args['frame_skip'] = 1
    env = dummy_vec_env_with_bots_curriculum(1, **eval_env_args)

    images = []
    actions = []
    health_values = []
    current_action = None
    frame_counter = 0

    obs = env.reset()

    done = False
    while not done and frame_counter < args.max_episode_length:
        if frame_counter % ACTION_REPEAT == 0:
            current_action, _ = agent.predict(obs)
            
        obs, _, done, _ = env.step(current_action)

        # Get the raw screen buffer from the Doom game instance
        screen = env.venv.envs[0].game.get_state().screen_buffer

        # Get the current health value
        health = env.venv.envs[0].game.get_game_variable(GameVariable.HEALTH)
        health_values.append(health)  # Store the health value

        actions.append(current_action)
        images.append(screen)

        frame_counter += 1

    print("Health values:", health_values)
    print("Number of health values:", len(health_values))
    print("Number of actions:", len(actions))
    print("Number of images:", len(images))

    # Save only the first 1000 frames to avoid large file size
    imageio.mimsave(args.output_dir, images, fps=20)
    env.close()
    logger.info(f"GIF saved to {args.output_dir}")
    
    return health_values

def make_array_records_dataset(agent, eval_env_args, num_episodes, split):
    """Generate a dataset by running the agent in the environment and saving the data as array record files.

    Args:
        agent: The trained PPO agent.
        output_dir (str): Directory to save the array record files.
        eval_env_args (dict): Arguments for the evaluation environment.
        num_episodes (int): Number of episodes to run.
    """
    # Set frame_skip to 1 to capture all frames
    eval_env_args['frame_skip'] = 1
    env = dummy_vec_env_with_bots_curriculum(1, **eval_env_args)

    current_action = None
    episode_idx = 0
    episode_metadata = []
    obs_chunks = []
    act_chunks = []
    file_idx = 0
    output_dir_split = os.path.join(args.output_dir, split)
    os.makedirs(output_dir_split, exist_ok=True)

    while episode_idx < num_episodes:
        obs = env.reset()
        done = False
        frame_counter = 0

        observations_seq = []
        actions_seq = []
        health_values_seq = []
        episode_obs_chunks = []
        episode_act_chunks = []

        # --- Run episode ---
        while not done and frame_counter < args.max_episode_length:
            screen = env.venv.envs[0].game.get_state().screen_buffer
            health = env.venv.envs[0].game.get_game_variable(GameVariable.HEALTH)
            if frame_counter % ACTION_REPEAT == 0:
                current_action, _ = agent.predict(obs)

            obs, _, done, _ = env.step(current_action)

            observations_seq.append(screen)
            actions_seq.append(int(current_action.item()))
            health_values_seq.append(int(health))

            if len(observations_seq) == args.chunk_size:
                episode_obs_chunks.append(observations_seq)
                episode_act_chunks.append(actions_seq)
                observations_seq = []
                actions_seq = []

            frame_counter += 1

        # --- Save episode ---
        if frame_counter >= args.min_episode_length:
            if observations_seq:
                if len(observations_seq) < args.chunk_size:
                    print(
                        f"Warning: Inconsistent chunk_sizes. Episode has {len(observations_seq)} frames, "
                        f"which is smaller than the requested chunk_size: {args.chunk_size}. "
                        "This might lead to performance degradation during training."
                    )
                episode_obs_chunks.append(observations_seq)
                episode_act_chunks.append(actions_seq)

            obs_chunks_data = [
                np.stack(seq).astype(np.uint8)
                for seq in episode_obs_chunks
            ]
            act_chunks_data = [
                np.stack(act).astype(np.uint8) for act in episode_act_chunks
            ]
            obs_chunks.extend(obs_chunks_data)
            act_chunks.extend(act_chunks_data)

            ep_metadata, file_idx, obs_chunks, act_chunks = save_chunks(
                file_idx, args.chunks_per_file, output_dir_split, obs_chunks, act_chunks
            )
            episode_metadata.extend(ep_metadata)

            print(f"Episode {episode_idx} completed, length: {frame_counter}.")
            episode_idx += 1
        else:
            print(f"Episode too short ({frame_counter}), resampling...")
    env.close()
    return episode_metadata

def main():
    scenario = "deathmatch_simple"

    env_args = {
        "scenario": scenario,
        "frame_skip": 1,
        "frame_processor": envs.default_frame_processor,
        "n_bots": 8,
        "shaping": True,
        "initial_level": 5,
        "max_level": 5,
        "rolling_mean_length": 10,
    }

    eval_env_args = dict(env_args)
    new_env = dummy_vec_env_with_bots_curriculum(1, **env_args)
    agent = envs.load_model(
        args.agent_path,
        new_env,
    )

    if args.generate_gif:
        make_gif(agent, eval_env_args)
        return 

    train_episode_metadata = make_array_records_dataset(agent, num_episodes=args.num_episodes_train, eval_env_args=eval_env_args, split="train")
    val_episode_metadata = make_array_records_dataset(agent, num_episodes=args.num_episodes_val, eval_env_args=eval_env_args, split="val")
    test_episode_metadata = make_array_records_dataset(agent, num_episodes=args.num_episodes_test, eval_env_args=eval_env_args, split="test")
    # --- Save metadata ---
    metadata = {
        "env": "coinrun",
        "num_actions": 0, # TODO mihir
        "num_episodes_train": args.num_episodes_train,
        "num_episodes_val": args.num_episodes_val,
        "num_episodes_test": args.num_episodes_test,
        "avg_episode_len_train": np.mean(
            [ep["avg_seq_len"] for ep in train_episode_metadata]
        ),
        "avg_episode_len_val": np.mean(
            [ep["avg_seq_len"] for ep in val_episode_metadata]
        ),
        "avg_episode_len_test": np.mean(
            [ep["avg_seq_len"] for ep in test_episode_metadata]
        ),
        "episode_metadata_train": train_episode_metadata,
        "episode_metadata_val": val_episode_metadata,
        "episode_metadata_test": test_episode_metadata,
    }
    with open(os.path.join(args.output_dir, "metadata.json"), "w") as f:
        json.dump(metadata, f)

    print(f"Done generating dataset.")

if __name__ == "__main__":
    main()