Repository: intelligent-environments-lab/CityLearn (language: python)

Issue:
[FEATURE REQUEST] RLlib single-agent and multi-agent wrapper
**Is your feature request related to a problem? Please describe.**
I want to be able to use RLlib library with CityLearn

**Describe the solution you'd like**
- Wrapper that provides interface between RLlib and CityLearn
- Examples in [quickstart.ipynb](https://github.com/intelligent-environments-lab/CityLearn/blob/master/examples/quickstart.ipynb) to use single-agent and multi-agent wrappers

**Describe alternatives you've considered**
NIL

**Additional context**
NIL



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/pypi_deploy.yml b/.github/workflows/pypi_deploy.yml
index 07e35a95..3563eed0 100644
--- a/.github/workflows/pypi_deploy.yml
+++ b/.github/workflows/pypi_deploy.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.12]
+        python-version: [3.11]
 
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml
index ff9bd735..eda05600 100644
--- a/.github/workflows/sphinx.yml
+++ b/.github/workflows/sphinx.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.12]
+        python-version: [3.11]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/MANIFEST.in b/MANIFEST.in
index 051da7de..91039768 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,3 @@
 include citylearn/assets/*
-recursive-include citylearn/data *
\ No newline at end of file
+recursive-include citylearn/data *
+recursive-include citylearn/misc *
\ No newline at end of file
diff --git a/assets/images/citylearn_systems.pptx b/assets/images/citylearn_systems.pptx
index f3ebb2b5..3df7fd14 100644
Binary files a/assets/images/citylearn_systems.pptx and b/assets/images/citylearn_systems.pptx differ
diff --git a/citylearn/__init__.py b/citylearn/__init__.py
index f8115612..04188a16 100644
--- a/citylearn/__init__.py
+++ b/citylearn/__init__.py
@@ -1,1 +1,1 @@
-__version__ = '2.1.2'
+__version__ = '2.2.0'
diff --git a/citylearn/__main__.py b/citylearn/__main__.py
index dbd569ff..86d75264 100644
--- a/citylearn/__main__.py
+++ b/citylearn/__main__.py
@@ -1,47 +1,511 @@
 import argparse
-from citylearn.__init__ import __version__
+import concurrent.futures
+import datetime
+import getpass
+import importlib
 import inspect
+from multiprocessing import cpu_count
+import os
 from pathlib import Path
-import pickle
+import shutil
+import socket
+import subprocess
 import sys
+from typing import Any, List, Mapping, Tuple, Union
+import uuid
+from citylearn.agents.base import Agent as CityLearnAgent
 from citylearn.citylearn import CityLearnEnv
+from citylearn.data import DataSet
+from citylearn.utilities import read_pickle, read_yaml, write_json, write_pickle
+import pandas as pd
+import simplejson as json
 
-def __learn(**kwargs):
-    env = CityLearnEnv(kwargs['schema'])
-    model = env.load_agent()
+try:
+    from stable_baselines3.common.base_class import BaseAlgorithm as StableBaselines3Agent
+
+except (ImportError, ModuleNotFoundError):
+    pass
+
+def run_work_order(work_order_filepath, max_workers=None, start_index=None, end_index=None, virtual_environment_path=None, windows_system=None):
+    work_order_filepath = Path(work_order_filepath)
+
+    if virtual_environment_path is not None:    
+        if windows_system:
+            virtual_environment_command = f'"{os.path.join(virtual_environment_path, "Scripts", "Activate.ps1")}"'
+        
+        else:
+            virtual_environment_command = f'source "{os.path.join(virtual_environment_path, "bin", "activate")}"'
+    
+    else:
+        virtual_environment_command = 'echo "No virtual environment"'
+
+    with open(work_order_filepath,mode='r') as f:
+        args = f.read()
+    
+    args = args.strip('\n').split('\n')
+    start_index = 0 if start_index is None else start_index
+    end_index = len(args) - 1 if end_index is None else end_index
+    assert start_index <= end_index, 'start_index must be <= end_index'
+    assert start_index < len(args), 'start_index must be < number of jobs'
+    args = args[start_index:end_index + 1]
+    args = [a for a in args if not a.startswith('#')]
+    args = [f'{virtual_environment_command} && {a}' for a in args]
+    max_workers = cpu_count() if max_workers is None else max_workers
+    
+    with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+        print(f'Will use {max_workers} workers for job.')
+        print(f'Pooling {len(args)} jobs to run in parallel...')
+        results = [executor.submit(subprocess.run,**{'args':a, 'shell':True}) for a in args]
+            
+        for future in concurrent.futures.as_completed(results):
+            try:
+                print(future.result())
+            
+            except Exception as e:
+                print(e)
+
+class Simulator:
+    def __init__(self, schema: str, agent_name: str = None, env_kwargs: Mapping[str, Any] = None, agent_kwargs: Mapping[str, Any] = None, wrappers: List[str] = None,
+     time_series_variables: List[str] = None, simulation_id: str = None, output_directory: Union[Path, str] = None, agent_filepath: Union[Path, str] = None,
+     random_seed: int = None, overwrite: bool = None
+    ) -> None:
+        self.schema = schema
+        self.agent_name = agent_name
+        self.env_kwargs = env_kwargs
+        self.agent_kwargs = agent_kwargs
+        self.random_seed = random_seed
+        self.wrappers = wrappers
+        self.time_series_variables = time_series_variables
+        self.simulation_id = simulation_id
+        self.overwrite = overwrite
+        self.output_directory = output_directory
+        self.agent_filepath = agent_filepath
+        self.__reset()
+
+    @property
+    def schema(self) -> str:
+        return self.__schema
+    
+    @property
+    def agent_name(self) -> str:
+        return self.__agent_name
     
-    try:
-        model.learn(
-            episodes=kwargs.get('episodes', env.schema['episodes']),
-            deterministic_finish=kwargs['deterministic_finish'],
-            logging_level=kwargs['logging_level']
-        )
+    @property
+    def env_kwargs(self) -> Mapping[str, Any]:
+        return self.__env_kwargs
     
-    finally:
-        with open(kwargs['filepath'], 'wb') as f:
-            pickle.dump(model, f)
+    @property
+    def agent_kwargs(self) -> Mapping[str, Any]:
+        return self.__agent_kwargs
+    
+    @property
+    def wrappers(self) -> List[str]:
+        return self.__wrappers
+    
+    @property
+    def time_series_variables(self) -> List[str]:
+        return self.__time_series_variables
+    
+    @property
+    def simulation_id(self) -> str:
+        return self.__simulation_id
+    
+    @property
+    def output_directory(self) -> Path:
+        return self.__output_directory
+    
+    @property
+    def agent_filepath(self) -> Path:
+        return self.__agent_filepath
+    
+    @property
+    def random_seed(self) -> int:
+        return self.__random_seed
+    
+    @property
+    def overwrite(self) -> bool:
+        return self.__overwrite
+    
+    @schema.setter
+    def schema(self, value: str):
+        self.__schema = value
+
+    @agent_name.setter
+    def agent_name(self, value: str):
+        self.__agent_name = value
+
+    @env_kwargs.setter
+    def env_kwargs(self, value: Mapping[str, Any]):
+        self.__env_kwargs = {} if value is None else value
+
+    @agent_kwargs.setter
+    def agent_kwargs(self, value: Mapping[str, Any]):
+        self.__agent_kwargs = {} if value is None else value
+
+    @wrappers.setter
+    def wrappers(self, value:List[str]):
+        self.__wrappers = [] if value is None else value
+
+    @time_series_variables.setter
+    def time_series_variables(self, value: List[str]):
+        self.__time_series_variables = self.get_default_time_series_variables() if value is None else value
+
+    @simulation_id.setter
+    def simulation_id(self, value: str):
+        self.__simulation_id = f'citylearn-simulation-{uuid.uuid4().hex}' if value is None else value
+
+    @output_directory.setter
+    def output_directory(self, value: Path):
+        self.__output_directory = Path(os.path.join('citylearn_simulations', self.simulation_id))\
+            if value is None else value
+        
+        if os.path.isdir(self.__output_directory) and self.overwrite:
+            shutil.rmtree(self.__output_directory)
+        
+        else:
+            pass
+
+        os.makedirs(self.__output_directory, exist_ok=True)
+
+    @agent_filepath.setter
+    def agent_filepath(self, value: Path):
+        self.__agent_filepath =  value if value is None else Path(value)
+
+    @random_seed.setter
+    def random_seed(self, value: int):
+        self.__random_seed = value
+
+        if self.random_seed is not None:
+            self.env_kwargs['random_seed'] = self.random_seed
+
+            if self.agent_name is not None:
+                random_seed_name = 'seed' if 'stable_baselines3' in self.agent_name else 'random_seed'
+                self.agent_kwargs[random_seed_name] = self.random_seed
+            
+            else:
+                pass
+
+        else:
+            pass
+
+    @overwrite.setter
+    def overwrite(self, value: bool):
+        self.__overwrite = True if value is None else value
+
+    def __get_evaluation_summary(self):        
+        return {
+            'evaluation_episode_time_steps': [
+                self.env.unwrapped.episode_tracker.episode_start_time_step, 
+                self.env.unwrapped.episode_tracker.episode_end_time_step
+            ],
+            'evaluation_start_timestamp': self.__evaluation_start_timestamp,
+            'evaluation_end_timestamp': self.__evaluation_end_timestamp,
+            'evaluation': self.env.unwrapped.evaluate().pivot(index='name', columns='cost_function', values='value').to_dict('index'),
+            'episode_reward_summary': self.env.unwrapped.episode_rewards[-1],
+            'episode_rewards': self.env.unwrapped.rewards,
+            'time_series': self.__get_time_series().to_dict('list'),
+            'actions': self.__actions_list,
+        }
+
+    def __get_time_series(self) -> pd.DataFrame:
+        data_list = []
+
+        for b in self.env.unwrapped.buildings:
+            data = {}
+
+            for variable in self.time_series_variables:
+                key = b
+                values = variable.split('.')
+
+                for i in range(len(values)):
+                    if hasattr(key, values[i]):
+                        value = getattr(key, values[i])
+                        key = value
+                    
+                    else:
+                        pass
+                
+                data[variable.replace('.', '_')] = value
+            
+            data = pd.DataFrame(data)
+            data.insert(0, 'time_step', data.index)
+            data.insert(1, 'building_name', b.name)
+            data_list.append(data)
+
+        return pd.concat(data_list, ignore_index=True)
+
+    def __get_training_summary(self):
+        return {
+            'hostname': socket.gethostname(),
+            'username': getpass.getuser(),
+            'simulation_id': self.simulation_id,
+            'agent': self.agent.__class__.__name__,
+            'agent_kwargs': self.agent_kwargs,
+            'wrappers': self.wrappers,
+            'train_episodes': self.env.unwrapped.episode_tracker.episode,
+            'train_episode_time_steps': self.env.unwrapped.episode_time_steps,
+            'train_start_timestamp': self.__train_start_timestamp,
+            'train_end_timestamp': self.__train_end_timestamp,
+            'train_episode_reward_summary': self.env.unwrapped.episode_rewards,
+            'env_metadata': self.env.unwrapped.get_metadata(),
+        }
+
+    def __evaluate(self):
+        observations, _ = self.env.reset()
+        actions_list = []
+        self.__evaluation_start_timestamp = datetime.datetime.now(datetime.UTC)
+        
+        while not self.env.terminated:
+            if isinstance(self.agent, CityLearnAgent):
+                actions = self.agent.predict(observations, deterministic=True)
+                actions_list.append(self.env.unwrapped._parse_actions(actions))
+            
+            elif isinstance(self.agent, StableBaselines3Agent):
+                actions, _ = self.agent.predict(observations, deterministic=True)
+                actions_list.append(self.env.unwrapped._parse_actions([actions]))
+
+            else:
+                raise Exception(f'Unknown agent type: {type(self.agent)}')
+
+            observations, _, _, _, _ = self.env.step(actions)
+
+        self.__evaluation_end_timestamp = datetime.datetime.now(datetime.UTC)
+        self.__actions_list = actions_list
+
+    def __train(self, episodes: int):
+        kwargs = {}
+        self.__train_start_timestamp = datetime.datetime.now(datetime.UTC)
+
+        if isinstance(self.agent, CityLearnAgent):
+            kwargs = {**kwargs, 'episodes': episodes}
+            self.agent.learn(**kwargs)
+        
+        else:
+            kwargs = {**kwargs, 'total_timesteps': episodes*self.env.unwrapped.time_steps}
+            self.agent = self.agent.learn(**kwargs)
+        
+        self.__train_end_timestamp = datetime.datetime.now(datetime.UTC)
+
+    def __save_agent(self):
+        if isinstance(self.agent, CityLearnAgent):
+            filepath = os.path.join(self.output_directory, f'{self.simulation_id}-agent.pkl')
+            write_pickle(filepath, self.agent)
+        
+        else:
+            filepath = os.path.join(self.output_directory, f'{self.simulation_id}-agent')
+            self.agent.save(filepath)
+
+    def __set_agent(self) -> Union[CityLearnAgent, StableBaselines3Agent]:
+        if self.agent_filepath is None:
+            agent = self.env.unwrapped.load_agent(
+                agent=self.agent_name, 
+                env=self.env,
+                **self.agent_kwargs,
+            )
+
+        else:
+            if str(self.agent_filepath).endswith('.pkl'):
+                agent = read_pickle(self.agent_filepath)
+                agent.env = self.env
+
+            else:
+                agent = StableBaselines3Agent.load(self.agent_filepath, self.env)
+
+        return agent
+    
+    def __set_env(self) -> CityLearnEnv:
+        env = CityLearnEnv(self.schema, **self.env_kwargs)
+
+        for wrapper in self.wrappers:
+            wrapper_module = '.'.join(wrapper.split('.')[0:-1])
+            wrapper_name = wrapper.split('.')[-1]
+            wrapper_constructor = getattr(importlib.import_module(wrapper_module), wrapper_name)
+            env = wrapper_constructor(env)
+
+        return env
+    
+    def __reset(self):
+        self.env = self.__set_env()
+        self.agent = self.__set_agent()
+        self.__train_start_timestamp = None
+        self.__train_end_timestamp = None
+        self.__evaluation_start_timestamp = None
+        self.__evaluation_end_timestamp = None
+        self.__actions_list = None
+    
+    @classmethod
+    def evaluate(cls, evaluation_episode_time_steps: Tuple[int, int] = None, **kwargs):
+        kwargs['env_kwargs'] = {} if kwargs.get('env_kwargs') is None else kwargs['env_kwargs']
+
+        if evaluation_episode_time_steps is not None:
+            kwargs['env_kwargs']['episode_time_steps'] = [evaluation_episode_time_steps]
+        
+        else:
+            pass
+            
+
+        simulator = cls(**kwargs)
+        simulator.__evaluate()
+        filepath = os.path.join(simulator.output_directory, f'{simulator.simulation_id}-evaluation.json')
+        write_json(filepath, simulator.__get_evaluation_summary())
+
+    @classmethod
+    def train(cls, episodes: int = None, evaluate: bool = None, evaluation_episode_time_steps: Tuple[int, int] = None, save_agent: bool = None, **kwargs):
+        simulator = cls(**kwargs)
+        episodes = 1 if episodes is None else episodes
+        simulator.__train(episodes)
+        filepath = os.path.join(simulator.output_directory, f'{simulator.simulation_id}-train.json')
+        write_json(filepath, simulator.__get_training_summary())
+
+        if save_agent:
+            simulator.__save_agent()
+
+        else:
+            pass
+
+        if evaluate:
+            evaluation_episode_time_steps = [[
+                simulator.env.unwrapped.episode_tracker.simulation_start_time_step, 
+                simulator.env.unwrapped.episode_tracker.simulation_end_time_step
+            ]] if evaluation_episode_time_steps is None else [evaluation_episode_time_steps]
+            kwargs['env_kwargs'] = {} if kwargs.get('env_kwargs') is None else kwargs['env_kwargs']
+            kwargs['overwrite'] = False
+            kwargs['output_directory'] = simulator.output_directory
+            kwargs['env_kwargs']['episode_time_steps'] = evaluation_episode_time_steps
+            kwargs['simulation_id'] = f'{simulator.simulation_id}'
+            simulator.evaluate(**kwargs)
+
+        else:
+            pass
+
+    @staticmethod
+    def get_default_time_series_variables():
+        return Simulator.get_settings()['default_time_series_variables']
+
+    @staticmethod
+    def get_settings():
+        directory = os.path.join(os.path.join(os.path.dirname(__file__), 'misc'))
+        filepath = os.path.join(directory, 'settings.yaml')
+        settings = read_yaml(filepath)
+
+        return settings
 
 def main():
-    parser = argparse.ArgumentParser(prog='citylearn', formatter_class=argparse.ArgumentDefaultsHelpFormatter, description=('''
-        An open source Farama Foundation Gymnasium environment for the implementation of Multi-Agent Reinforcement 
-        Learning (RL) for building energy coordination and demand response in cities.'''
-    ))
-    parser.add_argument('--version', action='version', version='%(prog)s' + f' {__version__}')
+    parser = argparse.ArgumentParser(
+        prog='citylearn', 
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, description=(
+            'An open source Farama Foundation Gymnasium environment for benchmarking distributed energy resource '
+            'control algorithms to provide energy flexibility in a district of buildings. '
+            'Compatible with training and evaluating internally defined CityLearn agents in `citylearn.agents`, '
+            'user-defined agents that inherit from `citylearn.agents.base.Agent` and use the same interface as it, and agents '
+            'provided by stable-baselines3.'))
     subparsers = parser.add_subparsers(title='subcommands', required=True, dest='subcommands')
-    subparser_simulate = subparsers.add_parser('learn', description='Run simulation.', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    subparser_simulate.add_argument('schema', type=str, help='CityLearn dataset name or schema path.')
-    subparser_simulate.add_argument('-e', '--episodes', type=int, dest='episodes', default=None, help='Number of training episodes.')
-    subparser_simulate.add_argument('-f', '--filepath', type=Path, dest='filepath', default='citylearn_learning.pkl', help='Filepath to write model pickle object to upon completion.')
-    subparser_simulate.add_argument('-n', '--deterministic_finish', action='store_true', dest='deterministic_finish', help='Take deterministic actions in the final episode.')
-    subparser_simulate.add_argument("-l", "--logging_level", type=int, default=50, dest='logging_level', help='Logging level where increasing the level silences lower level information.')
-    subparser_simulate.set_defaults(func=__learn)
-    
-    args = parser.parse_args()
-    arg_spec = inspect.getfullargspec(args.func)
-    kwargs = {key:value for (key, value) in args._get_kwargs() 
-        if (key in arg_spec.args or (arg_spec.varkw is not None and key not in ['func','subcommands']))
-    }
-    args.func(**kwargs)
+
+    # run many simulations in parallel
+    subparser_run_work_order = subparsers.add_parser(
+        'run_work_order', 
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, 
+        help='Run commands in parallel. Useful for running many `citylearn simulate` commands in parallel.'
+    )
+    subparser_run_work_order.add_argument('work_order_filepath', type=Path, help=(
+        'Filepath to script containing list of commands to be run in parallel with each command defined on a new line.'))
+    subparser_run_work_order.add_argument('-w', '--max_workers', dest='max_workers', type=int, default=cpu_count(), help=(
+        'Maximum number of commands to run at a time. Default is the number of CPUs.'))
+    subparser_run_work_order.add_argument('-is', '--start_index', default=0, dest='start_index', type=int, help=(
+        'Line index of first command to execute. Commands above this index are not executed. '
+        'The default is to execute from the first line.'))
+    subparser_run_work_order.add_argument('-ie', '--end_index', dest='end_index', type=int, help=(
+        'Line index of last command to execute. Commands below this index are not exectued. '
+        'The default is to execute till the last line.'))
+    subparser_run_work_order.set_defaults(func=run_work_order)
+
+    # get names of datasets
+    subparser_datasets = subparsers.add_parser(
+        'list_datasets', 
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, 
+        help='Lists available dataset names that can be parsed as `schema` in `citylearn simulate schema`.'
+    )
+    subparser_datasets.set_defaults(func=DataSet.get_names)
+
+     # get default time series variables
+    subparser_time_series_variables = subparsers.add_parser(
+        'list_default_time_series_variables', 
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, 
+        help='Lists the default time series variables that will be reported and saved in a `JSON` file post-evaluation.'
+    )
+    subparser_time_series_variables.set_defaults(func=Simulator.get_default_time_series_variables)
+
+    # run one simulation
+    subparser_simulate = subparsers.add_parser(
+        'simulate', 
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, 
+        help='Train or evaluate a trained agent against an environment.'
+    )
+    subparser_simulate.add_argument('schema', type=str, help=(
+        'Name of CityLearn dataset or filepath to a schema. Call `citylearn list_datasets` to get list of valid dataset names.'))
+    subparser_simulate.add_argument('-a', '--agent_name', dest='agent_name', default='citylearn.agents.base.BaselineAgent', type=str, help=(
+        'Name path to agent. Currently only compatible with internally defined CityLearn agents in `citylearn.agents`, '
+        'user-defined agents that inherit from `citylearn.agents.base.Agent` and use the same interface as it, and agents '
+        'provided by stable-baselines3. To use stable-baselines3 agents, make sure to run `pip install stable-baselines3` '
+        'before using the `simulate command.`'))
+    subparser_simulate.add_argument('-ke', '--env_kwargs', dest='env_kwargs', type=json.loads, help=(
+        'Initialization parameters for`citylearn.citylearn.CityLearnEnv`.'))
+    subparser_simulate.add_argument('-ka', '--agent_kwargs', dest='agent_kwargs', type=json.loads, help=(
+        'Initialization parameters for agent class.'))
+    subparser_simulate.add_argument('-w', '--wrappers', dest='wrappers', type=str, nargs='+', help=(
+        'Name path to environment wrappers e.g., \'citylearn.wrappers.ClippedObservationWrapper\'.'))
+    subparser_simulate.add_argument('-tv', '--time_series_variables', dest='time_series_variables', type=str, nargs='+', help=(
+        'Names of building-level time series properties to be stored in the evaluation `JSON` post-evaluation. '
+        'Call `citylearn list_default_time_series_variables` to see the default variable in use.'))
+    subparser_simulate.add_argument('-sid', '--simulation_id', dest='simulation_id', type=str, help=(
+        'SImulation reference ID used in directory and file names.' ))
+    subparser_simulate.add_argument('-fa', '--agent_filepath', dest='agent_filepath', type=str, help=(
+        'Filepath to previously saved agent to use for training or evaluation.'))
+    subparser_simulate.add_argument('-d', '--output_directory', dest='output_directory', type=str, help=(
+        'Directory to save all simulation output to.'))
+    subparser_simulate.add_argument('-te', '--evaluation_episode_time_steps', dest='evaluation_episode_time_steps', type=int, nargs=2,
+        action='append', help=('Start and end time steps in data set to evaluate on otherwise, the agent is evaluated on entire dataset.'))
+    subparser_simulate.add_argument('-p', '--append', dest='overwrite', action='store_false', help=(
+        'Add to output for existing simulation with `simulation_id` i.e. do not overwrite.'))
+    subparser_simulate.add_argument('-rs', '--random_seed', dest='random_seed', type=int, help=(
+        'Random seed used during environment and agent initialization.'))
+    simulation_subparsers = subparser_simulate.add_subparsers(title='simulate subcommands', required=True, dest='subcommands')
+
+    # -> train an agent
+    subparser_train = simulation_subparsers.add_parser(
+        'train', 
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, 
+        help='Train an agent.'
+    )
+    subparser_train.add_argument('-e', '--episodes', dest='episodes', type=int, help='Number of training episodes/epochs.')
+    subparser_train.add_argument('--save_agent', dest='save_agent', action='store_true', help='Whether to save agent to disk at the end of training.')
+    subparser_train.add_argument('--evaluate', dest='evaluate', action='store_true', help=(
+        'Whether to run deterministic evaluation for one episode at the end of training.'))
+    subparser_train.set_defaults(func=Simulator.train)
+
+    # -> evaluate a trained agent
+    subparser_evaluate = simulation_subparsers.add_parser(
+        'evaluate', 
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, 
+        help='Deterministically evaluate an agent.'
+    )
+    subparser_evaluate.set_defaults(func=Simulator.evaluate)
+
+    argv = sys.argv
+    argv = ['sphinx-build' in a for a in argv]
+    sphinx = any(argv)
+
+    if not sphinx:
+        args = parser.parse_args()
+        arg_spec = inspect.getfullargspec(args.func)
+        kwargs = {key:value for (key, value) in args._get_kwargs() 
+            if (key in arg_spec.args or (arg_spec.varkw is not None and key not in ['func','subcommands']))
+        }
+
+        return args.func(**kwargs)
+    
+    else:
+        return parser
 
 if __name__ == '__main__':
     sys.exit(main())
\ No newline at end of file
diff --git a/citylearn/agents/base.py b/citylearn/agents/base.py
index 93389007..0e9d2c93 100644
--- a/citylearn/agents/base.py
+++ b/citylearn/agents/base.py
@@ -32,10 +32,16 @@ def __init__(self, env: CityLearnEnv, **kwargs: Any):
         super().__init__(
             seconds_per_time_step=self.env.seconds_per_time_step,
             random_seed=self.env.random_seed,
-            episode_tracker=env.episode_tracker,
+            episode_tracker=self.env.episode_tracker,
         )
         self.reset()
 
+    @property
+    def env(self) -> CityLearnEnv:
+        """CityLearn environment."""
+
+        return self.__env
+
     @property
     def observation_names(self) -> List[List[str]]:
         """Names of active observations that can be used to map observation values."""
@@ -81,6 +87,10 @@ def actions(self) -> List[List[List[Any]]]:
         """Action history/time series."""
 
         return self.__actions
+    
+    @env.setter
+    def env(self, env: CityLearnEnv):
+        self.__env = env
 
     @observation_names.setter
     def observation_names(self, observation_names: List[List[str]]):
@@ -250,7 +260,11 @@ class BaselineAgent(Agent):
     """
 
     def __init__(self, env: CityLearnEnv, **kwargs: Any):
-        super().__init__(self.__deactivate_actions(env), **kwargs)
+        super().__init__(env, **kwargs)
+
+    @Agent.env.setter
+    def env(self, env: CityLearnEnv):
+        Agent.env.fset(self, self.__deactivate_actions(env))
 
     def __deactivate_actions(self, env: CityLearnEnv) -> CityLearnEnv:
         for b in env.unwrapped.buildings:
diff --git a/citylearn/agents/sac.py b/citylearn/agents/sac.py
index a6c435e3..1bbe4263 100644
--- a/citylearn/agents/sac.py
+++ b/citylearn/agents/sac.py
@@ -1,4 +1,4 @@
-from typing import Any, List
+from typing import Any, List, Union
 import numpy as np
 import numpy.typing as npt
 
@@ -166,7 +166,7 @@ def update(self, observations: List[List[float]], actions: List[List[float]], re
     def predict(self, observations: List[List[float]], deterministic: bool = None):
         r"""Provide actions for current time step.
 
-        Will return randomly sampled actions from `action_space` if :attr:`end_exploration_time_step` >= :attr:`time_step` 
+        Will return randomly sampled actions from `action_space` if :attr:`end_exploration_time_step` <= :attr:`time_step` 
         else will use policy to sample actions.
 
         Parameters
@@ -285,13 +285,14 @@ class SACRBC(SAC):
         Other keyword arguments used to initialize super class.
     """
     
-    def __init__(self, env: CityLearnEnv, rbc: RBC = None, **kwargs: Any):
+    def __init__(self, env: CityLearnEnv, rbc: Union[RBC, str] = None, **kwargs: Any):
         super().__init__(env, **kwargs)
         self.__set_rbc(rbc, **kwargs)
 
     @property
     def rbc(self) -> RBC:
-        """:py:class:`citylearn.agents.rbc.RBC` or child class, used to select actions during exploration."""
+        """:py:class:`citylearn.agents.rbc.RBC` class child class or string path to an RBC 
+        class e.g. 'citylearn.agents.rbc.RBC', used to select actions during exploration."""
 
         return self.__rbc
     
@@ -302,6 +303,9 @@ def __set_rbc(self, rbc: RBC, **kwargs):
         elif isinstance(rbc, RBC):
             pass
 
+        elif isinstance(rbc, str):
+            rbc = self.env.load_agent(rbc, env=self.env, **kwargs)
+
         else:
             rbc = rbc(self.env, **kwargs)
         
diff --git a/citylearn/building.py b/citylearn/building.py
index b7617d32..dd110632 100644
--- a/citylearn/building.py
+++ b/citylearn/building.py
@@ -1,3 +1,4 @@
+import logging
 from typing import Any, List, Mapping, Tuple, Union
 from gymnasium import spaces
 import numpy as np
@@ -9,6 +10,8 @@
 from citylearn.power_outage import PowerOutage
 from citylearn.preprocessing import Normalize, PeriodicNormalization
 
+LOGGER = logging.getLogger()
+
 class Building(Environment):
     r"""Base class for building.
 
@@ -523,6 +526,26 @@ def non_shiftable_load(self) -> np.ndarray:
         """Electricity load that must be met by the grid, or `PV` and/or `electrical_storage` if available time series, in [kWh]."""
 
         return self.energy_simulation.non_shiftable_load[0:self.time_step + 1]
+    
+    @property
+    def cooling_device_cop(self) -> np.ndarray:
+        """Heat pump `cooling_device` coefficient of performance time series."""
+
+        return self.cooling_device.get_cop(self.weather.outdoor_dry_bulb_temperature, heating=False)[0:self.time_step + 1]
+    
+    @property
+    def heating_device_cop(self) -> np.ndarray:
+        """Heat pump `heating_device` coefficient of performance or electric heater `heating_device` static technical efficiency time series."""
+
+        return self.heating_device.get_cop(self.weather.outdoor_dry_bulb_temperature, heating=True)[0:self.time_step + 1] \
+            if isinstance(self.heating_device, HeatPump) else np.zeros(self.time_step + 1, dtype='float32')
+    
+    @property
+    def dhw_device_cop(self) -> np.ndarray:
+        """Heat pump `dhw_device` coefficient of performance or electric heater `dhw_device` static technical efficiency time series."""
+
+        return self.dhw_device.get_cop(self.weather.outdoor_dry_bulb_temperature, heating=True)[0:self.time_step + 1] \
+            if isinstance(self.dhw_device, HeatPump) else np.zeros(self.time_step + 1, dtype='float32')
 
     @property
     def solar_generation(self) -> np.ndarray:
@@ -691,7 +714,7 @@ def maximum_temperature_delta(self, maximum_temperature_delta: float):
 
     @demand_observation_limit_factor.setter
     def demand_observation_limit_factor(self, demand_observation_limit_factor: float):
-        self.__demand_observation_limit_factor = 1.15 if demand_observation_limit_factor is None else demand_observation_limit_factor
+        self.__demand_observation_limit_factor = 2.0 if demand_observation_limit_factor is None else demand_observation_limit_factor
 
         if hasattr(self, 'observation_space') and self.observation_space is not None:
             self.observation_space = self.estimate_observation_space(include_all=False, normalize=False)
@@ -759,7 +782,7 @@ def get_metadata(self) -> Mapping[str, Any]:
             'annual_solar_generation_estimate': self.pv.get_generation(self.energy_simulation.solar_generation).sum()/n_years,
         }
 
-    def observations(self, include_all: bool = None, normalize: bool = None, periodic_normalization: bool = None) -> Mapping[str, float]:
+    def observations(self, include_all: bool = None, normalize: bool = None, periodic_normalization: bool = None, check_limits: bool = None) -> Mapping[str, float]:
         r"""Observations at current time step.
 
         Parameters
@@ -770,6 +793,9 @@ def observations(self, include_all: bool = None, normalize: bool = None, periodi
             Whether to apply min-max normalization bounded between [0, 1].
         periodic_normalization: bool, default: False
             Whether to apply sine-cosine normalization to cyclic observations including hour, day_type and month.
+        check_limits: bool, default: False
+            Whether to check if observations are within observation space and if not, will send output to log describing 
+            out of bounds observations. Useful for agents that will fail if observations fall outside space e.g. RLlib agents.
 
         Returns
         -------
@@ -785,6 +811,7 @@ def observations(self, include_all: bool = None, normalize: bool = None, periodi
         normalize = False if normalize is None else normalize
         periodic_normalization = False if periodic_normalization is None else periodic_normalization
         include_all = False if include_all is None else include_all
+        check_limits = False if check_limits is None else check_limits
 
         observations = {}
         data = {
@@ -844,9 +871,34 @@ def observations(self, include_all: bool = None, normalize: bool = None, periodi
         unknown_observations = list(set(valid_observations).difference(observations.keys()))
         assert len(unknown_observations) == 0, f'Unknown observations: {unknown_observations}'
 
-        low_limit, high_limit = self.periodic_normalized_observation_space_limits
+        non_periodic_low_limit, non_periodic_high_limit = self.non_periodic_normalized_observation_space_limits
+        periodic_low_limit, periodic_high_limit = self.periodic_normalized_observation_space_limits
         periodic_observations = self.get_periodic_observation_metadata()
 
+        if check_limits:
+            for k in self.active_observations:
+                value = observations[k]
+                lower = non_periodic_low_limit[k]
+                upper = non_periodic_high_limit[k]
+                
+                if not lower <= value <= upper:
+                    report = {
+                        'Building': self.name, 
+                        'episode': self.episode_tracker.episode, 
+                        'time_step': f'{self.time_step + 1}/{self.episode_tracker.episode_time_steps}', 
+                        'observation': k, 
+                        'value': value, 
+                        'lower': lower, 
+                        'upper': upper
+                    }
+                    LOGGER.debug(f'Observation outside space limit: {report}')
+
+                else:
+                    pass
+            
+            else:
+                pass
+
         if periodic_normalization:
             observations_copy = {k: v for k, v in observations.items()}
             observations = {}
@@ -867,8 +919,8 @@ def observations(self, include_all: bool = None, normalize: bool = None, periodi
             nm = Normalize(0.0, 1.0)
 
             for k, v in observations.items():
-                nm.x_min = low_limit[k]
-                nm.x_max = high_limit[k]
+                nm.x_min = periodic_low_limit[k]
+                nm.x_max = periodic_high_limit[k]
                 observations[k] = v*nm
 
         else:
@@ -1408,50 +1460,31 @@ def estimate_action_space(self) -> spaces.Box:
                 low_limit.append(0.0)
                 high_limit.append(1.0)
             
-            elif key == 'electrical_storage':
-                limit = self.electrical_storage.nominal_power/max(self.electrical_storage.capacity, ZERO_DIVISION_PLACEHOLDER)
-                low_limit.append(-limit)
-                high_limit.append(limit)
-            
-            else:
-                if key == 'cooling_storage':
-                    capacity = self.cooling_storage.capacity
-                    cooling_demand = self.energy_simulation.__getattr__(
-                        'cooling_demand', 
-                        start_time_step=self.episode_tracker.simulation_start_time_step, 
-                        end_time_step=self.episode_tracker.simulation_end_time_step
-                    )
-                    maximum_demand = cooling_demand.max()
+            elif 'storage' in key:
+                if key == 'electrical_storage':
+                    limit = self.electrical_storage.nominal_power/max(self.electrical_storage.capacity, ZERO_DIVISION_PLACEHOLDER)
                 
-                elif key == 'heating_storage':
-                    capacity = self.heating_storage.capacity
-                    heating_demand = self.energy_simulation.__getattr__(
-                        'heating_demand', 
-                        start_time_step=self.episode_tracker.simulation_start_time_step, 
-                        end_time_step=self.episode_tracker.simulation_end_time_step
-                    )
-                    maximum_demand = heating_demand.max()
-
-                elif key == 'dhw_storage':
-                    capacity = self.dhw_storage.capacity
-                    dhw_demand = self.energy_simulation.__getattr__(
-                        'dhw_demand', 
-                        start_time_step=self.episode_tracker.simulation_start_time_step, 
-                        end_time_step=self.episode_tracker.simulation_end_time_step
-                    )
-                    maximum_demand = dhw_demand.max()
-
                 else:
-                    raise Exception(f'Unknown action: {key}')
-
-                maximum_demand_ratio = maximum_demand/max(capacity, ZERO_DIVISION_PLACEHOLDER)
-
-                try:
-                    low_limit.append(max(-maximum_demand_ratio, -1.0))
-                    high_limit.append(min(maximum_demand_ratio, 1.0))
-                except ZeroDivisionError:
-                    low_limit.append(-1.0)
-                    high_limit.append(1.0)
+                    if key == 'cooling_storage':
+                        capacity = self.cooling_storage.capacity
+                        power = self.cooling_device.nominal_power
+                    
+                    elif key == 'heating_storage':
+                        capacity = self.heating_storage.capacity
+                        power = self.heating_device.nominal_power
+
+                    elif key == 'dhw_storage':
+                        capacity = self.dhw_storage.capacity
+                        power = self.dhw_device.nominal_power
+
+                    else:
+                        raise Exception(f'Unknown action: {key}')
+
+                    limit = power/max(capacity, ZERO_DIVISION_PLACEHOLDER)
+                    
+                limit = min(limit, 1.0)
+                low_limit.append(-limit)
+                high_limit.append(limit)
  
         return spaces.Box(low=np.array(low_limit, dtype='float32'), high=np.array(high_limit, dtype='float32'))
 
diff --git a/citylearn/citylearn.py b/citylearn/citylearn.py
index 26532443..cc5d7b78 100755
--- a/citylearn/citylearn.py
+++ b/citylearn/citylearn.py
@@ -71,12 +71,33 @@ class CityLearnEnv(Environment, Env):
         True if episode splits are to be selected at random during training otherwise, False to select sequentially.
     seconds_per_time_step: float
         Number of seconds in 1 `time_step` and must be set to >= 1.
-    reward_function: RewardFunction, optional
-        Reward function class instance. If provided, will override :code:`reward_function` definition in schema.
+    reward_function: Union[RewardFunction, str], optional
+        Reward function class instance or path to function class e.g. 'citylearn.reward_function.IndependentSACReward'.
+        If provided, will override :code:`reward_function` definition in schema.
+    reward_function_kwargs: Mapping[str, Any], optional
+        Parameters to be parsed to :py:attr:`reward_function` at intialization.
     central_agent: bool, optional
         Expect 1 central agent to control all buildings.
     shared_observations: List[str], optional
         Names of common observations across all buildings i.e. observations that have the same value irrespective of the building.
+    active_observations: Union[List[str], List[List[str]]], optional
+        List of observations to be made available in the buildings. Can be specified for all buildings in a :code:`List[str]` or for  
+        each building independently in a :code:`List[List[str]]`. Will override the observations defined in the :code:`schema`.
+    inactive_observations: Union[List[str], List[List[str]]], optional
+        List of observations to be made unavailable in the buildings. Can be specified for all buildings in a :code:`List[str]` or for  
+        each building independently in a :code:`List[List[str]]`. Will override the observations defined in the :code:`schema`.
+    active_actions: Union[List[str], List[List[str]]], optional
+        List of actions to be made available in the buildings. Can be specified for all buildings in a :code:`List[str]` or for  
+        each building independently in a :code:`List[List[str]]`. Will override the actions defined in the :code:`schema`.
+    inactive_actions: Union[List[str], List[List[str]]], optional
+        List of actions to be made unavailable in the buildings. Can be specified for all buildings in a :code:`List[str]` or for  
+        each building independently in a :code:`List[List[str]]`. Will override the actions defined in the :code:`schema`.
+    simulate_power_outage: Union[bool, List[bool]]
+        Whether to simulate power outages. Can be specified for all buildings as single :code:`bool` or for  
+        each building independently in a :code:`List[bool]`. Will override power outage defined in the :code:`schema`.
+    solar_generation: Union[bool, List[bool]]
+        Wehther to allow solar generation. Can be specified for all buildings as single :code:`bool` or for  
+        each building independently in a :code:`List[bool]`. Will override :code:`pv` defined in the :code:`schema`.
     random_seed: int, optional
         Pseudorandom number generator seed for repeatable results.
 
@@ -93,8 +114,10 @@ class CityLearnEnv(Environment, Env):
     def __init__(self, 
         schema: Union[str, Path, Mapping[str, Any]], root_directory: Union[str, Path] = None, buildings: Union[List[Building], List[str], List[int]] = None, 
         simulation_start_time_step: int = None, simulation_end_time_step: int = None, episode_time_steps: Union[int, List[Tuple[int, int]]] = None, rolling_episode_split: bool = None, 
-        random_episode_split: bool = None, seconds_per_time_step: float = None, reward_function: RewardFunction = None, 
-        central_agent: bool = None, shared_observations: List[str] = None, random_seed: int = None, **kwargs: Any
+        random_episode_split: bool = None, seconds_per_time_step: float = None, reward_function: Union[RewardFunction, str] = None, reward_function_kwargs: Mapping[str, Any] = None, 
+        central_agent: bool = None, shared_observations: List[str] = None, active_observations: Union[List[str], List[List[str]]] = None, 
+        inactive_observations: Union[List[str], List[List[str]]] = None, active_actions: Union[List[str], List[List[str]]] = None, 
+        inactive_actions: Union[List[str], List[List[str]]] = None, simulate_power_outage: bool = None, solar_generation: bool = None, random_seed: int = None, **kwargs: Any
     ):
         self.schema = schema
         self.__rewards = None
@@ -111,8 +134,15 @@ def __init__(self,
                 random_episode=random_episode_split,
                 seconds_per_time_step=seconds_per_time_step,
                 reward_function=reward_function,
+                reward_function_kwargs=reward_function_kwargs,
                 central_agent=central_agent,
                 shared_observations=shared_observations,
+                active_observations=active_observations,
+                inactive_observations=inactive_observations,
+                active_actions=active_actions,
+                inactive_actions=inactive_actions,
+                simulate_power_outage=simulate_power_outage,
+                solar_generation=solar_generation,
                 random_seed=self.random_seed,
             )
         self.root_directory = root_directory
@@ -319,7 +349,7 @@ def observations(self) -> List[List[float]]:
             shared_observations = []
 
             for i, b in enumerate(self.buildings):
-                for k, v in b.observations(normalize=False, periodic_normalization=False).items():
+                for k, v in b.observations(normalize=False, periodic_normalization=False, check_limits=True).items():
                     if i == 0 or k not in self.shared_observations or k not in shared_observations:
                         observations.append(v)
                     
@@ -335,7 +365,7 @@ def observations(self) -> List[List[float]]:
             observations = [observations]
         
         else:
-            observations = [list(b.observations().values()) for b in self.buildings]
+            observations = [list(b.observations(normalize=False, periodic_normalization=False, check_limits=True).values()) for b in self.buildings]
         
         return observations
 
@@ -1155,27 +1185,54 @@ def update_variables(self):
         # net electriciy consumption emission
         self.__net_electricity_consumption_emission.append(sum([b.net_electricity_consumption_emission[self.time_step] for b in self.buildings]))
 
-    def load_agent(self) -> 'citylearn.agents.base.Agent':
+    def load_agent(self, agent: Union[str, 'citylearn.agents.base.Agent'] = None, **kwargs) -> Union[Any, 'citylearn.agents.base.Agent']:
         """Return :class:`Agent` or sub class object as defined by the `schema`.
 
         Parameters
         ----------
+        agent: Union[str, 'citylearn.agents.base.Agent], optional
+            Agent class or string describing path to agent class, e.g. 'citylearn.agents.base.BaselineAgent'.
+            If a value is not provided, defaults to the agent defined in the schema:agent:type.
+
         **kwargs : dict
-            Parameters to override schema definitions. See :py:class:`citylearn.citylearn.CityLearnEnv` initialization parameters for valid kwargs.
+            Agent initialization attributes. For most agents e.g. CityLearn and Stable-Baselines3 agents, 
+            an intialized :py:attr:`env` must be parsed to the agent :py:meth:`init` function.
         
         Returns
         -------
-        agents: Agent
-            Simulation agent(s) for `citylearn_env.buildings` energy storage charging/discharging management.
+        agent: Agent
+            Initialized agent.
         """
 
-        agent_type = self.schema['agent']['type']
+        # set agent class
+        if agent is not None:
+            agent_type = agent
+
+            if not isinstance(agent_type, str):
+                agent_type = [agent_type.__module__] + [agent_type.__name__]
+                agent_type = '.'.join(agent_type)
+
+            else:
+                pass
+        
+        # set agent init attributes
+        else:
+            agent_type = self.schema['agent']['type']
+        
+        if kwargs is not None and len(kwargs) > 0:
+            agent_attributes = kwargs
+
+        elif agent is None:
+            agent_attributes = self.schema['agent'].get('attributes', {})
+
+        else:
+            agent_attributes = None
+        
         agent_module = '.'.join(agent_type.split('.')[0:-1])
         agent_name = agent_type.split('.')[-1]
         agent_constructor = getattr(importlib.import_module(agent_module), agent_name)
-        agent_attributes = self.schema['agent'].get('attributes', {})
-        agent_attributes = {'env': self, **agent_attributes}
-        agent = agent_constructor(**agent_attributes)
+        agent = agent_constructor() if agent_attributes is None else agent_constructor(**agent_attributes)
+
         return agent
 
     def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int, List[Tuple[int, int]]], bool, bool, float, RewardFunction, bool, List[str], EpisodeTracker]:
@@ -1209,12 +1266,15 @@ def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int,
             self.schema = read_json(self.schema)
             self.schema['root_directory'] = os.path.split(schema_filepath.absolute())[0] if self.schema['root_directory'] is None\
                 else self.schema['root_directory']
+        
         elif isinstance(self.schema, str) and self.schema in DataSet.get_names():
             self.schema = DataSet.get_schema(self.schema)
             self.schema['root_directory'] = '' if self.schema['root_directory'] is None else self.schema['root_directory']
+        
         elif isinstance(self.schema, dict):
             self.schema = deepcopy(self.schema)
             self.schema['root_directory'] = '' if self.schema['root_directory'] is None else self.schema['root_directory']
+        
         else:
             raise UnknownSchemaError()
 
@@ -1258,7 +1318,7 @@ def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int,
         else:
             buildings_to_include = [b for b in buildings_to_include if self.schema['buildings'][b]['include']]
 
-        for building_name in buildings_to_include:
+        for i, building_name in enumerate(buildings_to_include):
             building_schema = self.schema['buildings'][building_name]
             # data
             energy_simulation = pd.read_csv(os.path.join(root_directory,building_schema['energy_simulation']))
@@ -1270,6 +1330,7 @@ def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int,
                 carbon_intensity = pd.read_csv(os.path.join(root_directory,building_schema['carbon_intensity']))
                 carbon_intensity = carbon_intensity['kg_CO2/kWh'].tolist()
                 carbon_intensity = CarbonIntensity(carbon_intensity)
+            
             else:
                 carbon_intensity = None
 
@@ -1279,11 +1340,51 @@ def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int,
             else:
                 pricing = None
                 
-            # observation and action metadata
-            inactive_observations = [] if building_schema.get('inactive_observations', None) is None else building_schema['inactive_observations']
-            inactive_actions = [] if building_schema.get('inactive_actions', None) is None else building_schema['inactive_actions']
-            observation_metadata = {k: False if k in inactive_observations else v['active'] for k, v in observations.items()}
-            action_metadata = {k: False if k in inactive_actions else v['active'] for k, v in actions.items()}
+            # observation metadata
+            observation_metadata = {k: v['active'] for k, v in observations.items()}
+
+            if kwargs.get('active_observations') is not None:
+                active_observations = kwargs['active_observations']
+                active_observations = active_observations[i] if isinstance(active_observations[0], list) else active_observations
+                observation_metadata = {k: True if k in active_observations else False for k in observation_metadata}
+            
+            else:
+                pass
+
+            if kwargs.get('inactive_observations') is not None:
+                inactive_observations = kwargs['inactive_observations']
+                inactive_observations = inactive_observations[i] if isinstance(inactive_observations[0], list) else inactive_observations
+
+            elif building_schema.get('inactive_observations') is not None:
+                inactive_observations = building_schema['inactive_observations']
+
+            else:
+                inactive_observations = []
+
+            observation_metadata = {k: False if k in inactive_observations else v for k, v in observation_metadata.items()}
+
+            # action metadata
+            action_metadata = {k: v['active'] for k, v in actions.items()}
+
+            if kwargs.get('active_actions') is not None:
+                active_actions = kwargs['active_actions']
+                active_actions = active_actions[i] if isinstance(active_actions[0], list) else active_actions
+                action_metadata = {k: True if k in active_actions else False for k in action_metadata}
+            
+            else:
+                pass
+
+            if kwargs.get('inactive_actions') is not None:
+                inactive_actions = kwargs['inactive_actions']
+                inactive_actions = inactive_actions[i] if isinstance(inactive_actions[0], list) else inactive_actions
+
+            elif building_schema.get('inactive_actions') is not None:
+                inactive_actions = building_schema['inactive_actions']
+
+            else:
+                inactive_actions = []
+
+            action_metadata = {k: False if k in inactive_actions else v for k, v in action_metadata.items()}
 
             # construct building
             building_type = 'citylearn.citylearn.Building' if building_schema.get('type', None) is None else building_schema['type']
@@ -1311,7 +1412,9 @@ def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int,
 
             # set power outage model
             building_schema_power_outage = building_schema.get('power_outage', {})
-            simulate_power_outage = building_schema_power_outage.get('simulate_power_outage')
+            simulate_power_outage = kwargs.get('simulate_power_outage')
+            simulate_power_outage = building_schema_power_outage.get('simulate_power_outage') if simulate_power_outage is None else simulate_power_outage
+            simulate_power_outage = simulate_power_outage[i] if isinstance(simulate_power_outage, list) else simulate_power_outage
             stochastic_power_outage = building_schema_power_outage.get('stochastic_power_outage')
 
             if building_schema_power_outage.get('stochastic_power_outage_model', None) is not None:
@@ -1357,10 +1460,17 @@ def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int,
                 'dhw_device': {'autosizer': building.autosize_dhw_device}, 
                 'pv': {'autosizer': building.autosize_pv}
             }
+            solar_generation = kwargs.get('solar_generation')
+            solar_generation = True if solar_generation is None else solar_generation
+            solar_generation = solar_generation[i] if isinstance(solar_generation, list) else solar_generation
 
             for name in device_metadata:
                 if building_schema.get(name, None) is None:
                     device = None
+                
+                elif name == 'pv' and not solar_generation:
+                    device = None
+                
                 else:
                     device_type = building_schema[name]['type']
                     device_module = '.'.join(device_type.split('.')[0:-1])
@@ -1376,6 +1486,7 @@ def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int,
                         autosizer = device_metadata[name]['autosizer']
                         autosize_kwargs = {} if building_schema[name].get('autosize_attributes', None) is None else building_schema[name]['autosize_attributes']
                         autosizer(**autosize_kwargs)
+                    
                     else:
                         pass
             
@@ -1385,17 +1496,31 @@ def _load(self, **kwargs) -> Tuple[Union[Path, str], List[Building], Union[int,
         
         buildings = list(buildings)
 
+        # set reward function
         if kwargs.get('reward_function') is not None:
-            reward_function_constructor = kwargs['reward_function']
-            reward_function = reward_function_constructor(None)
+            reward_function_type = kwargs['reward_function']
+
+            if not isinstance(reward_function_type, str):
+                reward_function_type = [reward_function_type.__module__] + [reward_function_type.__name__]
+                reward_function_type = '.'.join(reward_function_type)
+
+            else:
+                pass
+
         else:
             reward_function_type = self.schema['reward_function']['type']
-            reward_function_attributes = self.schema['reward_function'].get('attributes',None)
+
+        if kwargs.get('reward_function_kwargs') is not None:
+            reward_function_attributes = kwargs['reward_function_kwargs']
+        
+        else:
+            reward_function_attributes = self.schema['reward_function'].get('attributes', None)
             reward_function_attributes = {} if reward_function_attributes is None else reward_function_attributes
-            reward_function_module = '.'.join(reward_function_type.split('.')[0:-1])
-            reward_function_name = reward_function_type.split('.')[-1]
-            reward_function_constructor = getattr(importlib.import_module(reward_function_module), reward_function_name)
-            reward_function = reward_function_constructor(None, **reward_function_attributes)
+        
+        reward_function_module = '.'.join(reward_function_type.split('.')[0:-1])
+        reward_function_name = reward_function_type.split('.')[-1]
+        reward_function_constructor = getattr(importlib.import_module(reward_function_module), reward_function_name)
+        reward_function = reward_function_constructor(None, **reward_function_attributes)
 
         return (
             root_directory, buildings, episode_time_steps, rolling_episode_split, random_episode_split, 
diff --git a/citylearn/data.py b/citylearn/data.py
index 3bb28345..a47e7e34 100644
--- a/citylearn/data.py
+++ b/citylearn/data.py
@@ -11,7 +11,7 @@
 class DataSet:
     """CityLearn input data set and schema class."""
     
-    __ROOT_DIRECTORY = os.path.join(os.path.dirname(__file__),'data')
+    __ROOT_DIRECTORY = os.path.join(os.path.dirname(__file__), 'data')
 
     @staticmethod
     def get_names() -> List[str]:
diff --git a/citylearn/misc/settings.yaml b/citylearn/misc/settings.yaml
new file mode 100644
index 00000000..a037e9fc
--- /dev/null
+++ b/citylearn/misc/settings.yaml
@@ -0,0 +1,50 @@
+---
+default_time_series_variables:
+  - net_electricity_consumption
+  - net_electricity_consumption_without_storage
+  - net_electricity_consumption_without_storage_and_partial_load
+  - net_electricity_consumption_without_storage_and_partial_load_and_pv
+  - solar_generation
+  - cooling_electricity_consumption
+  - heating_electricity_consumption
+  - dhw_electricity_consumption
+  - non_shiftable_load_electricity_consumption
+  - cooling_storage_electricity_consumption
+  - heating_storage_electricity_consumption
+  - dhw_storage_electricity_consumption
+  - electrical_storage_electricity_consumption
+  - cooling_demand
+  - cooling_demand_without_partial_load
+  - heating_demand
+  - heating_demand_without_partial_load
+  - dhw_demand
+  - non_shiftable_load
+  - energy_from_cooling_device
+  - energy_from_heating_device
+  - energy_from_dhw_device
+  - energy_from_cooling_storage
+  - energy_from_heating_storage
+  - energy_from_dhw_storage
+  - energy_from_electrical_storage
+  - energy_from_cooling_device_to_cooling_storage
+  - energy_from_heating_device_to_heating_storage
+  - energy_from_dhw_device_to_dhw_storage
+  - energy_to_non_shiftable_load
+  - energy_to_electrical_storage
+  - cooling_device_cop
+  - heating_device_cop
+  - dhw_device_cop
+  - cooling_storage.soc
+  - heating_storage.soc
+  - dhw_storage.soc
+  - electrical_storage.soc
+  - indoor_dry_bulb_temperature
+  - indoor_dry_bulb_temperature_without_partial_load
+  - energy_simulation.indoor_dry_bulb_temperature_set_point
+  - energy_simulation.occupant_count
+  - pricing.electricity_pricing
+  - carbon_intensity.carbon_intensity
+  - energy_simulation.power_outage
+  - weather.outdoor_dry_bulb_temperature
+  - weather.outdoor_relative_humidity
+...
\ No newline at end of file
diff --git a/citylearn/utilities.py b/citylearn/utilities.py
index ee55afac..822e7b14 100644
--- a/citylearn/utilities.py
+++ b/citylearn/utilities.py
@@ -1,7 +1,10 @@
+import pickle
+from typing import Any
 import simplejson as json
+import yaml
 
-def read_json(filepath: str, **kwargs):
-    """Return json document as dictionary.
+def read_json(filepath: str, **kwargs) -> dict:
+    """Return JSON document as dictionary.
     
     Parameters
     ----------
@@ -20,12 +23,12 @@ def read_json(filepath: str, **kwargs):
     """
 
     with open(filepath) as f:
-        json_file = json.load(f,**kwargs)
+        json_file = json.load(f, **kwargs)
 
     return json_file
 
 def write_json(filepath: str, dictionary: dict, **kwargs):
-    """Write dictionary to json file. Returns boolen value of operation success. 
+    """Write dictionary to JSON file.
     
     Parameters
     ----------
@@ -40,6 +43,90 @@ def write_json(filepath: str, dictionary: dict, **kwargs):
         Other infrequently used keyword arguments to be parsed to `simplejson.dump`.
     """
 
-    kwargs = {'ignore_nan':True,'sort_keys':False,'default':str,'indent':2,**kwargs}
+    kwargs = {'ignore_nan': True, 'sort_keys': False, 'default': str, 'indent': 4, **kwargs}
+    
     with open(filepath,'w') as f:
-        json.dump(dictionary,f,**kwargs)
\ No newline at end of file
+        json.dump(dictionary, f, **kwargs)
+
+def read_yaml(filepath: str) -> dict:
+    """Return YAML document as dictionary.
+    
+    Parameters
+    ----------
+    filepath : str
+       pathname of YAML document.
+       
+    Returns
+    -------
+    dict
+        YAML document converted to dictionary.
+    """
+
+    with open(filepath, 'r') as f:
+        data = yaml.safe_load(f)
+
+    return data
+
+def write_yaml(filepath: str, dictionary: dict, **kwargs):
+    """Write dictionary to YAML file. 
+    
+    Parameters
+    ----------
+    filepath : str
+        pathname of YAML document.
+    dictionary: dict
+        dictionary to convert to YAML.
+
+    Other Parameters
+    ----------------
+    **kwargs : dict
+        Other infrequently used keyword arguments to be parsed to `yaml.safe_dump`.
+    """
+
+    kwargs = {'sort_keys': False, 'indent': 4, **kwargs}
+    
+    with open(filepath, 'w') as f:
+        yaml.safe_dump(dictionary, f, **kwargs)
+
+def read_pickle(filepath: str, **kwargs) -> Any:
+    """Return pickle file as some Python class object.
+    
+    Parameters
+    ----------
+    filepath : str
+       pathname of pickle file.
+
+    Other Parameters
+    ----------------
+    **kwargs : dict
+        Other infrequently used keyword arguments to be parsed to `pickle.load`.
+       
+    Returns
+    -------
+    Any
+        Pickle file as a Python object.
+    """
+
+    with (open(filepath, 'rb')) as f:
+        data = pickle.load(f, **kwargs)
+
+    return data
+
+def write_pickle(filepath: str, data: Any, **kwargs):
+    """Write Python object to pickle file. 
+    
+    Parameters
+    ----------
+    filepath : str
+        pathname of pickle document.
+    data: dict
+        object to convert to pickle.
+
+    Other Parameters
+    ----------------
+    **kwargs : dict
+        Other infrequently used keyword arguments to be parsed to `pickle.dump`.
+    """
+
+    with open(filepath, 'wb') as f:
+        pickle.dump(data, f, **kwargs)
\ No newline at end of file
diff --git a/citylearn/wrappers.py b/citylearn/wrappers.py
index 890ebbb7..3ff599e8 100644
--- a/citylearn/wrappers.py
+++ b/citylearn/wrappers.py
@@ -1,10 +1,41 @@
 import itertools
-from typing import List, Mapping
-from gymnasium import ActionWrapper, ObservationWrapper, RewardWrapper, spaces, Wrapper
+from typing import Any, List, Mapping, Tuple
+from gymnasium import ActionWrapper, Env, ObservationWrapper, RewardWrapper, spaces, Wrapper
 import numpy as np
+import pandas as pd
+
+try:
+    from ray.rllib.env import MultiAgentEnv
+except (ModuleNotFoundError, ImportError) as e:
+    from gymnasium import Env as MultiAgentEnv
+
 from citylearn.citylearn import CityLearnEnv
 from citylearn.building import Building
 
+class ClippedObservationWrapper(ObservationWrapper):
+    """Wrapper for observations min-max and periodic normalization.
+    
+    Observations are clipped to be within the observation space limits.
+
+    Parameters
+    ----------
+    env: CityLearnEnv
+        CityLearn environment.
+    """
+
+    def __init__(self, env: CityLearnEnv) -> None:
+        super().__init__(env)
+        self.env: CityLearnEnv
+
+    def observation(self, observations: List[List[float]]) -> List[List[float]]:
+        """Returns normalized observations."""
+
+        for i, (o, s) in enumerate(zip(observations, self.observation_space)):
+            for j, (o_, l, u) in enumerate(zip(o, s.low, s.high)):
+                observations[i][j] = min(max(o_, l), u)
+
+        return observations
+
 class NormalizedObservationWrapper(ObservationWrapper):
     """Wrapper for observations min-max and periodic normalization.
     
@@ -460,8 +491,8 @@ def set_combinations(self) -> List[List[int]]:
 class TabularQLearningWrapper(Wrapper):
     """Wrapper for :py:class:`citylearn.agents.q_learning.TabularQLearning` agent.
 
-    Wraps `env` in :py:class:`citylearn.wrappers.DiscreteObservationWrapper` and :py:class:`citylearn.wrappers.DiscreteActionWrapper`.
-    
+    Discretizes observation and action spaces.
+
     Parameters
     ----------
     env: CityLearnEnv
@@ -572,9 +603,9 @@ def reward(self, reward: List[float]) -> float:
 class StableBaselines3Wrapper(Wrapper):
     """Wrapper for :code:`stable-baselines3` algorithms.
 
-    Wraps `env` in :py:class:`citylearn.wrappers.StableBaselines3ObservationWrapper`,
-    :py:class:`citylearn.wrappers.StableBaselines3ActionWrapper`
-    and :py:class:`citylearn.wrappers.StableBaselines3RewardWrapper`.
+    Wraps observations so that they are returned in a 1-dimensional numpy array.
+    Wraps actions so that they are returned in a 1-dimensional numpy array.
+    Wraps rewards so that it is returned as float value.
     
     Parameters
     ----------
@@ -587,4 +618,240 @@ def __init__(self, env: CityLearnEnv):
         env = StableBaselines3RewardWrapper(env)
         env = StableBaselines3ObservationWrapper(env)
         super().__init__(env)
-        self.env: CityLearnEnv
\ No newline at end of file
+        self.env: CityLearnEnv
+
+class RLlibSingleAgentWrapper(StableBaselines3Wrapper):
+    """Wrapper for :code:`RLlib` single-agent algorithms.
+
+    Uses the same wrapper as :code:`stable-baselines3` by wrapping
+    `env` in :py:class:`citylearn.wrappers.StableBaselines3Wrapper`.
+    
+    Parameters
+    ----------
+    env_config: Mapping[str, Any]
+        Dictionary providing initialization parameters for the environment.
+        Must contain `env_kwargs` as a key where `env_kwargs` is a `dict` used to 
+        initialize :py:class:`citylearn.citylearn.CityLearnEnv`. Thus it must 
+        contain all with positional arguments needed for intialization and 
+        optionally contain optional  intialization arguments. `env_config` can also 
+        contain a `wrappers` key that is a list of :py:mod:`citylearn.wrappers` 
+        classes to wrap :py:class:`citylearn.citylearn.CityLearnEnv` with. Wrapping
+        with :py:class:`citylearn.wrappers.ClippedObservationWrapper` is recommended
+        to avoid having the simulation terminating prematurely with an error due to
+        out of bound observations relative to the observation space.
+
+    Notes
+    -----
+    This wrapper is only compatible with an environment where 
+    :py:attr:`citylearn.citylearn.central_agent`=`True` and will initialize the 
+    environment as such, overriding any value for `central_agent` in `env_kwargs`.
+    """
+
+    def __init__(self, env_config: Mapping[str, Any]):
+        env_kwargs = env_config['env_kwargs']
+        env_kwargs['central_agent'] = True
+        assert 'schema' in env_kwargs, 'missing schema key in env_kwargs.'
+        env = CityLearnEnv(**env_kwargs)
+
+        wrappers = env_config.get('wrappers')
+        wrappers = [] if wrappers is None else wrappers
+
+        for w in wrappers:
+            env = w(env)
+
+        super().__init__(env)
+
+class RLlibMultiAgentObservationWrapper(ObservationWrapper):
+    """Observation wrapper for :code:`RLlib` multi-agent algorithms.
+
+    Wraps observation space and observations so that they are returned 
+    as :py:class:`gymnasium.spaces.Dict` and `dict` objects respectively.
+    The keys in these objects correspond to the agent IDs i.e., 
+    policy IDs in the multi-agent.
+    
+    Parameters
+    ----------
+    env: CityLearnEnv
+        CityLearn environment.
+    """
+
+    def __init__(self, env: CityLearnEnv):
+        assert not env.central_agent, 'RLlibMultiAgentObservationWrapper is'\
+            ' compatible only when env.central_agent = False.'\
+                ' First set env.central_agent = False to use this wrapper.'
+
+        super().__init__(env)
+        self.env: CityLearnEnv
+
+    @property
+    def observation_space(self) -> spaces.Dict:
+        """Parses observation space into a :py:class:`gymnasium.spaces.Dict`."""
+
+        return spaces.Dict({f'agent_{i}': s for i, s in enumerate(self.env.observation_space)})
+
+    def observation(
+        self, observations: List[List[float]]
+    ) -> Mapping[str, np.ndarray]:
+        """Parses observation into a dictionary."""
+
+        return {f'agent_{i}': np.array(o, dtype='float32') for i, o in enumerate(observations)}
+
+class RLlibMultiAgentActionWrapper(ActionWrapper):
+    """Action wrapper for :code:`RLlib` multi-agent algorithms.
+
+    Wraps action space so that it is returned as :py:class:`gymnasium.spaces.Dict`. 
+    The keys correspond to the agent IDs i.e., policy IDs in the multi-agent. 
+    Also converts agent actions from `dict` to  data structure need by 
+    :py:meth:`citylearn.citylearn.CityLearnEnv.step`.
+    
+    Parameters
+    ----------
+    env: CityLearnEnv
+        CityLearn environment.
+    """
+
+    def __init__(self, env: CityLearnEnv):
+        assert not env.central_agent, 'RLlibMultiAgentActionWrapper is'\
+            ' compatible only when env.central_agent = False.'\
+                ' First set env.central_agent = False to use this wrapper.'
+
+        super().__init__(env)
+        self.env: CityLearnEnv
+
+    @property
+    def action_space(self) -> spaces.Dict:
+        """Parses action space into a :py:class:`gymnasium.spaces.Dict`."""
+
+        return spaces.Dict({f'agent_{i}': s for i, s in enumerate(self.env.action_space)})
+
+    def action(self, actions: Mapping[str, np.ndarray]) -> List[List[float]]:
+        """Parses actions into data structure for :py:meth:`citylearn.citylearn.CityLearnEnv.step`."""
+
+        return [list(v) for v in actions.values()]
+
+class RLlibMultiAgentRewardWrapper(RewardWrapper):
+    """Action wrapper for :code:`RLlib` multi-agent algorithms.
+
+    Wraps action space so that it is returned as a `dict` mapping agent IDs to reward values.
+    
+    Parameters
+    ----------
+    env: CityLearnEnv
+        CityLearn environment.
+    """
+
+    def __init__(self, env: CityLearnEnv):
+        assert not env.central_agent, 'RLlibMultiAgentRewardWrapper is'\
+            ' compatible only when env.central_agent = False.'\
+                ' First set env.central_agent = False to use this wrapper.'
+
+        super().__init__(env)
+        self.env: CityLearnEnv
+
+    def reward(self, reward: List[float]) -> Mapping[str, float]:
+        """Parses reward into a `dict`."""
+
+        return {f'agent_{i}': r for i, r in enumerate(reward)}
+
+class RLlibMultiAgentEnv(MultiAgentEnv):
+    """Wrapper for :code:`RLlib` multi-agent algorithms.
+
+    Converts, observation and action spaces to :py:class:`gymnasium.spaces.Dict`.
+    Also converts `observations`, `actions`, `rewards`, `terminated`, and `truncated`
+    to dictionaries where necessary. The dictionary keys correspond to the agent IDs i.e., 
+    policy IDs in the multi-agent. Agent IDs are accessible through the `_agent_ids` property. 
+    The initialized environment is a :py:class:`ray.rllib.env.MultiAgentEnv` object 
+    and has an `env` attribute that is :py:class:`citylearn.citylearn.CityLearnEnv` object.
+    
+    Parameters
+    ----------
+    env_config: Mapping[str, Any]
+        Dictionary providing initialization parameters for the environment.
+        Must contain `env_kwargs` as a key where `env_kwargs` is a `dict` used to 
+        initialize :py:class:`citylearn.citylearn.CityLearnEnv`. Thus it must 
+        contain all with positional arguments needed for intialization and 
+        optionally contain optional  intialization arguments. `env_config` can also 
+        contain a `wrappers` key that is a list of :py:mod:`citylearn.wrappers` 
+        classes to wrap :py:class:`citylearn.citylearn.CityLearnEnv` with. Wrapping
+        with :py:class:`citylearn.wrappers.ClippedObservationWrapper` is recommended
+        to avoid having the simulation terminating prematurely with an error due to
+        out of bound observations relative to the observation space.
+
+    Notes
+    -----
+    This wrapper is only compatible with an environment where 
+    :py:attr:`citylearn.citylearn.central_agent`=`False` and will initialize the 
+    environment as such, overriding any value for `central_agent` in `env_kwargs`.
+    """
+    
+    def __init__(self, env_config: Mapping[str, Any]):
+        if MultiAgentEnv == Env:
+            raise Exception('This functionality requires you to install RLlib.'\
+                'You can install RLlib from pip: pip install "ray[rllib]", '\
+                    'or for more detailed instructions please visit https://docs.ray.io/en/latest/rllib/index.html.')
+        
+        else:
+            pass
+
+        super().__init__()
+        env_kwargs = env_config['env_kwargs']
+        env_kwargs['central_agent'] = False
+        assert 'schema' in env_kwargs, 'missing schema key in env_kwargs.'
+        env = CityLearnEnv(**env_kwargs)
+
+        wrappers = env_config.get('wrappers')
+        wrappers = [] if wrappers is None else wrappers
+
+        for w in wrappers:
+            env = w(env)
+
+        env = RLlibMultiAgentActionWrapper(env)
+        env = RLlibMultiAgentObservationWrapper(env)
+        env = RLlibMultiAgentRewardWrapper(env)
+        self.env: CityLearnEnv = env
+        self._agent_ids = [f'agent_{i}' for i in range(len(self.buildings))]
+        self.observation_space: spaces.Dict = self.env.observation_space
+        self.action_space: spaces.Dict = self.env.action_space
+
+    @property
+    def time_step(self) -> int:
+        """Convenience property for :py:meth:`citylearn.citylearn.CityLearnEnv.time_step`."""
+
+        return self.env.time_step
+
+    @property
+    def buildings(self) -> List[Building]:
+        """Convenience property for :py:meth:`citylearn.citylearn.CityLearnEnv.buildings`."""
+
+        return self.env.buildings
+    
+    @property
+    def terminated(self) -> bool:
+        """Convenience property for :py:meth:`citylearn.citylearn.CityLearnEnv.terminated`."""
+
+        return self.env.terminated
+
+    def step(
+            self, action_dict: Mapping[str, np.ndarray]
+    ) -> Tuple[Mapping[str, np.ndarray], Mapping[str, float], Mapping[str, bool], Mapping[str, bool], Mapping[str, dict]]:
+        """Calls :py:meth:`citylearn.citylearn.CityLearnEnv.step` and parses returned values into dictionaries."""
+
+        observations, reward, terminated, truncated, info = self.env.step(action_dict)
+        terminated = {'__all__': terminated, **{a: terminated for a in self._agent_ids}}
+        truncated = {'__all__': truncated, **{a: truncated for a in self._agent_ids}}
+        info = {a: info for a in self._agent_ids}
+
+        return observations, reward, terminated, truncated, info
+
+    def evaluate(self, **kwargs) -> pd.DataFrame:
+        """Convenience method for :py:meth:`citylearn.citylearn.CityLearnEnv.evaluate`."""
+
+        return self.env.evaluate(**kwargs)
+
+    def reset(self, *, seed: int = None, options: Mapping[str, Any] = None) -> Tuple[Mapping[str, np.ndarray], Mapping[str, dict]]:
+        """Calls :py:meth:`citylearn.citylearn.CityLearnEnv.reset` and parses returned values into dictionaries."""
+
+        observations, info = self.env.reset(seed=seed, options=options)
+        info = {a: info for a in self._agent_ids}
+
+        return observations, info
\ No newline at end of file
diff --git a/docs/requirements.txt b/docs/requirements.txt
index ab80c3c6..558095ce 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -5,6 +5,7 @@ nbsphinx-link
 Sphinx
 sphinxemoji
 sphinxcontrib-bibtex
+sphinx-argparse
 sphinx-copybutton
 sphinx-panels
 sphinx-rtd-theme
diff --git a/docs/source/cli.rst b/docs/source/cli.rst
new file mode 100644
index 00000000..78fc98f3
--- /dev/null
+++ b/docs/source/cli.rst
@@ -0,0 +1,6 @@
+======================
+Command Line Interface
+======================
+
+.. argparse::
+    :ref: citylearn.__main__.main
\ No newline at end of file
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 2d03fa93..631288e2 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -41,6 +41,7 @@
     'myst_nb', # jupyter notebook
     'nbsphinx_link', # link jupyter notebook from dir that is not in docs/
     'sphinx_panels', # for backgrounds
+    'sphinxarg.ext', # document CLI
 ]
 
 # source for bib references
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 363990d1..9547d9a5 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -47,6 +47,7 @@ CityLearn has been utilized in the following projects and publications:
    quickstart
    overview/index
    usage/index
+   cli
    api/modules
    citylearn_challenge/index
    contributing
diff --git a/examples/quickstart.ipynb b/examples/quickstart.ipynb
index 31d6431b..2c1ca983 100644
--- a/examples/quickstart.ipynb
+++ b/examples/quickstart.ipynb
@@ -38,7 +38,14 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## No Control (Baseline)\n",
+        "## CityLearn Control Agents"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### No Control (Baseline)\n",
         "\n",
         "Run the following to simulate an environment where the storage systems and heat pumps are not controlled (baseline). The storage actions prescribed will be 0.0 and the heat pump will have no action, i.e. `None`, causing it to deliver the ideal load in the building time series files:"
       ]
@@ -48,6 +55,14 @@
       "execution_count": 1,
       "metadata": {},
       "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "/Users/kingsleyenweye/Desktop/INTELLIGENT_ENVIRONMENT_LAB/citylearn/CityLearn/test-py311-env/lib/python3.11/site-packages/gymnasium/spaces/box.py:130: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
+            "  gym.logger.warn(f\"Box bound precision lowered by casting to {self.dtype}\")\n"
+          ]
+        },
         {
           "data": {
             "text/html": [
@@ -292,12 +307,14 @@
         "from citylearn.agents.base import BaselineAgent as Agent\n",
         "from citylearn.citylearn import CityLearnEnv\n",
         "\n",
-        "dataset_name = 'citylearn_challenge_2023_phase_2_local_evaluation'\n",
-        "env = CityLearnEnv(dataset_name, central_agent=True)\n",
+        "# initialize\n",
+        "env = CityLearnEnv('citylearn_challenge_2023_phase_2_local_evaluation', central_agent=True)\n",
         "model = Agent(env)\n",
+        "\n",
+        "# train\n",
         "model.learn(episodes=1)\n",
         "\n",
-        "# print cost functions at the end of episode\n",
+        "# test\n",
         "kpis = model.env.evaluate()\n",
         "kpis = kpis.pivot(index='cost_function', columns='name', values='value').round(3)\n",
         "kpis = kpis.dropna(how='all')\n",
@@ -310,7 +327,7 @@
         "id": "F93k9GV7Hnep"
       },
       "source": [
-        "## Centralized RBC\n",
+        "### Centralized RBC\n",
         "Run the following to simulate an environment controlled by centralized RBC agent for a single episode:"
       ]
     },
@@ -570,12 +587,14 @@
         "from citylearn.agents.rbc import BasicRBC as Agent\n",
         "from citylearn.citylearn import CityLearnEnv\n",
         "\n",
-        "dataset_name = 'citylearn_challenge_2023_phase_2_local_evaluation'\n",
-        "env = CityLearnEnv(dataset_name, central_agent=True)\n",
+        "# initialize\n",
+        "env = CityLearnEnv('citylearn_challenge_2023_phase_2_local_evaluation', central_agent=True)\n",
         "model = Agent(env)\n",
+        "\n",
+        "# train\n",
         "model.learn(episodes=1)\n",
         "\n",
-        "# print cost functions at the end of episode\n",
+        "# test\n",
         "kpis = model.env.evaluate()\n",
         "kpis = kpis.pivot(index='cost_function', columns='name', values='value').round(3)\n",
         "kpis = kpis.dropna(how='all')\n",
@@ -588,7 +607,7 @@
         "id": "RGSdXQRCHner"
       },
       "source": [
-        "## Decentralized-Independent SAC\n",
+        "### Decentralized-Independent SAC\n",
         "\n",
         "Run the following to simulate an environment controlled by decentralized-independent SAC agents for 1 training episode:"
       ]
@@ -645,7 +664,7 @@
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.947</td>\n",
+              "      <td>0.948</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>annual_normalized_unserved_energy_total</th>\n",
@@ -656,45 +675,45 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>carbon_emissions_total</th>\n",
-              "      <td>0.943</td>\n",
-              "      <td>0.960</td>\n",
-              "      <td>0.939</td>\n",
+              "      <td>0.929</td>\n",
+              "      <td>0.969</td>\n",
+              "      <td>0.944</td>\n",
               "      <td>0.947</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>cost_total</th>\n",
-              "      <td>0.906</td>\n",
-              "      <td>0.921</td>\n",
-              "      <td>0.907</td>\n",
-              "      <td>0.911</td>\n",
+              "      <td>0.892</td>\n",
+              "      <td>0.931</td>\n",
+              "      <td>0.912</td>\n",
+              "      <td>0.912</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>daily_one_minus_load_factor_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.945</td>\n",
+              "      <td>0.946</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>daily_peak_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.928</td>\n",
+              "      <td>0.927</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_delta_average</th>\n",
-              "      <td>1.899</td>\n",
-              "      <td>0.878</td>\n",
-              "      <td>0.823</td>\n",
-              "      <td>1.200</td>\n",
+              "      <td>1.819</td>\n",
+              "      <td>0.911</td>\n",
+              "      <td>0.875</td>\n",
+              "      <td>1.201</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_delta_maximum</th>\n",
-              "      <td>6.387</td>\n",
-              "      <td>4.487</td>\n",
-              "      <td>2.908</td>\n",
-              "      <td>4.594</td>\n",
+              "      <td>6.324</td>\n",
+              "      <td>4.555</td>\n",
+              "      <td>2.977</td>\n",
+              "      <td>4.619</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_delta_minimum</th>\n",
@@ -705,24 +724,24 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_proportion</th>\n",
-              "      <td>0.435</td>\n",
-              "      <td>0.313</td>\n",
-              "      <td>0.208</td>\n",
-              "      <td>0.319</td>\n",
+              "      <td>0.412</td>\n",
+              "      <td>0.321</td>\n",
+              "      <td>0.238</td>\n",
+              "      <td>0.324</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_average</th>\n",
-              "      <td>0.250</td>\n",
-              "      <td>0.567</td>\n",
-              "      <td>0.330</td>\n",
-              "      <td>0.382</td>\n",
+              "      <td>0.270</td>\n",
+              "      <td>0.550</td>\n",
+              "      <td>0.302</td>\n",
+              "      <td>0.374</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_maximum</th>\n",
-              "      <td>4.740</td>\n",
-              "      <td>6.272</td>\n",
-              "      <td>4.024</td>\n",
-              "      <td>5.012</td>\n",
+              "      <td>4.846</td>\n",
+              "      <td>6.216</td>\n",
+              "      <td>3.977</td>\n",
+              "      <td>5.013</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_minimum</th>\n",
@@ -733,23 +752,23 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_proportion</th>\n",
-              "      <td>0.045</td>\n",
-              "      <td>0.160</td>\n",
-              "      <td>0.038</td>\n",
-              "      <td>0.081</td>\n",
+              "      <td>0.046</td>\n",
+              "      <td>0.153</td>\n",
+              "      <td>0.030</td>\n",
+              "      <td>0.076</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_proportion</th>\n",
-              "      <td>0.480</td>\n",
+              "      <td>0.459</td>\n",
               "      <td>0.474</td>\n",
-              "      <td>0.246</td>\n",
+              "      <td>0.268</td>\n",
               "      <td>0.400</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>electricity_consumption_total</th>\n",
-              "      <td>0.953</td>\n",
-              "      <td>0.977</td>\n",
-              "      <td>0.954</td>\n",
+              "      <td>0.939</td>\n",
+              "      <td>0.987</td>\n",
+              "      <td>0.958</td>\n",
               "      <td>0.961</td>\n",
               "    </tr>\n",
               "    <tr>\n",
@@ -768,23 +787,23 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>power_outage_normalized_unserved_energy_total</th>\n",
-              "      <td>0.629</td>\n",
-              "      <td>0.643</td>\n",
-              "      <td>0.593</td>\n",
-              "      <td>0.621</td>\n",
+              "      <td>0.626</td>\n",
+              "      <td>0.649</td>\n",
+              "      <td>0.595</td>\n",
+              "      <td>0.624</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>ramping_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.882</td>\n",
+              "      <td>0.883</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>zero_net_energy</th>\n",
-              "      <td>0.968</td>\n",
-              "      <td>0.979</td>\n",
-              "      <td>0.954</td>\n",
+              "      <td>0.953</td>\n",
+              "      <td>0.989</td>\n",
+              "      <td>0.959</td>\n",
               "      <td>0.967</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
@@ -796,49 +815,49 @@
               "cost_function                                                           \n",
               "all_time_peak_average                                 NaN         NaN   \n",
               "annual_normalized_unserved_energy_total             0.013       0.013   \n",
-              "carbon_emissions_total                              0.943       0.960   \n",
-              "cost_total                                          0.906       0.921   \n",
+              "carbon_emissions_total                              0.929       0.969   \n",
+              "cost_total                                          0.892       0.931   \n",
               "daily_one_minus_load_factor_average                   NaN         NaN   \n",
               "daily_peak_average                                    NaN         NaN   \n",
-              "discomfort_cold_delta_average                       1.899       0.878   \n",
-              "discomfort_cold_delta_maximum                       6.387       4.487   \n",
+              "discomfort_cold_delta_average                       1.819       0.911   \n",
+              "discomfort_cold_delta_maximum                       6.324       4.555   \n",
               "discomfort_cold_delta_minimum                       0.000       0.000   \n",
-              "discomfort_cold_proportion                          0.435       0.313   \n",
-              "discomfort_hot_delta_average                        0.250       0.567   \n",
-              "discomfort_hot_delta_maximum                        4.740       6.272   \n",
+              "discomfort_cold_proportion                          0.412       0.321   \n",
+              "discomfort_hot_delta_average                        0.270       0.550   \n",
+              "discomfort_hot_delta_maximum                        4.846       6.216   \n",
               "discomfort_hot_delta_minimum                        0.000       0.000   \n",
-              "discomfort_hot_proportion                           0.045       0.160   \n",
-              "discomfort_proportion                               0.480       0.474   \n",
-              "electricity_consumption_total                       0.953       0.977   \n",
+              "discomfort_hot_proportion                           0.046       0.153   \n",
+              "discomfort_proportion                               0.459       0.474   \n",
+              "electricity_consumption_total                       0.939       0.987   \n",
               "monthly_one_minus_load_factor_average                 NaN         NaN   \n",
               "one_minus_thermal_resilience_proportion             0.733       0.500   \n",
-              "power_outage_normalized_unserved_energy_total       0.629       0.643   \n",
+              "power_outage_normalized_unserved_energy_total       0.626       0.649   \n",
               "ramping_average                                       NaN         NaN   \n",
-              "zero_net_energy                                     0.968       0.979   \n",
+              "zero_net_energy                                     0.953       0.989   \n",
               "\n",
               "name                                           Building_3  District  \n",
               "cost_function                                                        \n",
-              "all_time_peak_average                                 NaN     0.947  \n",
+              "all_time_peak_average                                 NaN     0.948  \n",
               "annual_normalized_unserved_energy_total             0.012     0.013  \n",
-              "carbon_emissions_total                              0.939     0.947  \n",
-              "cost_total                                          0.907     0.911  \n",
-              "daily_one_minus_load_factor_average                   NaN     0.945  \n",
-              "daily_peak_average                                    NaN     0.928  \n",
-              "discomfort_cold_delta_average                       0.823     1.200  \n",
-              "discomfort_cold_delta_maximum                       2.908     4.594  \n",
+              "carbon_emissions_total                              0.944     0.947  \n",
+              "cost_total                                          0.912     0.912  \n",
+              "daily_one_minus_load_factor_average                   NaN     0.946  \n",
+              "daily_peak_average                                    NaN     0.927  \n",
+              "discomfort_cold_delta_average                       0.875     1.201  \n",
+              "discomfort_cold_delta_maximum                       2.977     4.619  \n",
               "discomfort_cold_delta_minimum                       0.000     0.000  \n",
-              "discomfort_cold_proportion                          0.208     0.319  \n",
-              "discomfort_hot_delta_average                        0.330     0.382  \n",
-              "discomfort_hot_delta_maximum                        4.024     5.012  \n",
+              "discomfort_cold_proportion                          0.238     0.324  \n",
+              "discomfort_hot_delta_average                        0.302     0.374  \n",
+              "discomfort_hot_delta_maximum                        3.977     5.013  \n",
               "discomfort_hot_delta_minimum                        0.000     0.000  \n",
-              "discomfort_hot_proportion                           0.038     0.081  \n",
-              "discomfort_proportion                               0.246     0.400  \n",
-              "electricity_consumption_total                       0.954     0.961  \n",
+              "discomfort_hot_proportion                           0.030     0.076  \n",
+              "discomfort_proportion                               0.268     0.400  \n",
+              "electricity_consumption_total                       0.958     0.961  \n",
               "monthly_one_minus_load_factor_average                 NaN     0.996  \n",
               "one_minus_thermal_resilience_proportion             0.133     0.456  \n",
-              "power_outage_normalized_unserved_energy_total       0.593     0.621  \n",
-              "ramping_average                                       NaN     0.882  \n",
-              "zero_net_energy                                     0.954     0.967  "
+              "power_outage_normalized_unserved_energy_total       0.595     0.624  \n",
+              "ramping_average                                       NaN     0.883  \n",
+              "zero_net_energy                                     0.959     0.967  "
             ]
           },
           "metadata": {},
@@ -849,12 +868,14 @@
         "from citylearn.agents.sac import SAC as Agent\n",
         "from citylearn.citylearn import CityLearnEnv\n",
         "\n",
-        "dataset_name = 'citylearn_challenge_2023_phase_2_local_evaluation'\n",
-        "env = CityLearnEnv(dataset_name, central_agent=False)\n",
+        "# initialize\n",
+        "env = CityLearnEnv('citylearn_challenge_2023_phase_2_local_evaluation', central_agent=False)\n",
         "model = Agent(env)\n",
+        "\n",
+        "# train\n",
         "model.learn(episodes=2, deterministic_finish=True)\n",
         "\n",
-        "# print cost functions at the end of episode\n",
+        "# test\n",
         "kpis = model.env.evaluate()\n",
         "kpis = kpis.pivot(index='cost_function', columns='name', values='value').round(3)\n",
         "kpis = kpis.dropna(how='all')\n",
@@ -867,7 +888,7 @@
         "id": "31AHVfD4Hnes"
       },
       "source": [
-        "## Decentralized-Cooperative MARLISA\n",
+        "### Decentralized-Cooperative MARLISA\n",
         "\n",
         "Run the following to simulate an environment controlled by decentralized-cooperative MARLISA agents for 1 training episodes:"
       ]
@@ -924,7 +945,7 @@
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.949</td>\n",
+              "      <td>0.948</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>annual_normalized_unserved_energy_total</th>\n",
@@ -935,45 +956,45 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>carbon_emissions_total</th>\n",
-              "      <td>0.951</td>\n",
-              "      <td>0.974</td>\n",
-              "      <td>0.957</td>\n",
-              "      <td>0.961</td>\n",
+              "      <td>0.938</td>\n",
+              "      <td>0.964</td>\n",
+              "      <td>0.941</td>\n",
+              "      <td>0.948</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>cost_total</th>\n",
-              "      <td>0.914</td>\n",
-              "      <td>0.935</td>\n",
-              "      <td>0.925</td>\n",
+              "      <td>0.900</td>\n",
               "      <td>0.925</td>\n",
+              "      <td>0.909</td>\n",
+              "      <td>0.912</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>daily_one_minus_load_factor_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.939</td>\n",
+              "      <td>0.945</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>daily_peak_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.932</td>\n",
+              "      <td>0.927</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_delta_average</th>\n",
-              "      <td>1.918</td>\n",
-              "      <td>0.908</td>\n",
-              "      <td>0.865</td>\n",
-              "      <td>1.230</td>\n",
+              "      <td>1.875</td>\n",
+              "      <td>0.912</td>\n",
+              "      <td>0.866</td>\n",
+              "      <td>1.218</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_delta_maximum</th>\n",
-              "      <td>6.424</td>\n",
-              "      <td>4.540</td>\n",
-              "      <td>2.976</td>\n",
-              "      <td>4.647</td>\n",
+              "      <td>6.396</td>\n",
+              "      <td>4.549</td>\n",
+              "      <td>2.959</td>\n",
+              "      <td>4.635</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_delta_minimum</th>\n",
@@ -984,24 +1005,24 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_proportion</th>\n",
-              "      <td>0.445</td>\n",
-              "      <td>0.315</td>\n",
+              "      <td>0.431</td>\n",
+              "      <td>0.317</td>\n",
               "      <td>0.235</td>\n",
-              "      <td>0.332</td>\n",
+              "      <td>0.327</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_average</th>\n",
-              "      <td>0.247</td>\n",
-              "      <td>0.557</td>\n",
-              "      <td>0.312</td>\n",
-              "      <td>0.372</td>\n",
+              "      <td>0.256</td>\n",
+              "      <td>0.553</td>\n",
+              "      <td>0.309</td>\n",
+              "      <td>0.373</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_maximum</th>\n",
-              "      <td>4.721</td>\n",
-              "      <td>6.294</td>\n",
+              "      <td>4.776</td>\n",
+              "      <td>6.274</td>\n",
               "      <td>3.994</td>\n",
-              "      <td>5.003</td>\n",
+              "      <td>5.015</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_minimum</th>\n",
@@ -1012,31 +1033,31 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_proportion</th>\n",
-              "      <td>0.042</td>\n",
-              "      <td>0.157</td>\n",
-              "      <td>0.033</td>\n",
-              "      <td>0.077</td>\n",
+              "      <td>0.046</td>\n",
+              "      <td>0.153</td>\n",
+              "      <td>0.030</td>\n",
+              "      <td>0.076</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_proportion</th>\n",
-              "      <td>0.487</td>\n",
-              "      <td>0.472</td>\n",
-              "      <td>0.268</td>\n",
-              "      <td>0.409</td>\n",
+              "      <td>0.477</td>\n",
+              "      <td>0.470</td>\n",
+              "      <td>0.265</td>\n",
+              "      <td>0.404</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>electricity_consumption_total</th>\n",
-              "      <td>0.961</td>\n",
-              "      <td>0.992</td>\n",
-              "      <td>0.972</td>\n",
-              "      <td>0.975</td>\n",
+              "      <td>0.948</td>\n",
+              "      <td>0.982</td>\n",
+              "      <td>0.956</td>\n",
+              "      <td>0.962</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>monthly_one_minus_load_factor_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.993</td>\n",
+              "      <td>0.996</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>one_minus_thermal_resilience_proportion</th>\n",
@@ -1047,24 +1068,24 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>power_outage_normalized_unserved_energy_total</th>\n",
-              "      <td>0.629</td>\n",
-              "      <td>0.647</td>\n",
-              "      <td>0.596</td>\n",
-              "      <td>0.624</td>\n",
+              "      <td>0.628</td>\n",
+              "      <td>0.638</td>\n",
+              "      <td>0.595</td>\n",
+              "      <td>0.620</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>ramping_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.881</td>\n",
+              "      <td>0.884</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>zero_net_energy</th>\n",
-              "      <td>0.977</td>\n",
-              "      <td>0.995</td>\n",
-              "      <td>0.973</td>\n",
-              "      <td>0.981</td>\n",
+              "      <td>0.962</td>\n",
+              "      <td>0.985</td>\n",
+              "      <td>0.956</td>\n",
+              "      <td>0.968</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
               "</table>\n",
@@ -1075,49 +1096,49 @@
               "cost_function                                                           \n",
               "all_time_peak_average                                 NaN         NaN   \n",
               "annual_normalized_unserved_energy_total             0.013       0.013   \n",
-              "carbon_emissions_total                              0.951       0.974   \n",
-              "cost_total                                          0.914       0.935   \n",
+              "carbon_emissions_total                              0.938       0.964   \n",
+              "cost_total                                          0.900       0.925   \n",
               "daily_one_minus_load_factor_average                   NaN         NaN   \n",
               "daily_peak_average                                    NaN         NaN   \n",
-              "discomfort_cold_delta_average                       1.918       0.908   \n",
-              "discomfort_cold_delta_maximum                       6.424       4.540   \n",
+              "discomfort_cold_delta_average                       1.875       0.912   \n",
+              "discomfort_cold_delta_maximum                       6.396       4.549   \n",
               "discomfort_cold_delta_minimum                       0.000       0.000   \n",
-              "discomfort_cold_proportion                          0.445       0.315   \n",
-              "discomfort_hot_delta_average                        0.247       0.557   \n",
-              "discomfort_hot_delta_maximum                        4.721       6.294   \n",
+              "discomfort_cold_proportion                          0.431       0.317   \n",
+              "discomfort_hot_delta_average                        0.256       0.553   \n",
+              "discomfort_hot_delta_maximum                        4.776       6.274   \n",
               "discomfort_hot_delta_minimum                        0.000       0.000   \n",
-              "discomfort_hot_proportion                           0.042       0.157   \n",
-              "discomfort_proportion                               0.487       0.472   \n",
-              "electricity_consumption_total                       0.961       0.992   \n",
+              "discomfort_hot_proportion                           0.046       0.153   \n",
+              "discomfort_proportion                               0.477       0.470   \n",
+              "electricity_consumption_total                       0.948       0.982   \n",
               "monthly_one_minus_load_factor_average                 NaN         NaN   \n",
               "one_minus_thermal_resilience_proportion             0.733       0.500   \n",
-              "power_outage_normalized_unserved_energy_total       0.629       0.647   \n",
+              "power_outage_normalized_unserved_energy_total       0.628       0.638   \n",
               "ramping_average                                       NaN         NaN   \n",
-              "zero_net_energy                                     0.977       0.995   \n",
+              "zero_net_energy                                     0.962       0.985   \n",
               "\n",
               "name                                           Building_3  District  \n",
               "cost_function                                                        \n",
-              "all_time_peak_average                                 NaN     0.949  \n",
+              "all_time_peak_average                                 NaN     0.948  \n",
               "annual_normalized_unserved_energy_total             0.012     0.013  \n",
-              "carbon_emissions_total                              0.957     0.961  \n",
-              "cost_total                                          0.925     0.925  \n",
-              "daily_one_minus_load_factor_average                   NaN     0.939  \n",
-              "daily_peak_average                                    NaN     0.932  \n",
-              "discomfort_cold_delta_average                       0.865     1.230  \n",
-              "discomfort_cold_delta_maximum                       2.976     4.647  \n",
+              "carbon_emissions_total                              0.941     0.948  \n",
+              "cost_total                                          0.909     0.912  \n",
+              "daily_one_minus_load_factor_average                   NaN     0.945  \n",
+              "daily_peak_average                                    NaN     0.927  \n",
+              "discomfort_cold_delta_average                       0.866     1.218  \n",
+              "discomfort_cold_delta_maximum                       2.959     4.635  \n",
               "discomfort_cold_delta_minimum                       0.000     0.000  \n",
-              "discomfort_cold_proportion                          0.235     0.332  \n",
-              "discomfort_hot_delta_average                        0.312     0.372  \n",
-              "discomfort_hot_delta_maximum                        3.994     5.003  \n",
+              "discomfort_cold_proportion                          0.235     0.327  \n",
+              "discomfort_hot_delta_average                        0.309     0.373  \n",
+              "discomfort_hot_delta_maximum                        3.994     5.015  \n",
               "discomfort_hot_delta_minimum                        0.000     0.000  \n",
-              "discomfort_hot_proportion                           0.033     0.077  \n",
-              "discomfort_proportion                               0.268     0.409  \n",
-              "electricity_consumption_total                       0.972     0.975  \n",
-              "monthly_one_minus_load_factor_average                 NaN     0.993  \n",
+              "discomfort_hot_proportion                           0.030     0.076  \n",
+              "discomfort_proportion                               0.265     0.404  \n",
+              "electricity_consumption_total                       0.956     0.962  \n",
+              "monthly_one_minus_load_factor_average                 NaN     0.996  \n",
               "one_minus_thermal_resilience_proportion             0.133     0.456  \n",
-              "power_outage_normalized_unserved_energy_total       0.596     0.624  \n",
-              "ramping_average                                       NaN     0.881  \n",
-              "zero_net_energy                                     0.973     0.981  "
+              "power_outage_normalized_unserved_energy_total       0.595     0.620  \n",
+              "ramping_average                                       NaN     0.884  \n",
+              "zero_net_energy                                     0.956     0.968  "
             ]
           },
           "metadata": {},
@@ -1128,24 +1149,34 @@
         "from citylearn.agents.marlisa import MARLISA as Agent\n",
         "from citylearn.citylearn import CityLearnEnv\n",
         "\n",
-        "dataset_name = 'citylearn_challenge_2023_phase_2_local_evaluation'\n",
-        "env = CityLearnEnv(dataset_name, central_agent=False)\n",
+        "# initialize\n",
+        "env = CityLearnEnv('citylearn_challenge_2023_phase_2_local_evaluation', central_agent=False)\n",
         "model = Agent(env)\n",
+        "\n",
+        "# train\n",
         "model.learn(episodes=2, deterministic_finish=True)\n",
         "\n",
+        "# test\n",
         "kpis = model.env.evaluate()\n",
         "kpis = kpis.pivot(index='cost_function', columns='name', values='value').round(3)\n",
         "kpis = kpis.dropna(how='all')\n",
         "display(kpis)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Other Standard Reinforcement Learning Libraries"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "XY-UbGD6Hnet"
       },
       "source": [
-        "## Stable Baselines3 Reinforcement Learning Algorithms\n",
+        "### Stable Baselines3 Reinforcement Learning Algorithms\n",
         "\n",
         "Install the latest version of Stable Baselines3:"
       ]
@@ -1171,8 +1202,6 @@
         "\n",
         "Next, we wrap with the `StableBaselines3Wrapper` (see [docs](https://www.citylearn.net/api/citylearn.wrappers.html#citylearn.wrappers.StableBaselines3Wrapper)) that ensures observations, actions and rewards are served in manner that is compatible with Stable Baselines3 interface.\n",
         "\n",
-        "For the following Stable Baselines3 example, the `baeda_3dem` dataset that support building temperature dynamics is used.\n",
-        "\n",
         "> ⚠️ **NOTE**: `central_agent` in the `env` must be `True` when using Stable Baselines3  as it does not support multi-agents."
       ]
     },
@@ -1188,6 +1217,14 @@
         "outputId": "fbe45090-51b7-4e31-ec9b-cbc647e0e397"
       },
       "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "2024-04-01 20:37:07,558\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n",
+            "2024-04-01 20:37:08,286\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n"
+          ]
+        },
         {
           "data": {
             "text/html": [
@@ -1228,42 +1265,42 @@
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.839</td>\n",
+              "      <td>0.834</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>annual_normalized_unserved_energy_total</th>\n",
               "      <td>0.015</td>\n",
               "      <td>0.012</td>\n",
-              "      <td>0.013</td>\n",
-              "      <td>0.013</td>\n",
+              "      <td>0.014</td>\n",
+              "      <td>0.014</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>carbon_emissions_total</th>\n",
-              "      <td>0.386</td>\n",
-              "      <td>0.363</td>\n",
-              "      <td>0.554</td>\n",
-              "      <td>0.435</td>\n",
+              "      <td>0.396</td>\n",
+              "      <td>0.449</td>\n",
+              "      <td>0.501</td>\n",
+              "      <td>0.449</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>cost_total</th>\n",
-              "      <td>0.366</td>\n",
-              "      <td>0.339</td>\n",
-              "      <td>0.523</td>\n",
-              "      <td>0.409</td>\n",
+              "      <td>0.375</td>\n",
+              "      <td>0.413</td>\n",
+              "      <td>0.471</td>\n",
+              "      <td>0.419</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>daily_one_minus_load_factor_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>1.291</td>\n",
+              "      <td>1.316</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>daily_peak_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.700</td>\n",
+              "      <td>0.709</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_cold_delta_average</th>\n",
@@ -1295,17 +1332,17 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_average</th>\n",
-              "      <td>9.453</td>\n",
-              "      <td>7.169</td>\n",
-              "      <td>5.916</td>\n",
-              "      <td>7.513</td>\n",
+              "      <td>9.465</td>\n",
+              "      <td>7.146</td>\n",
+              "      <td>8.046</td>\n",
+              "      <td>8.219</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_maximum</th>\n",
-              "      <td>16.778</td>\n",
-              "      <td>14.863</td>\n",
-              "      <td>12.096</td>\n",
-              "      <td>14.579</td>\n",
+              "      <td>16.783</td>\n",
+              "      <td>14.850</td>\n",
+              "      <td>14.003</td>\n",
+              "      <td>15.212</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_hot_delta_minimum</th>\n",
@@ -1318,29 +1355,29 @@
               "      <th>discomfort_hot_proportion</th>\n",
               "      <td>0.982</td>\n",
               "      <td>0.978</td>\n",
-              "      <td>0.930</td>\n",
-              "      <td>0.963</td>\n",
+              "      <td>0.978</td>\n",
+              "      <td>0.979</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>discomfort_proportion</th>\n",
               "      <td>0.982</td>\n",
               "      <td>0.978</td>\n",
-              "      <td>0.930</td>\n",
-              "      <td>0.963</td>\n",
+              "      <td>0.978</td>\n",
+              "      <td>0.979</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>electricity_consumption_total</th>\n",
-              "      <td>0.390</td>\n",
-              "      <td>0.371</td>\n",
-              "      <td>0.562</td>\n",
-              "      <td>0.441</td>\n",
+              "      <td>0.401</td>\n",
+              "      <td>0.453</td>\n",
+              "      <td>0.508</td>\n",
+              "      <td>0.454</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>monthly_one_minus_load_factor_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>1.122</td>\n",
+              "      <td>1.126</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>one_minus_thermal_resilience_proportion</th>\n",
@@ -1351,24 +1388,24 @@
               "    </tr>\n",
               "    <tr>\n",
               "      <th>power_outage_normalized_unserved_energy_total</th>\n",
-              "      <td>0.569</td>\n",
-              "      <td>0.499</td>\n",
-              "      <td>0.531</td>\n",
-              "      <td>0.533</td>\n",
+              "      <td>0.570</td>\n",
+              "      <td>0.509</td>\n",
+              "      <td>0.555</td>\n",
+              "      <td>0.545</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>ramping_average</th>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
               "      <td>NaN</td>\n",
-              "      <td>0.900</td>\n",
+              "      <td>0.974</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>zero_net_energy</th>\n",
-              "      <td>0.273</td>\n",
-              "      <td>0.334</td>\n",
-              "      <td>0.520</td>\n",
-              "      <td>0.376</td>\n",
+              "      <td>0.275</td>\n",
+              "      <td>0.362</td>\n",
+              "      <td>0.453</td>\n",
+              "      <td>0.363</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
               "</table>\n",
@@ -1379,49 +1416,49 @@
               "cost_function                                                           \n",
               "all_time_peak_average                                 NaN         NaN   \n",
               "annual_normalized_unserved_energy_total             0.015       0.012   \n",
-              "carbon_emissions_total                              0.386       0.363   \n",
-              "cost_total                                          0.366       0.339   \n",
+              "carbon_emissions_total                              0.396       0.449   \n",
+              "cost_total                                          0.375       0.413   \n",
               "daily_one_minus_load_factor_average                   NaN         NaN   \n",
               "daily_peak_average                                    NaN         NaN   \n",
               "discomfort_cold_delta_average                       0.000       0.004   \n",
               "discomfort_cold_delta_maximum                       0.124       0.581   \n",
               "discomfort_cold_delta_minimum                       0.000       0.000   \n",
               "discomfort_cold_proportion                          0.000       0.000   \n",
-              "discomfort_hot_delta_average                        9.453       7.169   \n",
-              "discomfort_hot_delta_maximum                       16.778      14.863   \n",
+              "discomfort_hot_delta_average                        9.465       7.146   \n",
+              "discomfort_hot_delta_maximum                       16.783      14.850   \n",
               "discomfort_hot_delta_minimum                        0.000       0.000   \n",
               "discomfort_hot_proportion                           0.982       0.978   \n",
               "discomfort_proportion                               0.982       0.978   \n",
-              "electricity_consumption_total                       0.390       0.371   \n",
+              "electricity_consumption_total                       0.401       0.453   \n",
               "monthly_one_minus_load_factor_average                 NaN         NaN   \n",
               "one_minus_thermal_resilience_proportion             1.000       1.000   \n",
-              "power_outage_normalized_unserved_energy_total       0.569       0.499   \n",
+              "power_outage_normalized_unserved_energy_total       0.570       0.509   \n",
               "ramping_average                                       NaN         NaN   \n",
-              "zero_net_energy                                     0.273       0.334   \n",
+              "zero_net_energy                                     0.275       0.362   \n",
               "\n",
               "name                                           Building_3  District  \n",
               "cost_function                                                        \n",
-              "all_time_peak_average                                 NaN     0.839  \n",
-              "annual_normalized_unserved_energy_total             0.013     0.013  \n",
-              "carbon_emissions_total                              0.554     0.435  \n",
-              "cost_total                                          0.523     0.409  \n",
-              "daily_one_minus_load_factor_average                   NaN     1.291  \n",
-              "daily_peak_average                                    NaN     0.700  \n",
+              "all_time_peak_average                                 NaN     0.834  \n",
+              "annual_normalized_unserved_energy_total             0.014     0.014  \n",
+              "carbon_emissions_total                              0.501     0.449  \n",
+              "cost_total                                          0.471     0.419  \n",
+              "daily_one_minus_load_factor_average                   NaN     1.316  \n",
+              "daily_peak_average                                    NaN     0.709  \n",
               "discomfort_cold_delta_average                       0.001     0.002  \n",
               "discomfort_cold_delta_maximum                       0.372     0.359  \n",
               "discomfort_cold_delta_minimum                       0.000     0.000  \n",
               "discomfort_cold_proportion                          0.000     0.000  \n",
-              "discomfort_hot_delta_average                        5.916     7.513  \n",
-              "discomfort_hot_delta_maximum                       12.096    14.579  \n",
+              "discomfort_hot_delta_average                        8.046     8.219  \n",
+              "discomfort_hot_delta_maximum                       14.003    15.212  \n",
               "discomfort_hot_delta_minimum                        0.000     0.000  \n",
-              "discomfort_hot_proportion                           0.930     0.963  \n",
-              "discomfort_proportion                               0.930     0.963  \n",
-              "electricity_consumption_total                       0.562     0.441  \n",
-              "monthly_one_minus_load_factor_average                 NaN     1.122  \n",
+              "discomfort_hot_proportion                           0.978     0.979  \n",
+              "discomfort_proportion                               0.978     0.979  \n",
+              "electricity_consumption_total                       0.508     0.454  \n",
+              "monthly_one_minus_load_factor_average                 NaN     1.126  \n",
               "one_minus_thermal_resilience_proportion             1.000     1.000  \n",
-              "power_outage_normalized_unserved_energy_total       0.531     0.533  \n",
-              "ramping_average                                       NaN     0.900  \n",
-              "zero_net_energy                                     0.520     0.376  "
+              "power_outage_normalized_unserved_energy_total       0.555     0.545  \n",
+              "ramping_average                                       NaN     0.974  \n",
+              "zero_net_energy                                     0.453     0.363  "
             ]
           },
           "metadata": {},
@@ -1433,15 +1470,17 @@
         "from citylearn.citylearn import CityLearnEnv\n",
         "from citylearn.wrappers import NormalizedObservationWrapper, StableBaselines3Wrapper\n",
         "\n",
-        "dataset_name = 'citylearn_challenge_2023_phase_2_local_evaluation'\n",
-        "env = CityLearnEnv(dataset_name, central_agent=True)\n",
+        "# initialize\n",
+        "env = CityLearnEnv('citylearn_challenge_2023_phase_2_local_evaluation', central_agent=True)\n",
         "env = NormalizedObservationWrapper(env)\n",
         "env = StableBaselines3Wrapper(env)\n",
         "model = Agent('MlpPolicy', env)\n",
+        "\n",
+        "# train\n",
         "episodes = 2\n",
         "model.learn(total_timesteps=env.unwrapped.time_steps*episodes)\n",
         "\n",
-        "# evaluate\n",
+        "# test\n",
         "observations, _ = env.reset()\n",
         "\n",
         "while not env.unwrapped.terminated:\n",
@@ -1453,6 +1492,644 @@
         "kpis = kpis.dropna(how='all')\n",
         "display(kpis)"
       ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### RLlib\n",
+        "\n",
+        "Install the latest version of RLlib:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install \"ray[rllib]\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "We advise that you include the `ClippedObservationWrapper` (see [docs](https://www.citylearn.net/api/citylearn.wrappers.html#citylearn.wrappers.ClippedObservationWrapper)) wrapper when working with RLlib so that observations are always clipped within the observation space before sending to the agent if not, out-of-bound observations will raise a `ValueError` and terminate the training.\n",
+        "\n",
+        "We also wrap the environment with `NormalizedObservationWrapper` (see [docs](https://www.citylearn.net/api/citylearn.wrappers.html#citylearn.wrappers.NormalizedObservationWrapper)) to ensure that observations served to the agent are min-max normalized between [0, 1] and cyclical observations e.g. hour, are encoded using the cosine transformation.\n",
+        "\n",
+        "RLlib supports both [single-agent and multi-agent algorithms](https://docs.ray.io/en/latest/rllib/rllib-algorithms.html). See below for an example for either case."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "####  Single Agent\n",
+        "\n",
+        "The single-agent interface for RLlib is the `RLlibSingleAgentWrapper` wrapper."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 10,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "2024-04-01 20:53:15,333\tWARNING util.py:62 -- Install gputil for GPU system monitoring.\n"
+          ]
+        },
+        {
+          "data": {
+            "text/html": [
+              "<div>\n",
+              "<style scoped>\n",
+              "    .dataframe tbody tr th:only-of-type {\n",
+              "        vertical-align: middle;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe tbody tr th {\n",
+              "        vertical-align: top;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe thead th {\n",
+              "        text-align: right;\n",
+              "    }\n",
+              "</style>\n",
+              "<table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              "    <tr style=\"text-align: right;\">\n",
+              "      <th>name</th>\n",
+              "      <th>Building_1</th>\n",
+              "      <th>Building_2</th>\n",
+              "      <th>Building_3</th>\n",
+              "      <th>District</th>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>cost_function</th>\n",
+              "      <th></th>\n",
+              "      <th></th>\n",
+              "      <th></th>\n",
+              "      <th></th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <th>all_time_peak_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>1.063</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>annual_normalized_unserved_energy_total</th>\n",
+              "      <td>0.015</td>\n",
+              "      <td>0.016</td>\n",
+              "      <td>0.014</td>\n",
+              "      <td>0.015</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>carbon_emissions_total</th>\n",
+              "      <td>1.632</td>\n",
+              "      <td>1.598</td>\n",
+              "      <td>1.459</td>\n",
+              "      <td>1.563</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>cost_total</th>\n",
+              "      <td>1.572</td>\n",
+              "      <td>1.548</td>\n",
+              "      <td>1.427</td>\n",
+              "      <td>1.516</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>daily_one_minus_load_factor_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>0.740</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>daily_peak_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>1.166</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_cold_delta_average</th>\n",
+              "      <td>7.743</td>\n",
+              "      <td>2.896</td>\n",
+              "      <td>2.563</td>\n",
+              "      <td>4.401</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_cold_delta_maximum</th>\n",
+              "      <td>12.215</td>\n",
+              "      <td>8.302</td>\n",
+              "      <td>5.262</td>\n",
+              "      <td>8.593</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_cold_delta_minimum</th>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_cold_proportion</th>\n",
+              "      <td>0.968</td>\n",
+              "      <td>0.733</td>\n",
+              "      <td>0.757</td>\n",
+              "      <td>0.819</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_hot_delta_average</th>\n",
+              "      <td>0.013</td>\n",
+              "      <td>0.093</td>\n",
+              "      <td>0.013</td>\n",
+              "      <td>0.039</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_hot_delta_maximum</th>\n",
+              "      <td>2.547</td>\n",
+              "      <td>5.028</td>\n",
+              "      <td>3.506</td>\n",
+              "      <td>3.694</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_hot_delta_minimum</th>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_hot_proportion</th>\n",
+              "      <td>0.003</td>\n",
+              "      <td>0.017</td>\n",
+              "      <td>0.003</td>\n",
+              "      <td>0.008</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_proportion</th>\n",
+              "      <td>0.971</td>\n",
+              "      <td>0.750</td>\n",
+              "      <td>0.760</td>\n",
+              "      <td>0.827</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>electricity_consumption_total</th>\n",
+              "      <td>1.643</td>\n",
+              "      <td>1.626</td>\n",
+              "      <td>1.479</td>\n",
+              "      <td>1.583</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>monthly_one_minus_load_factor_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>0.891</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>one_minus_thermal_resilience_proportion</th>\n",
+              "      <td>0.800</td>\n",
+              "      <td>0.571</td>\n",
+              "      <td>0.267</td>\n",
+              "      <td>0.546</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>power_outage_normalized_unserved_energy_total</th>\n",
+              "      <td>0.781</td>\n",
+              "      <td>0.796</td>\n",
+              "      <td>0.708</td>\n",
+              "      <td>0.762</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>ramping_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>0.885</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>zero_net_energy</th>\n",
+              "      <td>1.687</td>\n",
+              "      <td>1.643</td>\n",
+              "      <td>1.488</td>\n",
+              "      <td>1.606</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table>\n",
+              "</div>"
+            ],
+            "text/plain": [
+              "name                                           Building_1  Building_2  \\\n",
+              "cost_function                                                           \n",
+              "all_time_peak_average                                 NaN         NaN   \n",
+              "annual_normalized_unserved_energy_total             0.015       0.016   \n",
+              "carbon_emissions_total                              1.632       1.598   \n",
+              "cost_total                                          1.572       1.548   \n",
+              "daily_one_minus_load_factor_average                   NaN         NaN   \n",
+              "daily_peak_average                                    NaN         NaN   \n",
+              "discomfort_cold_delta_average                       7.743       2.896   \n",
+              "discomfort_cold_delta_maximum                      12.215       8.302   \n",
+              "discomfort_cold_delta_minimum                       0.000       0.000   \n",
+              "discomfort_cold_proportion                          0.968       0.733   \n",
+              "discomfort_hot_delta_average                        0.013       0.093   \n",
+              "discomfort_hot_delta_maximum                        2.547       5.028   \n",
+              "discomfort_hot_delta_minimum                        0.000       0.000   \n",
+              "discomfort_hot_proportion                           0.003       0.017   \n",
+              "discomfort_proportion                               0.971       0.750   \n",
+              "electricity_consumption_total                       1.643       1.626   \n",
+              "monthly_one_minus_load_factor_average                 NaN         NaN   \n",
+              "one_minus_thermal_resilience_proportion             0.800       0.571   \n",
+              "power_outage_normalized_unserved_energy_total       0.781       0.796   \n",
+              "ramping_average                                       NaN         NaN   \n",
+              "zero_net_energy                                     1.687       1.643   \n",
+              "\n",
+              "name                                           Building_3  District  \n",
+              "cost_function                                                        \n",
+              "all_time_peak_average                                 NaN     1.063  \n",
+              "annual_normalized_unserved_energy_total             0.014     0.015  \n",
+              "carbon_emissions_total                              1.459     1.563  \n",
+              "cost_total                                          1.427     1.516  \n",
+              "daily_one_minus_load_factor_average                   NaN     0.740  \n",
+              "daily_peak_average                                    NaN     1.166  \n",
+              "discomfort_cold_delta_average                       2.563     4.401  \n",
+              "discomfort_cold_delta_maximum                       5.262     8.593  \n",
+              "discomfort_cold_delta_minimum                       0.000     0.000  \n",
+              "discomfort_cold_proportion                          0.757     0.819  \n",
+              "discomfort_hot_delta_average                        0.013     0.039  \n",
+              "discomfort_hot_delta_maximum                        3.506     3.694  \n",
+              "discomfort_hot_delta_minimum                        0.000     0.000  \n",
+              "discomfort_hot_proportion                           0.003     0.008  \n",
+              "discomfort_proportion                               0.760     0.827  \n",
+              "electricity_consumption_total                       1.479     1.583  \n",
+              "monthly_one_minus_load_factor_average                 NaN     0.891  \n",
+              "one_minus_thermal_resilience_proportion             0.267     0.546  \n",
+              "power_outage_normalized_unserved_energy_total       0.708     0.762  \n",
+              "ramping_average                                       NaN     0.885  \n",
+              "zero_net_energy                                     1.488     1.606  "
+            ]
+          },
+          "metadata": {},
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "import warnings\n",
+        "from citylearn.wrappers import ClippedObservationWrapper, NormalizedObservationWrapper, RLlibSingleAgentWrapper\n",
+        "from ray.rllib.algorithms.sac import SACConfig as Config\n",
+        "\n",
+        "warnings.filterwarnings('ignore', category=DeprecationWarning)\n",
+        "\n",
+        "# initialize\n",
+        "env_config = {\n",
+        "    'env_kwargs': {\n",
+        "        'schema': 'citylearn_challenge_2023_phase_2_local_evaluation',\n",
+        "    },\n",
+        "    'wrappers': [\n",
+        "        NormalizedObservationWrapper, \n",
+        "        ClippedObservationWrapper\n",
+        "    ]\n",
+        "}\n",
+        "config = (\n",
+        "    Config()\n",
+        "    .environment(RLlibSingleAgentWrapper, env_config=env_config)\n",
+        ")\n",
+        "model = config.build()\n",
+        "\n",
+        "# train\n",
+        "for i in range(2):\n",
+        "    _ = model.train()\n",
+        "\n",
+        "# test\n",
+        "env = RLlibSingleAgentWrapper(env_config)\n",
+        "observations, _ = env.reset()\n",
+        "\n",
+        "while not env.unwrapped.terminated:\n",
+        "    actions = model.compute_single_action(observations, explore=False)\n",
+        "    observations, _, _, _, _ = env.step(actions)\n",
+        "\n",
+        "kpis = env.unwrapped.evaluate()\n",
+        "kpis = kpis.pivot(index='cost_function', columns='name', values='value').round(3)\n",
+        "kpis = kpis.dropna(how='all')\n",
+        "display(kpis)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Multi-agent\n",
+        "\n",
+        "The multi-agent interface for RLlib is the `RLlibMultiAgentEnv` wrapper."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 12,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "2024-04-01 20:56:03,937\tWARNING util.py:62 -- Install gputil for GPU system monitoring.\n"
+          ]
+        },
+        {
+          "data": {
+            "text/html": [
+              "<div>\n",
+              "<style scoped>\n",
+              "    .dataframe tbody tr th:only-of-type {\n",
+              "        vertical-align: middle;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe tbody tr th {\n",
+              "        vertical-align: top;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe thead th {\n",
+              "        text-align: right;\n",
+              "    }\n",
+              "</style>\n",
+              "<table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              "    <tr style=\"text-align: right;\">\n",
+              "      <th>name</th>\n",
+              "      <th>Building_1</th>\n",
+              "      <th>Building_2</th>\n",
+              "      <th>Building_3</th>\n",
+              "      <th>District</th>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>cost_function</th>\n",
+              "      <th></th>\n",
+              "      <th></th>\n",
+              "      <th></th>\n",
+              "      <th></th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <th>all_time_peak_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>1.063</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>annual_normalized_unserved_energy_total</th>\n",
+              "      <td>0.015</td>\n",
+              "      <td>0.016</td>\n",
+              "      <td>0.014</td>\n",
+              "      <td>0.015</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>carbon_emissions_total</th>\n",
+              "      <td>1.639</td>\n",
+              "      <td>1.599</td>\n",
+              "      <td>1.462</td>\n",
+              "      <td>1.566</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>cost_total</th>\n",
+              "      <td>1.579</td>\n",
+              "      <td>1.548</td>\n",
+              "      <td>1.431</td>\n",
+              "      <td>1.519</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>daily_one_minus_load_factor_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>0.739</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>daily_peak_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>1.168</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_cold_delta_average</th>\n",
+              "      <td>7.789</td>\n",
+              "      <td>2.896</td>\n",
+              "      <td>2.566</td>\n",
+              "      <td>4.417</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_cold_delta_maximum</th>\n",
+              "      <td>12.260</td>\n",
+              "      <td>8.308</td>\n",
+              "      <td>5.261</td>\n",
+              "      <td>8.610</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_cold_delta_minimum</th>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_cold_proportion</th>\n",
+              "      <td>0.968</td>\n",
+              "      <td>0.731</td>\n",
+              "      <td>0.759</td>\n",
+              "      <td>0.819</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_hot_delta_average</th>\n",
+              "      <td>0.013</td>\n",
+              "      <td>0.093</td>\n",
+              "      <td>0.013</td>\n",
+              "      <td>0.039</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_hot_delta_maximum</th>\n",
+              "      <td>2.521</td>\n",
+              "      <td>5.046</td>\n",
+              "      <td>3.506</td>\n",
+              "      <td>3.691</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_hot_delta_minimum</th>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "      <td>0.000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_hot_proportion</th>\n",
+              "      <td>0.003</td>\n",
+              "      <td>0.017</td>\n",
+              "      <td>0.003</td>\n",
+              "      <td>0.008</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>discomfort_proportion</th>\n",
+              "      <td>0.971</td>\n",
+              "      <td>0.748</td>\n",
+              "      <td>0.762</td>\n",
+              "      <td>0.827</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>electricity_consumption_total</th>\n",
+              "      <td>1.650</td>\n",
+              "      <td>1.626</td>\n",
+              "      <td>1.482</td>\n",
+              "      <td>1.586</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>monthly_one_minus_load_factor_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>0.890</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>one_minus_thermal_resilience_proportion</th>\n",
+              "      <td>0.800</td>\n",
+              "      <td>0.571</td>\n",
+              "      <td>0.267</td>\n",
+              "      <td>0.546</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>power_outage_normalized_unserved_energy_total</th>\n",
+              "      <td>0.782</td>\n",
+              "      <td>0.797</td>\n",
+              "      <td>0.708</td>\n",
+              "      <td>0.762</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>ramping_average</th>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>NaN</td>\n",
+              "      <td>0.885</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>zero_net_energy</th>\n",
+              "      <td>1.695</td>\n",
+              "      <td>1.643</td>\n",
+              "      <td>1.491</td>\n",
+              "      <td>1.609</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table>\n",
+              "</div>"
+            ],
+            "text/plain": [
+              "name                                           Building_1  Building_2  \\\n",
+              "cost_function                                                           \n",
+              "all_time_peak_average                                 NaN         NaN   \n",
+              "annual_normalized_unserved_energy_total             0.015       0.016   \n",
+              "carbon_emissions_total                              1.639       1.599   \n",
+              "cost_total                                          1.579       1.548   \n",
+              "daily_one_minus_load_factor_average                   NaN         NaN   \n",
+              "daily_peak_average                                    NaN         NaN   \n",
+              "discomfort_cold_delta_average                       7.789       2.896   \n",
+              "discomfort_cold_delta_maximum                      12.260       8.308   \n",
+              "discomfort_cold_delta_minimum                       0.000       0.000   \n",
+              "discomfort_cold_proportion                          0.968       0.731   \n",
+              "discomfort_hot_delta_average                        0.013       0.093   \n",
+              "discomfort_hot_delta_maximum                        2.521       5.046   \n",
+              "discomfort_hot_delta_minimum                        0.000       0.000   \n",
+              "discomfort_hot_proportion                           0.003       0.017   \n",
+              "discomfort_proportion                               0.971       0.748   \n",
+              "electricity_consumption_total                       1.650       1.626   \n",
+              "monthly_one_minus_load_factor_average                 NaN         NaN   \n",
+              "one_minus_thermal_resilience_proportion             0.800       0.571   \n",
+              "power_outage_normalized_unserved_energy_total       0.782       0.797   \n",
+              "ramping_average                                       NaN         NaN   \n",
+              "zero_net_energy                                     1.695       1.643   \n",
+              "\n",
+              "name                                           Building_3  District  \n",
+              "cost_function                                                        \n",
+              "all_time_peak_average                                 NaN     1.063  \n",
+              "annual_normalized_unserved_energy_total             0.014     0.015  \n",
+              "carbon_emissions_total                              1.462     1.566  \n",
+              "cost_total                                          1.431     1.519  \n",
+              "daily_one_minus_load_factor_average                   NaN     0.739  \n",
+              "daily_peak_average                                    NaN     1.168  \n",
+              "discomfort_cold_delta_average                       2.566     4.417  \n",
+              "discomfort_cold_delta_maximum                       5.261     8.610  \n",
+              "discomfort_cold_delta_minimum                       0.000     0.000  \n",
+              "discomfort_cold_proportion                          0.759     0.819  \n",
+              "discomfort_hot_delta_average                        0.013     0.039  \n",
+              "discomfort_hot_delta_maximum                        3.506     3.691  \n",
+              "discomfort_hot_delta_minimum                        0.000     0.000  \n",
+              "discomfort_hot_proportion                           0.003     0.008  \n",
+              "discomfort_proportion                               0.762     0.827  \n",
+              "electricity_consumption_total                       1.482     1.586  \n",
+              "monthly_one_minus_load_factor_average                 NaN     0.890  \n",
+              "one_minus_thermal_resilience_proportion             0.267     0.546  \n",
+              "power_outage_normalized_unserved_energy_total       0.708     0.762  \n",
+              "ramping_average                                       NaN     0.885  \n",
+              "zero_net_energy                                     1.491     1.609  "
+            ]
+          },
+          "metadata": {},
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "import warnings\n",
+        "from citylearn.wrappers import ClippedObservationWrapper, NormalizedObservationWrapper, RLlibMultiAgentEnv\n",
+        "from ray.rllib.algorithms.sac import SACConfig as Config\n",
+        "from ray.rllib.policy.policy import PolicySpec\n",
+        "\n",
+        "warnings.filterwarnings('ignore', category=DeprecationWarning)\n",
+        "\n",
+        "# initialize\n",
+        "env_config = {\n",
+        "    'env_kwargs': {\n",
+        "        'schema': 'citylearn_challenge_2023_phase_2_local_evaluation',\n",
+        "    },\n",
+        "    'wrappers': [\n",
+        "        NormalizedObservationWrapper, \n",
+        "        ClippedObservationWrapper\n",
+        "    ]\n",
+        "}\n",
+        "config = (\n",
+        "    Config()\n",
+        "    .environment(RLlibMultiAgentEnv, env_config=env_config)\n",
+        "    .multi_agent(\n",
+        "        policies={a: PolicySpec() for a in RLlibMultiAgentEnv(env_config)._agent_ids},\n",
+        "        policy_mapping_fn=lambda agent_id, episode, worker, **kwargs: agent_id,\n",
+        "    )\n",
+        ")\n",
+        "model = config.build()\n",
+        "\n",
+        "# train\n",
+        "for i in range(2):\n",
+        "    _ = model.train()\n",
+        "\n",
+        "# test\n",
+        "env = RLlibMultiAgentEnv(env_config)\n",
+        "observations, _ = env.reset()\n",
+        "\n",
+        "while not env.terminated:\n",
+        "    actions = {p: model.compute_single_action(o, policy_id=p, explore=False) for p, o in observations.items()}\n",
+        "    observations, _, _, _, _ = env.step(actions)\n",
+        "\n",
+        "kpis = env.unwrapped.evaluate()\n",
+        "kpis = kpis.pivot(index='cost_function', columns='name', values='value').round(3)\n",
+        "kpis = kpis.dropna(how='all')\n",
+        "display(kpis)"
+      ]
     }
   ],
   "metadata": {
@@ -1474,7 +2151,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.12.2"
+      "version": "3.11.8"
     },
     "orig_nbformat": 4,
     "vscode": {
diff --git a/requirements.txt b/requirements.txt
index c27e363b..23628661 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 gymnasium
 numpy
 pandas
+pyyaml
 scikit-learn
 simplejson
 torch
diff --git a/setup.py b/setup.py
index 4c99a8fd..b3421b07 100644
--- a/setup.py
+++ b/setup.py
@@ -22,14 +22,16 @@ def get_version():
     version=get_version(),
     author='Jose Ramon Vazquez-Canteli, Kingsley Nweye, Zoltan Nagy',
     author_email='nweye@utexas.edu',
-    description='An open source Farama Foundation Gymnasium environment for the implementation of Multi-Agent Reinforcement Learning (RL) for building energy coordination and demand response in cities.',
+    description=(
+        'An open source Farama Foundation Gymnasium environment for benchmarking distributed '
+        'energy resource control algorithms to provide energy flexibility in a district of buildings.'),
     long_description=long_description,
     long_description_content_type='text/markdown',
     url='https://github.com/intelligent-environments-lab/CityLearn',
     packages=setuptools.find_packages(),
     include_package_data=True,
     install_requires=requirements,
-    entry_points={'console_scripts': ['citylearn = citylearn.main:main']},
+    entry_points={'console_scripts': ['citylearn = citylearn.__main__:main']},
     classifiers=[
         'Programming Language :: Python :: 3',
         'License :: OSI Approved :: MIT License',


Test patch:
diff --git a/test_requirements.txt b/test_requirements.txt
index a020c9fd..4b5991a6 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -1,3 +1,4 @@
 -e .
 ipykernel
+ray[rllib]
 stable-baselines3