def get_object(obj, predefined_objects=None, default_object=None, kwargs=None):

args = ()<EOL>kwargs = dict() if kwargs is None else kwargs<EOL>if isinstance(obj, str) and os.path.isfile(obj):<EOL><INDENT>with open(obj, '<STR_LIT:r>') as fp:<EOL><INDENT>obj = json.load(fp=fp)<EOL><DEDENT><DEDENT>if isinstance(obj, dict):<EOL><INDENT>kwargs.update(obj)<EOL>obj = kwargs.pop('<STR_LIT:type>', None)<EOL><DEDENT>if predefined_objects is not None and obj in predefined_objects:<EOL><INDENT>obj = predefined_objects[obj]<EOL><DEDENT>elif isinstance(obj, str):<EOL><INDENT>if obj.find('<STR_LIT:.>') != -<NUM_LIT:1>:<EOL><INDENT>module_name, function_name = obj.rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>module = importlib.import_module(module_name)<EOL>obj = getattr(module, function_name)<EOL><DEDENT>else:<EOL><INDENT>raise TensorForceError("<STR_LIT>".format(<EOL>obj,<EOL>list(predefined_objects or ())<EOL>))<EOL><DEDENT><DEDENT>elif callable(obj):<EOL><INDENT>pass<EOL><DEDENT>elif default_object is not None:<EOL><INDENT>args = (obj,)<EOL>obj = default_object<EOL><DEDENT>else:<EOL><INDENT>return obj<EOL><DEDENT>return obj(*args, **kwargs)<EOL>

Utility method to map some kind of object specification to its content,
e.g. optimizer or baseline specifications to the respective classes.

Args:
    obj: A specification dict (value for key 'type' optionally specifies
            the object, options as follows), a module path (e.g.,
            my_module.MyClass), a key in predefined_objects, or a callable
            (e.g., the class type object).
    predefined_objects: Dict containing predefined set of objects,
            accessible via their key
    default_object: Default object is no other is specified
    kwargs: Arguments for object creation

Returns: The retrieved object


kwargs = dict()<EOL>if isinstance(raw, dict):<EOL><INDENT>kwargs.update(raw)<EOL><DEDENT>elif isinstance(raw, str):<EOL><INDENT>kwargs[string_parameter] = raw<EOL><DEDENT>return kwargs<EOL>

Utility method to convert raw string/diction input into a dictionary to pass
into a function.  Always returns a dictionary.

Args:
    raw: string or dictionary, string is assumed to be the name of the activation
            activation function.  Dictionary will be passed through unchanged.

Returns: kwargs dictionary for **kwargs


variables = self.get_savable_variables()<EOL>if variables is None or len(variables) == <NUM_LIT:0>:<EOL><INDENT>self._saver = None<EOL>return<EOL><DEDENT>base_scope = self._get_base_variable_scope()<EOL>variables_map = {strip_name_scope(v.name, base_scope): v for v in variables}<EOL>self._saver = tf.train.Saver(<EOL>var_list=variables_map,<EOL>reshape=False,<EOL>sharded=False,<EOL>max_to_keep=<NUM_LIT:5>,<EOL>keep_checkpoint_every_n_hours=<NUM_LIT>,<EOL>name=None,<EOL>restore_sequentially=False,<EOL>saver_def=None,<EOL>builder=None,<EOL>defer_build=False,<EOL>allow_empty=True,<EOL>write_version=tf.train.SaverDef.V2,<EOL>pad_step_number=False,<EOL>save_relative_paths=True<EOL>)<EOL>

Registers the saver operations to the graph in context.


raise NotImplementedError()<EOL>

Returns the list of all the variables this component is responsible to save and restore.

Returns:
    The list of variables that will be saved or restored.


def save(self, sess, save_path, timestep=None):

if self._saver is None:<EOL><INDENT>raise TensorForceError("<STR_LIT>")<EOL><DEDENT>return self._saver.save(<EOL>sess=sess,<EOL>save_path=save_path,<EOL>global_step=timestep,<EOL>write_meta_graph=False,<EOL>write_state=True,  <EOL>)<EOL>

Saves this component's managed variables.

Args:
    sess: The session for which to save the managed variables.
    save_path: The path to save data to.
    timestep: Optional, the timestep to append to the file name.

Returns:
    Checkpoint path where the model was saved.


if self._saver is None:<EOL><INDENT>raise TensorForceError("<STR_LIT>")<EOL><DEDENT>self._saver.restore(sess=sess, save_path=save_path)<EOL>

Restores the values of the managed variables from disk location.

Args:
    sess: The session for which to save the managed variables.
    save_path: The path used to save the data to.


raise NotImplementedError()<EOL>

Returns the portion of the variable scope that is considered a base for this component. The variables will be
saved with names relative to that scope.

Returns:
    The name of the base variable scope, should always end with "/".


<EOL>if isinstance(agent_class, str):<EOL><INDENT>agent_class = AgentsDictionary.get(agent_class)<EOL>if not agent_class and agent_class.find('<STR_LIT:.>') != -<NUM_LIT:1>:<EOL><INDENT>module_name, function_name = agent_class.rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>module = importlib.import_module(module_name)<EOL>agent_class = getattr(module, function_name)<EOL><DEDENT><DEDENT>class WorkerAgent(agent_class):<EOL><INDENT>"""<STR_LIT>"""<EOL>def __init__(self, model=None, **kwargs):<EOL><INDENT>self.model = model<EOL>if not issubclass(agent_class, LearningAgent):<EOL><INDENT>kwargs.pop("<STR_LIT>")<EOL><DEDENT>super(WorkerAgent, self).__init__(**kwargs)<EOL><DEDENT>def initialize_model(self):<EOL><INDENT>return self.model<EOL><DEDENT><DEDENT>return WorkerAgent<EOL>

Worker Agent generator, receives an Agent class and creates a Worker Agent class that inherits from that Agent.


def clone_worker_agent(agent, factor, environment, network, agent_config):

ret = [agent]<EOL>for i in xrange(factor - <NUM_LIT:1>):<EOL><INDENT>worker = WorkerAgentGenerator(type(agent))(<EOL>states=environment.states,<EOL>actions=environment.actions,<EOL>network=network,<EOL>model=agent.model,<EOL>**agent_config<EOL>)<EOL>ret.append(worker)<EOL><DEDENT>return ret<EOL>

Clones a given Agent (`factor` times) and returns a list of the cloned Agents with the original Agent
in the first slot.

Args:
    agent (Agent): The Agent object to clone.
    factor (int): The length of the final list.
    environment (Environment): The Environment to use for all cloned agents.
    network (LayeredNetwork): The Network to use (or None) for an Agent's Model.
    agent_config (dict): A dict of Agent specifications passed into the Agent's c'tor as kwargs.
Returns:
    The list with `factor` cloned agents (including the original one).


def __init__(self, agent, environment, repeat_actions=<NUM_LIT:1>, save_path=None, save_episodes=None, save_frequency=None,<EOL>save_frequency_unit=None, agents=None, environments=None):

if agents is not None:<EOL><INDENT>warnings.warn("<STR_LIT>",<EOL>category=DeprecationWarning)<EOL>agent = agents<EOL><DEDENT>if environments is not None:<EOL><INDENT>warnings.warn("<STR_LIT>",<EOL>category=DeprecationWarning)<EOL>environment = environments<EOL><DEDENT>super(ThreadedRunner, self).__init__(agent, environment, repeat_actions)<EOL>if len(agent) != len(environment):<EOL><INDENT>raise TensorForceError("<STR_LIT>".<EOL>format(a=len(self.agent), e=len(self.environment)))<EOL><DEDENT>self.save_path = save_path<EOL>self.save_episodes = save_episodes<EOL>if self.save_episodes is not None:<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>category=DeprecationWarning)<EOL>self.save_frequency = self.save_episodes<EOL>self.save_frequency_unit = "<STR_LIT:e>"<EOL><DEDENT>else:<EOL><INDENT>self.save_frequency = save_frequency<EOL>self.save_frequency_unit = save_frequency_unit<EOL><DEDENT>self.episode_list_lock = threading.Lock()<EOL>self.should_stop = False<EOL>self.time = None<EOL>

Initialize a ThreadedRunner object.

Args:
    save_path (str): Path where to save the shared model.
    save_episodes (int): Deprecated: Every how many (global) episodes do we save the shared model?
    save_frequency (int): The frequency with which to save the model (could be sec, steps, or episodes).
    save_frequency_unit (str): "s" (sec), "t" (timesteps), "e" (episodes)
    agents (List[Agent]): Deprecated: List of Agent objects. Use `agent`, instead.
    environments (List[Environment]): Deprecated: List of Environment objects. Use `environment`, instead.


def __init__(self, agent, environment, repeat_actions=<NUM_LIT:1>, history=None):

self.agent = agent<EOL>self.environment = environment<EOL>self.repeat_actions = repeat_actions<EOL>self.global_episode = None  <EOL>self.global_timestep = None  <EOL>self.start_time = None  <EOL>self.episode_rewards = None  <EOL>self.episode_timesteps = None  <EOL>self.episode_times = None  <EOL>self.reset(history)<EOL>

Args:
    agent (Agent): Agent object (or list of Agent objects) to use for the run.
    environment (Environment): Environment object (or list of Environment objects) to use for the run.
    repeat_actions (int): How many times the same given action will be repeated in subsequent calls to
        Environment's `execute` method. Rewards collected in these calls are accumulated and reported
        as a sum in the following call to Agent's `observe` method.
    history (dict): A dictionary containing an already run experiment's results. Keys should be:
        episode_rewards (list of rewards), episode_timesteps (lengths of episodes), episode_times (run-times)


if not history:<EOL><INDENT>history = dict()<EOL><DEDENT>self.episode_rewards = history.get("<STR_LIT>", list())<EOL>self.episode_timesteps = history.get("<STR_LIT>", list())<EOL>self.episode_times = history.get("<STR_LIT>", list())<EOL>

Resets the Runner's internal stats counters.
If history is empty, use default values in history.get().

Args:
    history (dict): A dictionary containing an already run experiment's results. Keys should be:
        episode_rewards (list of rewards), episode_timesteps (lengths of episodes), episode_times (run-times)


raise NotImplementedError<EOL>

Should perform clean up operations on Runner's Agent(s) and Environment(s).


def run(self, num_episodes, num_timesteps, max_episode_timesteps, deterministic, episode_finished, summary_report,<EOL>summary_interval):

raise NotImplementedError<EOL>

Executes this runner by starting to act (via Agent(s)) in the given Environment(s).
Stops execution according to certain conditions (e.g. max. number of episodes, etc..).
Calls callback functions after each episode and/or after some summary criteria are met.

Args:
    num_episodes (int): Max. number of episodes to run globally in total (across all threads/workers).
    num_timesteps (int): Max. number of time steps to run globally in total (across all threads/workers)
    max_episode_timesteps (int): Max. number of timesteps per episode.
    deterministic (bool): Whether to use exploration when selecting actions.
    episode_finished (callable): A function to be called once an episodes has finished. Should take
        a BaseRunner object and some worker ID (e.g. thread-ID or task-ID). Can decide for itself
        every how many episodes it should report something and what to report.
    summary_report (callable): Deprecated; Function that could produce a summary over the training
        progress so far.
    summary_interval (int): Deprecated; The number of time steps to execute (globally)
        before summary_report is called.


return self.global_episode<EOL>

Deprecated property `episode` -> global_episode.


return self.global_timestep<EOL>

Deprecated property `timestep` -> global_timestep.


def __init__(self, agent, environment, repeat_actions=<NUM_LIT:1>, history=None, id_=<NUM_LIT:0>):

super(ParallelRunner, self).__init__(agent, environment, repeat_actions, history)<EOL>self.id = id_  <EOL>self.current_timestep = None  <EOL>self.episode_actions = []<EOL>self.num_parallel = self.agent.execution['<STR_LIT>']<EOL>print('<STR_LIT>'.format(self.num_parallel))<EOL>

Initialize a single Runner object (one Agent/one Environment).

Args:
    id_ (int): The ID of this Runner (for distributed TF runs).


super(Runner, self).__init__(agent, environment, repeat_actions, history)<EOL>self.id = id_  <EOL>self.current_timestep = None<EOL>


self.specification = dict()<EOL>for action_type, shape in specification.items():<EOL><INDENT>if action_type in ('<STR_LIT:bool>', '<STR_LIT:int>', '<STR_LIT:float>', '<STR_LIT>'):<EOL><INDENT>if isinstance(shape, int):<EOL><INDENT>self.specification[action_type] = (shape,)<EOL><DEDENT>else:<EOL><INDENT>self.specification[action_type] = tuple(shape)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TensorForceError('<STR_LIT>')<EOL><DEDENT><DEDENT>self.single_state_action = (len(specification) == <NUM_LIT:1>)<EOL>

Initializes a minimal test environment, which is used for the unit tests.
Given a specification of actions types and shapes, the environment states consist
of the same number of pairs (x, y). The (mean of) an action a gives the next state via (1-a, a),
and the 'correct' state is always (0, 1).

Args:
    specification: Takes a dict type (keys)-> shape (values specifying the action
        structure of the environment. Use shape () for single scalar actions.


pass<EOL>


pass<EOL>

Close environment. No other method calls possible afterwards.


return None<EOL>

Sets the random seed of the environment to the given value (current time, if seed=None).
Naturally deterministic Environments (e.g. ALE or some gym Envs) don't have to implement this method.

Args:
    seed (int): The seed to use for initializing the pseudo-random number generator (default=epoch time in sec).
Returns: The actual seed (int) used OR None if Environment did not override this method (no seeding supported).


raise NotImplementedError<EOL>

Reset environment and setup for new episode.

Returns:
    initial state of reset environment.


raise NotImplementedError<EOL>

Executes action, observes next state(s) and reward.

Args:
    actions: Actions to execute.

Returns:
    Tuple of (next state, bool indicating terminal, reward)


raise NotImplementedError<EOL>

Return the state space. Might include subdicts if multiple states are
available simultaneously.

Returns:
    States specification, with the following attributes
        (required):
        - type: one of 'bool', 'int', 'float' (default: 'float').
        - shape: integer, or list/tuple of integers (required).


raise NotImplementedError<EOL>

Return the action space. Might include subdicts if multiple actions are
available simultaneously.

Returns:
    actions (spec, or dict of specs): Actions specification, with the following attributes
        (required):
        - type: one of 'bool', 'int', 'float' (required).
        - shape: integer, or list/tuple of integers (default: []).
        - num_actions: integer (required if type == 'int').
        - min_value and max_value: float (optional if type == 'float', default: none).


env = tensorforce.util.get_object(<EOL>obj=spec,<EOL>predefined_objects=tensorforce.environments.environments,<EOL>kwargs=kwargs<EOL>)<EOL>assert isinstance(env, Environment)<EOL>return env<EOL>

Creates an environment from a specification dict.


<EOL>self.network = Network.from_spec(<EOL>spec=self.network_spec,<EOL>kwargs=dict(summary_labels=self.summary_labels)<EOL>)<EOL>assert len(self.internals_spec) == <NUM_LIT:0><EOL>self.internals_spec = self.network.internals_spec()<EOL>for name in sorted(self.internals_spec):<EOL><INDENT>internal = self.internals_spec[name]<EOL>self.internals_input[name] = tf.placeholder(<EOL>dtype=util.tf_dtype(internal['<STR_LIT:type>']),<EOL>shape=(None,) + tuple(internal['<STR_LIT>']),<EOL>name=('<STR_LIT>' + name)<EOL>)<EOL>if internal['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>self.internals_init[name] = np.zeros(shape=internal['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>raise TensorForceError("<STR_LIT>")<EOL><DEDENT><DEDENT>custom_getter = super(DistributionModel, self).setup_components_and_tf_funcs(custom_getter)<EOL>self.distributions = self.create_distributions()<EOL>self.fn_kl_divergence = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_kl_divergence,<EOL>custom_getter_=custom_getter<EOL>)<EOL>return custom_getter<EOL>

Creates and stores Network and Distribution objects.
Generates and stores all template functions.


distributions = dict()<EOL>for name in sorted(self.actions_spec):<EOL><INDENT>action = self.actions_spec[name]<EOL>if self.distributions_spec is not None and name in self.distributions_spec:<EOL><INDENT>kwargs = dict(action)<EOL>kwargs['<STR_LIT>'] = name<EOL>kwargs['<STR_LIT>'] = self.summary_labels<EOL>distributions[name] = Distribution.from_spec(<EOL>spec=self.distributions_spec[name],<EOL>kwargs=kwargs<EOL>)<EOL><DEDENT>elif action['<STR_LIT:type>'] == '<STR_LIT:bool>':<EOL><INDENT>distributions[name] = Bernoulli(<EOL>shape=action['<STR_LIT>'],<EOL>scope=name,<EOL>summary_labels=self.summary_labels<EOL>)<EOL><DEDENT>elif action['<STR_LIT:type>'] == '<STR_LIT:int>':<EOL><INDENT>distributions[name] = Categorical(<EOL>shape=action['<STR_LIT>'],<EOL>num_actions=action['<STR_LIT>'],<EOL>scope=name,<EOL>summary_labels=self.summary_labels<EOL>)<EOL><DEDENT>elif action['<STR_LIT:type>'] == '<STR_LIT:float>':<EOL><INDENT>if '<STR_LIT>' in action:<EOL><INDENT>distributions[name] = Beta(<EOL>shape=action['<STR_LIT>'],<EOL>min_value=action['<STR_LIT>'],<EOL>max_value=action['<STR_LIT>'],<EOL>scope=name,<EOL>summary_labels=self.summary_labels<EOL>)<EOL><DEDENT>else:<EOL><INDENT>distributions[name] = Gaussian(<EOL>shape=action['<STR_LIT>'],<EOL>scope=name,<EOL>summary_labels=self.summary_labels<EOL>)<EOL><DEDENT><DEDENT><DEDENT>return distributions<EOL>

Creates and returns the Distribution objects based on self.distributions_spec.

Returns: Dict of distributions according to self.distributions_spec.


<EOL>self.network = None<EOL>self.states_spec = states<EOL>self.internals_spec = dict()<EOL>self.actions_spec = actions<EOL>self.scope = scope<EOL>self.device = device<EOL>if saver is None or saver.get('<STR_LIT>') is None:<EOL><INDENT>self.saver_spec = None<EOL><DEDENT>else:<EOL><INDENT>self.saver_spec = saver<EOL><DEDENT>if summarizer is None or summarizer.get('<STR_LIT>') is None:<EOL><INDENT>self.summarizer_spec = None<EOL>self.summary_labels = set()<EOL><DEDENT>else:<EOL><INDENT>self.summarizer_spec = summarizer<EOL>self.summary_labels = set(self.summarizer_spec.get('<STR_LIT>', ()))<EOL><DEDENT>self.summarizer = None<EOL>self.graph_summary = None<EOL>self.summarizer_init_op = None<EOL>self.flush_summarizer = None<EOL>self.execution_spec = execution<EOL>self.execution_type = self.execution_spec["<STR_LIT:type>"]<EOL>self.session_config = self.execution_spec["<STR_LIT>"]<EOL>self.distributed_spec = self.execution_spec["<STR_LIT>"]<EOL>assert batching_capacity is None or (isinstance(batching_capacity, int) and batching_capacity > <NUM_LIT:0>)<EOL>self.batching_capacity = batching_capacity or <NUM_LIT:1>  <EOL>self.tf_session_dump_dir = tf_session_dump_dir<EOL>self.num_parallel = self.execution_spec.get('<STR_LIT>')<EOL>if self.num_parallel is None:<EOL><INDENT>self.num_parallel = <NUM_LIT:1><EOL><DEDENT>self.list_states_buffer = dict()<EOL>self.list_internals_buffer = dict()<EOL>self.list_actions_buffer = dict()<EOL>self.list_buffer_index = [None for _ in range(self.num_parallel)]<EOL>self.episode_output = None<EOL>self.episode_index_input = None<EOL>self.unbuffered_episode_output = None<EOL>assert variable_noise is None or variable_noise > <NUM_LIT:0.0><EOL>self.variable_noise = variable_noise<EOL>self.states_preprocessing_spec = states_preprocessing<EOL>self.actions_exploration_spec = actions_exploration<EOL>self.reward_preprocessing_spec = reward_preprocessing<EOL>self.variables = None<EOL>self.all_variables = None<EOL>self.registered_variables = None<EOL>self.list_timestep = [None for _ in range(self.num_parallel)]<EOL>self.episode = None<EOL>self.global_timestep = None<EOL>self.global_episode = None<EOL>self.states_input = dict()<EOL>self.states_preprocessing = dict()<EOL>self.internals_input = dict()<EOL>self.internals_init = dict()<EOL>self.actions_input = dict()<EOL>self.actions_exploration = dict()<EOL>self.terminal_input = None<EOL>self.reward_input = None<EOL>self.reward_preprocessing = None<EOL>self.deterministic_input = None<EOL>self.independent_input = None<EOL>self.update_input = None<EOL>self.fn_initialize = None<EOL>self.fn_preprocess = None<EOL>self.fn_actions_and_internals = None<EOL>self.fn_observe_timestep = None<EOL>self.fn_action_exploration = None<EOL>self.graph = None<EOL>self.global_model = None<EOL>self.is_local_model = True<EOL>self.server = None<EOL>self.summarizer = None<EOL>self.saver = None<EOL>self.saver_directory = None<EOL>self.scaffold = None<EOL>self.session = None<EOL>self.monitored_session = None<EOL>self.actions_output = None<EOL>self.internals_output = None<EOL>self.timestep_output = None<EOL>self.list_buffer_index_reset_op = None<EOL>self.setup()<EOL>

Model.

Args:
    states (spec): The state-space description dictionary.
    actions (spec): The action-space description dictionary.
    scope (str): The root scope str to use for tf variable scoping.
    device (str): The name of the device to run the graph of this model on.
    saver (spec): Dict specifying whether and how to save the model's parameters.
    summarizer (spec): Dict specifying which tensorboard summaries should be created and added to the graph.
    execution (spec): Dict specifying whether and how to do distributed training on the model's graph.
    batching_capacity (int): Batching capacity.
    variable_noise (float): The stddev value of a Normal distribution used for adding random
        noise to the model's output (for each batch, noise can be toggled and - if active - will be resampled).
        Use None for not adding any noise.
    states_preprocessing (spec / dict of specs): Dict specifying whether and how to preprocess state signals
        (e.g. normalization, greyscale, etc..).
    actions_exploration (spec / dict of specs): Dict specifying whether and how to add exploration to the model's
        "action outputs" (e.g. epsilon-greedy).
    reward_preprocessing (spec): Dict specifying whether and how to preprocess rewards coming
        from the Environment (e.g. reward normalization).
    tf_session_dump_dir (str): If non-empty string, all session.run calls will be dumped using the tensorflow
        offline-debug session into the given directory.
    execution: (dict)
        - num_parallel: (int) number of parallel episodes


<EOL>graph_default_context = self.setup_graph()<EOL>if self.execution_type == "<STR_LIT>" and self.server is None and self.is_local_model:<EOL><INDENT>self.start_server()<EOL><DEDENT>with tf.device(device_name_or_function=self.device):<EOL><INDENT>with tf.variable_scope(name_or_scope=self.scope, reuse=False):<EOL><INDENT>self.variables = dict()<EOL>self.all_variables = dict()<EOL>self.registered_variables = set()<EOL>self.setup_placeholders()<EOL>self.setup_components_and_tf_funcs()<EOL>self.fn_initialize()<EOL>if self.summarizer_spec is not None:<EOL><INDENT>with tf.name_scope(name='<STR_LIT>'):<EOL><INDENT>self.summarizer = tf.contrib.summary.create_file_writer(<EOL>logdir=self.summarizer_spec['<STR_LIT>'],<EOL>max_queue=None,<EOL>flush_millis=(self.summarizer_spec.get('<STR_LIT>', <NUM_LIT:10>) * <NUM_LIT:1000>),<EOL>filename_suffix=None,<EOL>name=None<EOL>)<EOL>default_summarizer = self.summarizer.as_default()<EOL>assert '<STR_LIT>' not in self.summarizer_spec<EOL>record_summaries = tf.contrib.summary.always_record_summaries()<EOL><DEDENT>default_summarizer.__enter__()<EOL>record_summaries.__enter__()<EOL><DEDENT>states = util.map_tensors(fn=tf.identity, tensors=self.states_input)<EOL>internals = util.map_tensors(fn=tf.identity, tensors=self.internals_input)<EOL>actions = util.map_tensors(fn=tf.identity, tensors=self.actions_input)<EOL>terminal = tf.identity(input=self.terminal_input)<EOL>reward = tf.identity(input=self.reward_input)<EOL>deterministic = tf.identity(input=self.deterministic_input)<EOL>independent = tf.identity(input=self.independent_input)<EOL>episode_index = tf.identity(input=self.episode_index_input)<EOL>states, actions, reward = self.fn_preprocess(states=states, actions=actions, reward=reward)<EOL>self.create_operations(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward,<EOL>deterministic=deterministic,<EOL>independent=independent,<EOL>index=episode_index<EOL>)<EOL>if '<STR_LIT>' in self.summary_labels or '<STR_LIT>' in self.summary_labels:<EOL><INDENT>for name in sorted(states):<EOL><INDENT>tf.contrib.summary.histogram(name=('<STR_LIT>' + name), tensor=states[name])<EOL><DEDENT><DEDENT>if '<STR_LIT>' in self.summary_labels or '<STR_LIT>' in self.summary_labels:<EOL><INDENT>for name in sorted(actions):<EOL><INDENT>tf.contrib.summary.histogram(name=('<STR_LIT>' + name), tensor=actions[name])<EOL><DEDENT><DEDENT>if '<STR_LIT>' in self.summary_labels or '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name='<STR_LIT>', tensor=reward)<EOL><DEDENT>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>with tf.name_scope(name='<STR_LIT>'):<EOL><INDENT>graph_def = self.graph.as_graph_def()<EOL>graph_str = tf.constant(<EOL>value=graph_def.SerializeToString(),<EOL>dtype=tf.string,<EOL>shape=()<EOL>)<EOL>self.graph_summary = tf.contrib.summary.graph(<EOL>param=graph_str,<EOL>step=self.global_timestep<EOL>)<EOL>if '<STR_LIT>' in self.summarizer_spec:<EOL><INDENT>self.graph_summary = tf.group(<EOL>self.graph_summary,<EOL>*self.summarizer_spec['<STR_LIT>'].build_metagraph_list()<EOL>)<EOL><DEDENT><DEDENT><DEDENT>if self.summarizer_spec is not None:<EOL><INDENT>record_summaries.__exit__(None, None, None)<EOL>default_summarizer.__exit__(None, None, None)<EOL>with tf.name_scope(name='<STR_LIT>'):<EOL><INDENT>self.flush_summarizer = tf.contrib.summary.flush()<EOL>self.summarizer_init_op = tf.contrib.summary.summary_writer_initializer_op()<EOL>assert len(self.summarizer_init_op) == <NUM_LIT:1><EOL>self.summarizer_init_op = self.summarizer_init_op[<NUM_LIT:0>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if self.execution_type == "<STR_LIT>" and not self.is_local_model:<EOL><INDENT>return<EOL><DEDENT>self.setup_saver()<EOL>self.setup_scaffold()<EOL>hooks = self.setup_hooks()<EOL>self.setup_session(self.server, hooks, graph_default_context)<EOL>

Sets up the TensorFlow model graph, starts the servers (distributed mode), creates summarizers
and savers, initializes (and enters) the TensorFlow session.


graph_default_context = None<EOL>if self.execution_type == "<STR_LIT>":<EOL><INDENT>self.graph = tf.Graph()<EOL>graph_default_context = self.graph.as_default()<EOL>graph_default_context.__enter__()<EOL>self.global_model = None<EOL><DEDENT>elif self.execution_type == "<STR_LIT>":<EOL><INDENT>if self.distributed_spec["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>return None<EOL><DEDENT>elif self.distributed_spec["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>if self.is_local_model:<EOL><INDENT>graph = tf.Graph()<EOL>graph_default_context = graph.as_default()<EOL>graph_default_context.__enter__()<EOL>self.global_model = deepcopy(self)<EOL>self.global_model.is_local_model = False<EOL>self.global_model.setup()<EOL>self.graph = graph<EOL>self.as_local_model()<EOL>self.scope += '<STR_LIT>' + str(self.distributed_spec["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>self.graph = tf.get_default_graph()  <EOL>self.global_model = None<EOL>self.device = tf.train.replica_device_setter(<EOL>worker_device=self.device,<EOL>cluster=self.distributed_spec["<STR_LIT>"]<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TensorForceError("<STR_LIT>".format(self.distributed_spec["<STR_LIT>"]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TensorForceError("<STR_LIT>".format(self.distributed_spec["<STR_LIT:type>"]))<EOL><DEDENT>return graph_default_context<EOL>

Creates our Graph and figures out, which shared/global model to hook up to.
If we are in a global-model's setup procedure, we do not create
a new graph (return None as the context). We will instead use the already existing local replica graph
of the model.

Returns: None or the graph's as_default()-context.


Creates and stores a tf server (and optionally joins it if we are a parameter-server).
Only relevant, if we are running in distributed mode.


<EOL>for name in sorted(self.states_spec):<EOL><INDENT>self.states_input[name] = tf.placeholder(<EOL>dtype=util.tf_dtype(self.states_spec[name]['<STR_LIT:type>']),<EOL>shape=(None,) + tuple(self.states_spec[name]['<STR_LIT>']),<EOL>name=('<STR_LIT>' + name)<EOL>)<EOL><DEDENT>if self.states_preprocessing_spec is None:<EOL><INDENT>for name in sorted(self.states_spec):<EOL><INDENT>self.states_spec[name]['<STR_LIT>'] = self.states_spec[name]['<STR_LIT>']<EOL><DEDENT><DEDENT>elif not isinstance(self.states_preprocessing_spec, list) andall(name in self.states_spec for name in self.states_preprocessing_spec):<EOL><INDENT>for name in sorted(self.states_spec):<EOL><INDENT>if name in self.states_preprocessing_spec:<EOL><INDENT>preprocessing = PreprocessorStack.from_spec(<EOL>spec=self.states_preprocessing_spec[name],<EOL>kwargs=dict(shape=self.states_spec[name]['<STR_LIT>'])<EOL>)<EOL>self.states_spec[name]['<STR_LIT>'] = self.states_spec[name]['<STR_LIT>']<EOL>self.states_spec[name]['<STR_LIT>'] = preprocessing.processed_shape(shape=self.states_spec[name]['<STR_LIT>'])<EOL>self.states_preprocessing[name] = preprocessing<EOL><DEDENT>else:<EOL><INDENT>self.states_spec[name]['<STR_LIT>'] = self.states_spec[name]['<STR_LIT>']<EOL><DEDENT><DEDENT><DEDENT>elif "<STR_LIT:type>" in self.states_preprocessing_spec:<EOL><INDENT>preprocessing = PreprocessorStack.from_spec(spec=self.states_preprocessing_spec,<EOL>kwargs=dict(shape=self.states_spec[name]['<STR_LIT>']))<EOL>for name in sorted(self.states_spec):<EOL><INDENT>self.states_spec[name]['<STR_LIT>'] = self.states_spec[name]['<STR_LIT>']<EOL>self.states_spec[name]['<STR_LIT>'] = preprocessing.processed_shape(shape=self.states_spec[name]['<STR_LIT>'])<EOL>self.states_preprocessing[name] = preprocessing<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for name in sorted(self.states_spec):<EOL><INDENT>preprocessing = PreprocessorStack.from_spec(<EOL>spec=self.states_preprocessing_spec,<EOL>kwargs=dict(shape=self.states_spec[name]['<STR_LIT>'])<EOL>)<EOL>self.states_spec[name]['<STR_LIT>'] = self.states_spec[name]['<STR_LIT>']<EOL>self.states_spec[name]['<STR_LIT>'] = preprocessing.processed_shape(shape=self.states_spec[name]['<STR_LIT>'])<EOL>self.states_preprocessing[name] = preprocessing<EOL><DEDENT><DEDENT>for name in sorted(self.actions_spec):<EOL><INDENT>self.actions_input[name] = tf.placeholder(<EOL>dtype=util.tf_dtype(self.actions_spec[name]['<STR_LIT:type>']),<EOL>shape=(None,) + tuple(self.actions_spec[name]['<STR_LIT>']),<EOL>name=('<STR_LIT>' + name)<EOL>)<EOL><DEDENT>if self.actions_exploration_spec is None:<EOL><INDENT>pass<EOL><DEDENT>elif all(name in self.actions_spec for name in self.actions_exploration_spec):<EOL><INDENT>for name in sorted(self.actions_spec):<EOL><INDENT>if name in self.actions_exploration:<EOL><INDENT>self.actions_exploration[name] = Exploration.from_spec(spec=self.actions_exploration_spec[name])<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for name in sorted(self.actions_spec):<EOL><INDENT>self.actions_exploration[name] = Exploration.from_spec(spec=self.actions_exploration_spec)<EOL><DEDENT><DEDENT>self.terminal_input = tf.placeholder(dtype=util.tf_dtype('<STR_LIT:bool>'), shape=(None,), name='<STR_LIT>')<EOL>self.reward_input = tf.placeholder(dtype=util.tf_dtype('<STR_LIT:float>'), shape=(None,), name='<STR_LIT>')<EOL>if self.reward_preprocessing_spec is not None:<EOL><INDENT>self.reward_preprocessing = PreprocessorStack.from_spec(<EOL>spec=self.reward_preprocessing_spec,<EOL>kwargs=dict(shape=())<EOL>)<EOL>if self.reward_preprocessing.processed_shape(shape=()) != ():<EOL><INDENT>raise TensorForceError("<STR_LIT>")<EOL><DEDENT><DEDENT>self.deterministic_input = tf.placeholder(dtype=util.tf_dtype('<STR_LIT:bool>'), shape=(), name='<STR_LIT>')<EOL>self.independent_input = tf.placeholder(dtype=util.tf_dtype('<STR_LIT:bool>'), shape=(), name='<STR_LIT>')<EOL>

Creates the TensorFlow placeholders, variables, ops and functions for this model.
NOTE: Does not add the internal state placeholders and initialization values to the model yet as that requires
the model's Network (if any) to be generated first.


if custom_getter is None:<EOL><INDENT>def custom_getter(getter, name, registered=False, **kwargs):<EOL><INDENT>"""<STR_LIT>"""<EOL>if registered:<EOL><INDENT>self.registered_variables.add(name)<EOL><DEDENT>elif name in self.registered_variables:<EOL><INDENT>registered = True<EOL><DEDENT>variable = getter(name=name, **kwargs)<EOL>if registered:<EOL><INDENT>pass<EOL><DEDENT>elif name in self.all_variables:<EOL><INDENT>assert variable is self.all_variables[name]<EOL>if kwargs.get('<STR_LIT>', True):<EOL><INDENT>assert variable is self.variables[name]<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name=name, tensor=variable)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self.all_variables[name] = variable<EOL>if kwargs.get('<STR_LIT>', True):<EOL><INDENT>self.variables[name] = variable<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name=name, tensor=variable)<EOL><DEDENT><DEDENT><DEDENT>return variable<EOL><DEDENT><DEDENT>self.fn_initialize = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_initialize,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.fn_preprocess = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_preprocess,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.fn_actions_and_internals = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_actions_and_internals,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.fn_observe_timestep = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_observe_timestep,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.fn_action_exploration = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_action_exploration,<EOL>custom_getter_=custom_getter<EOL>)<EOL>return custom_getter<EOL>

Allows child models to create model's component objects, such as optimizer(s), memory(s), etc..
Creates all tensorflow functions via tf.make_template calls on all the class' "tf_"-methods.

Args:
    custom_getter: The `custom_getter_` object to use for `tf.make_template` when creating TensorFlow functions.
        If None, use a default custom_getter_.

Returns: The custom_getter passed in (or a default one if custom_getter was None).


if self.execution_type == "<STR_LIT>":<EOL><INDENT>global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)<EOL><DEDENT>else:<EOL><INDENT>global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)<EOL><DEDENT>for c in self.get_savable_components():<EOL><INDENT>c.register_saver_ops()<EOL><DEDENT>self.saver = tf.train.Saver(<EOL>var_list=global_variables,  <EOL>reshape=False,<EOL>sharded=False,<EOL>max_to_keep=<NUM_LIT:5>,<EOL>keep_checkpoint_every_n_hours=<NUM_LIT>,<EOL>name=None,<EOL>restore_sequentially=False,<EOL>saver_def=None,<EOL>builder=None,<EOL>defer_build=False,<EOL>allow_empty=True,<EOL>write_version=tf.train.SaverDef.V2,<EOL>pad_step_number=False,<EOL>save_relative_paths=True<EOL>)<EOL>

Creates the tf.train.Saver object and stores it in self.saver.


if self.execution_type == "<STR_LIT>":<EOL><INDENT>global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)<EOL>init_op = tf.variables_initializer(var_list=global_variables)<EOL>if self.summarizer_init_op is not None:<EOL><INDENT>init_op = tf.group(init_op, self.summarizer_init_op)<EOL><DEDENT>if self.graph_summary is None:<EOL><INDENT>ready_op = tf.report_uninitialized_variables(var_list=global_variables)<EOL>ready_for_local_init_op = None<EOL>local_init_op = None<EOL><DEDENT>else:<EOL><INDENT>ready_op = None<EOL>ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)<EOL>local_init_op = self.graph_summary<EOL><DEDENT><DEDENT>else:<EOL><INDENT>global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)<EOL>local_variables = self.get_variables(include_submodules=True, include_nontrainable=True)<EOL>init_op = tf.variables_initializer(var_list=global_variables)<EOL>if self.summarizer_init_op is not None:<EOL><INDENT>init_op = tf.group(init_op, self.summarizer_init_op)<EOL><DEDENT>ready_op = tf.report_uninitialized_variables(var_list=(global_variables + local_variables))<EOL>ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)<EOL>if self.graph_summary is None:<EOL><INDENT>local_init_op = tf.group(<EOL>tf.variables_initializer(var_list=local_variables),<EOL>*(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(<EOL>self.get_variables(include_submodules=True),<EOL>self.global_model.get_variables(include_submodules=True)<EOL>))<EOL>)<EOL><DEDENT>else:<EOL><INDENT>local_init_op = tf.group(<EOL>tf.variables_initializer(var_list=local_variables),<EOL>self.graph_summary,<EOL>*(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(<EOL>self.get_variables(include_submodules=True),<EOL>self.global_model.get_variables(include_submodules=True)<EOL>))<EOL>)<EOL><DEDENT><DEDENT>def init_fn(scaffold, session):<EOL><INDENT>if self.saver_spec is not None and self.saver_spec.get('<STR_LIT>', True):<EOL><INDENT>directory = self.saver_spec['<STR_LIT>']<EOL>file = self.saver_spec.get('<STR_LIT:file>')<EOL>if file is None:<EOL><INDENT>file = tf.train.latest_checkpoint(<EOL>checkpoint_dir=directory,<EOL>latest_filename=None  <EOL>)<EOL><DEDENT>elif not os.path.isfile(file):<EOL><INDENT>file = os.path.join(directory, file)<EOL><DEDENT>if file is not None:<EOL><INDENT>try:<EOL><INDENT>scaffold.saver.restore(sess=session, save_path=file)<EOL>session.run(fetches=self.list_buffer_index_reset_op)<EOL><DEDENT>except tf.errors.NotFoundError:<EOL><INDENT>raise TensorForceError("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.scaffold = tf.train.Scaffold(<EOL>init_op=init_op,<EOL>init_feed_dict=None,<EOL>init_fn=init_fn,<EOL>ready_op=ready_op,<EOL>ready_for_local_init_op=ready_for_local_init_op,<EOL>local_init_op=local_init_op,<EOL>summary_op=None,<EOL>saver=self.saver,<EOL>copy_from_scaffold=None<EOL>)<EOL>

Creates the tf.train.Scaffold object and assigns it to self.scaffold.
Other fields of the Scaffold are generated automatically.


hooks = list()<EOL>if self.saver_spec is not None and (self.execution_type == '<STR_LIT>' or self.distributed_spec['<STR_LIT>'] == <NUM_LIT:0>):<EOL><INDENT>self.saver_directory = self.saver_spec['<STR_LIT>']<EOL>hooks.append(tf.train.CheckpointSaverHook(<EOL>checkpoint_dir=self.saver_directory,<EOL>save_secs=self.saver_spec.get('<STR_LIT>', None if '<STR_LIT>' in self.saver_spec else <NUM_LIT>),<EOL>save_steps=self.saver_spec.get('<STR_LIT>'),  <EOL>saver=None,  <EOL>checkpoint_basename=self.saver_spec.get('<STR_LIT>', '<STR_LIT>'),<EOL>scaffold=self.scaffold,<EOL>listeners=None<EOL>))<EOL><DEDENT>else:<EOL><INDENT>self.saver_directory = None<EOL><DEDENT>return hooks<EOL>

Creates and returns a list of hooks to use in a session. Populates self.saver_directory.

Returns: List of hooks to use in a session.


def setup_session(self, server, hooks, graph_default_context):

if self.execution_type == "<STR_LIT>":<EOL><INDENT>session_creator = tf.train.ChiefSessionCreator(<EOL>scaffold=self.scaffold,<EOL>master=server.target,<EOL>config=self.session_config,<EOL>checkpoint_dir=None,<EOL>checkpoint_filename_with_path=None<EOL>)<EOL>self.monitored_session = tf.train.MonitoredSession(<EOL>session_creator=session_creator,<EOL>hooks=hooks,<EOL>stop_grace_period_secs=<NUM_LIT>  <EOL>)<EOL>if self.tf_session_dump_dir != "<STR_LIT>":<EOL><INDENT>self.monitored_session = DumpingDebugWrapperSession(self.monitored_session, self.tf_session_dump_dir)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.monitored_session = tf.train.SingularMonitoredSession(<EOL>hooks=hooks,<EOL>scaffold=self.scaffold,<EOL>master='<STR_LIT>',  <EOL>config=self.session_config,  <EOL>checkpoint_dir=None<EOL>)<EOL><DEDENT>if graph_default_context:<EOL><INDENT>graph_default_context.__exit__(None, None, None)<EOL><DEDENT>self.graph.finalize()<EOL>self.monitored_session.__enter__()<EOL>self.session = self.monitored_session._tf_sess()<EOL>

Creates and then enters the session for this model (finalizes the graph).

Args:
    server (tf.train.Server): The tf.train.Server object to connect to (None for single execution).
    hooks (list): A list of (saver, summary, etc..) hooks to be passed to the session.
    graph_default_context: The graph as_default() context that we are currently in.


if self.flush_summarizer is not None:<EOL><INDENT>self.monitored_session.run(fetches=self.flush_summarizer)<EOL><DEDENT>if self.saver_directory is not None:<EOL><INDENT>self.save(append_timestep=True)<EOL><DEDENT>self.monitored_session.__exit__(None, None, None)<EOL>

Saves the model (of saver dir is given) and closes the session.


<EOL>with tf.device(device_name_or_function=(self.global_model.device if self.global_model else self.device)):<EOL><INDENT>collection = self.graph.get_collection(name='<STR_LIT>')<EOL>if len(collection) == <NUM_LIT:0>:<EOL><INDENT>self.global_timestep = tf.get_variable(<EOL>name='<STR_LIT>',<EOL>shape=(),<EOL>dtype=tf.int64,<EOL>trainable=False,<EOL>initializer=tf.constant_initializer(value=<NUM_LIT:0>, dtype=tf.int64),<EOL>collections=['<STR_LIT>', tf.GraphKeys.GLOBAL_STEP]<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert len(collection) == <NUM_LIT:1><EOL>self.global_timestep = collection[<NUM_LIT:0>]<EOL><DEDENT>collection = self.graph.get_collection(name='<STR_LIT>')<EOL>if len(collection) == <NUM_LIT:0>:<EOL><INDENT>self.global_episode = tf.get_variable(<EOL>name='<STR_LIT>',<EOL>shape=(),<EOL>dtype=tf.int64,<EOL>trainable=False,<EOL>initializer=tf.constant_initializer(value=<NUM_LIT:0>, dtype=tf.int64),<EOL>collections=['<STR_LIT>']<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert len(collection) == <NUM_LIT:1><EOL>self.global_episode = collection[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>self.timestep = tf.get_variable(<EOL>name='<STR_LIT>',<EOL>shape=(),<EOL>dtype=tf.int64,<EOL>initializer=tf.constant_initializer(value=<NUM_LIT:0>, dtype=tf.int64),<EOL>trainable=False<EOL>)<EOL>self.episode = tf.get_variable(<EOL>name='<STR_LIT>',<EOL>shape=(),<EOL>dtype=tf.int64,<EOL>initializer=tf.constant_initializer(value=<NUM_LIT:0>, dtype=tf.int64),<EOL>trainable=False<EOL>)<EOL>self.episode_index_input = tf.placeholder(<EOL>name='<STR_LIT>',<EOL>shape=(),<EOL>dtype=tf.int32,<EOL>)<EOL>for name in sorted(self.states_spec):<EOL><INDENT>self.list_states_buffer[name] = tf.get_variable(<EOL>name=('<STR_LIT>'.format(name)),<EOL>shape=((self.num_parallel, self.batching_capacity,) + tuple(self.states_spec[name]['<STR_LIT>'])),<EOL>dtype=util.tf_dtype(self.states_spec[name]['<STR_LIT:type>']),<EOL>trainable=False<EOL>)<EOL><DEDENT>for name in sorted(self.internals_spec):<EOL><INDENT>self.list_internals_buffer[name] = tf.get_variable(<EOL>name=('<STR_LIT>'.format(name)),<EOL>shape=((self.num_parallel, self.batching_capacity,) + tuple(self.internals_spec[name]['<STR_LIT>'])),<EOL>dtype=util.tf_dtype(self.internals_spec[name]['<STR_LIT:type>']),<EOL>trainable=False<EOL>)<EOL><DEDENT>for name in sorted(self.actions_spec):<EOL><INDENT>self.list_actions_buffer[name]= tf.get_variable(<EOL>name=('<STR_LIT>'.format(name)),<EOL>shape=((self.num_parallel, self.batching_capacity,) + tuple(self.actions_spec[name]['<STR_LIT>'])),<EOL>dtype=util.tf_dtype(self.actions_spec[name]['<STR_LIT:type>']),<EOL>trainable=False<EOL>)<EOL><DEDENT>self.list_buffer_index = tf.get_variable(<EOL>name='<STR_LIT>',<EOL>shape=(self.num_parallel,),<EOL>dtype=util.tf_dtype('<STR_LIT:int>'),<EOL>trainable=False<EOL>)<EOL>

Creates tf Variables for the local state/internals/action-buffers and for the local and global counters
for timestep and episode.


def tf_preprocess(self, states, actions, reward):

<EOL>for name in sorted(self.states_preprocessing):<EOL><INDENT>states[name] = self.states_preprocessing[name].process(tensor=states[name])<EOL><DEDENT>if self.reward_preprocessing is not None:<EOL><INDENT>reward = self.reward_preprocessing.process(tensor=reward)<EOL><DEDENT>return states, actions, reward<EOL>

Applies preprocessing ops to the raw states/action/reward inputs.

Args:
    states (dict): Dict of raw state tensors.
    actions (dict): Dict or raw action tensors.
    reward: 1D (float) raw rewards tensor.

Returns: The preprocessed versions of the input tensors.


def tf_action_exploration(self, action, exploration, action_spec):

action_shape = tf.shape(input=action)<EOL>exploration_value = exploration.tf_explore(<EOL>episode=self.global_episode,<EOL>timestep=self.global_timestep,<EOL>shape=action_spec['<STR_LIT>']<EOL>)<EOL>exploration_value = tf.expand_dims(input=exploration_value, axis=<NUM_LIT:0>)<EOL>if action_spec['<STR_LIT:type>'] == '<STR_LIT:bool>':<EOL><INDENT>action = tf.where(<EOL>condition=(tf.random_uniform(shape=action_shape) < exploration_value),<EOL>x=(tf.random_uniform(shape=action_shape) < <NUM_LIT:0.5>),<EOL>y=action<EOL>)<EOL><DEDENT>elif action_spec['<STR_LIT:type>'] == '<STR_LIT:int>':<EOL><INDENT>action = tf.where(<EOL>condition=(tf.random_uniform(shape=action_shape) < exploration_value),<EOL>x=tf.random_uniform(shape=action_shape, maxval=action_spec['<STR_LIT>'], dtype=util.tf_dtype('<STR_LIT:int>')),<EOL>y=action<EOL>)<EOL><DEDENT>elif action_spec['<STR_LIT:type>'] == '<STR_LIT:float>':<EOL><INDENT>noise = tf.random_normal(shape=action_shape, dtype=util.tf_dtype('<STR_LIT:float>'))<EOL>action += noise * exploration_value<EOL>if '<STR_LIT>' in action_spec:<EOL><INDENT>action = tf.clip_by_value(<EOL>t=action,<EOL>clip_value_min=action_spec['<STR_LIT>'],<EOL>clip_value_max=action_spec['<STR_LIT>']<EOL>)<EOL><DEDENT><DEDENT>return action<EOL>

Applies optional exploration to the action (post-processor for action outputs).

Args:
     action (tf.Tensor): The original output action tensor (to be post-processed).
     exploration (Exploration): The Exploration object to use.
     action_spec (dict): Dict specifying the action space.
Returns:
    The post-processed action output tensor.


def tf_actions_and_internals(self, states, internals, deterministic):

raise NotImplementedError<EOL>

Creates and returns the TensorFlow operations for retrieving the actions and - if applicable -
the posterior internal state Tensors in reaction to the given input states (and prior internal states).

Args:
    states (dict): Dict of state tensors (each key represents one state space component).
    internals (dict): Dict of internal state tensors (each key represents one internal space component).
    deterministic: Boolean tensor indicating whether action should be chosen
        deterministically.

Returns:
    tuple:
        1) dict of output actions (with or without exploration applied (see `deterministic`))
        2) list of posterior internal state Tensors (empty for non-internal state models)


def tf_observe_timestep(self, states, internals, actions, terminal, reward):

raise NotImplementedError<EOL>

Creates the TensorFlow operations for processing a batch of observations coming in from our buffer (state,
action, internals) as well as from the agent's python-batch (terminal-signals and rewards from the env).

Args:
    states (dict): Dict of state tensors (each key represents one state space component).
    internals (dict): Dict of prior internal state tensors (each key represents one internal state component).
    actions (dict): Dict of action tensors (each key represents one action space component).
    terminal: 1D (bool) tensor of terminal signals.
    reward: 1D (float) tensor of rewards.

Returns:
    The observation operation depending on the model type.


def create_act_operations(self, states, internals, deterministic, independent, index):

<EOL>operations = list()<EOL>if self.variable_noise is not None and self.variable_noise > <NUM_LIT:0.0>:<EOL><INDENT>self.fn_actions_and_internals(<EOL>states=states,<EOL>internals=internals,<EOL>deterministic=deterministic<EOL>)<EOL>noise_deltas = list()<EOL>for variable in self.get_variables():<EOL><INDENT>noise_delta = tf.random_normal(shape=util.shape(variable), mean=<NUM_LIT:0.0>, stddev=self.variable_noise)<EOL>noise_deltas.append(noise_delta)<EOL>operations.append(variable.assign_add(delta=noise_delta))<EOL><DEDENT><DEDENT>with tf.control_dependencies(control_inputs=operations):<EOL><INDENT>self.actions_output, self.internals_output = self.fn_actions_and_internals(<EOL>states=states,<EOL>internals=internals,<EOL>deterministic=deterministic<EOL>)<EOL><DEDENT>with tf.control_dependencies(control_inputs=[self.actions_output[name] for name in sorted(self.actions_output)]):<EOL><INDENT>operations = list()<EOL>if self.variable_noise is not None and self.variable_noise > <NUM_LIT:0.0>:<EOL><INDENT>for variable, noise_delta in zip(self.get_variables(), noise_deltas):<EOL><INDENT>operations.append(variable.assign_sub(delta=noise_delta))<EOL><DEDENT><DEDENT><DEDENT>with tf.control_dependencies(control_inputs=operations):<EOL><INDENT>for name in sorted(self.actions_exploration):<EOL><INDENT>self.actions_output[name] = tf.cond(<EOL>pred=self.deterministic_input,<EOL>true_fn=(lambda: self.actions_output[name]),<EOL>false_fn=(lambda: self.fn_action_exploration(<EOL>action=self.actions_output[name],<EOL>exploration=self.actions_exploration[name],<EOL>action_spec=self.actions_spec[name]<EOL>))<EOL>)<EOL><DEDENT><DEDENT>def independent_act():<EOL><INDENT>"""<STR_LIT>"""<EOL>return self.global_timestep<EOL><DEDENT>def normal_act():<EOL><INDENT>"""<STR_LIT>"""<EOL>operations = list()<EOL>batch_size = tf.shape(input=states[next(iter(sorted(states)))])[<NUM_LIT:0>]<EOL>for name in sorted(states):<EOL><INDENT>operations.append(tf.assign(<EOL>ref=self.list_states_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],<EOL>value=states[name]<EOL>))<EOL><DEDENT>for name in sorted(internals):<EOL><INDENT>operations.append(tf.assign(<EOL>ref=self.list_internals_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],<EOL>value=internals[name]<EOL>))<EOL><DEDENT>for name in sorted(self.actions_output):<EOL><INDENT>operations.append(tf.assign(<EOL>ref=self.list_actions_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],<EOL>value=self.actions_output[name]<EOL>))<EOL><DEDENT>with tf.control_dependencies(control_inputs=operations):<EOL><INDENT>operations = list()<EOL>operations.append(tf.assign(<EOL>ref=self.list_buffer_index[index: index+<NUM_LIT:1>],<EOL>value=tf.add(self.list_buffer_index[index: index+<NUM_LIT:1>], tf.constant([<NUM_LIT:1>]))<EOL>))<EOL>operations.append(tf.assign_add(<EOL>ref=self.timestep,<EOL>value=tf.to_int64(x=batch_size)<EOL>))<EOL>operations.append(tf.assign_add(<EOL>ref=self.global_timestep,<EOL>value=tf.to_int64(x=batch_size)<EOL>))<EOL><DEDENT>with tf.control_dependencies(control_inputs=operations):<EOL><INDENT>return self.global_timestep + <NUM_LIT:0><EOL><DEDENT><DEDENT>self.timestep_output = tf.cond(<EOL>pred=independent,<EOL>true_fn=independent_act,<EOL>false_fn=normal_act<EOL>)<EOL>

Creates and stores tf operations that are fetched when calling act(): actions_output, internals_output and
timestep_output.

Args:
    states (dict): Dict of state tensors (each key represents one state space component).
    internals (dict): Dict of prior internal state tensors (each key represents one internal state component).
    deterministic: 0D (bool) tensor (whether to not use action exploration).
    independent (bool): 0D (bool) tensor (whether to store states/internals/action in local buffer).


def create_observe_operations(self, terminal, reward, index):

<EOL>num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('<STR_LIT:int>'))<EOL>increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))<EOL>increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))<EOL>with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):<EOL><INDENT>fn = (lambda x: tf.stop_gradient(input=x[:self.list_buffer_index[index]]))<EOL>states = util.map_tensors(fn=fn, tensors=self.list_states_buffer, index=index)<EOL>internals = util.map_tensors(fn=fn, tensors=self.list_internals_buffer, index=index)<EOL>actions = util.map_tensors(fn=fn, tensors=self.list_actions_buffer, index=index)<EOL>terminal = tf.stop_gradient(input=terminal)<EOL>reward = tf.stop_gradient(input=reward)<EOL>observation = self.fn_observe_timestep(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL><DEDENT>with tf.control_dependencies(control_inputs=(observation,)):<EOL><INDENT>reset_index = tf.assign(ref=self.list_buffer_index[index], value=<NUM_LIT:0>)<EOL><DEDENT>with tf.control_dependencies(control_inputs=(reset_index,)):<EOL><INDENT>self.episode_output = self.global_episode + <NUM_LIT:0><EOL><DEDENT>self.list_buffer_index_reset_op = tf.group(<EOL>*(tf.assign(ref=self.list_buffer_index[n], value=<NUM_LIT:0>) for n in range(self.num_parallel))<EOL>)<EOL>

Returns the tf op to fetch when an observation batch is passed in (e.g. an episode's rewards and
terminals). Uses the filled tf buffers for states, actions and internals to run
the tf_observe_timestep (model-dependent), resets buffer index and increases counters (episodes,
timesteps).

Args:
    terminal: The 1D tensor (bool) of terminal signals to process (more than one True within that list is ok).
    reward: The 1D tensor (float) of rewards to process.

Returns: Tf op to fetch when `observe()` is called.


def create_atomic_observe_operations(self, states, actions, internals, terminal, reward, index):

<EOL>num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('<STR_LIT:int>'))<EOL>increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))<EOL>increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))<EOL>with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):<EOL><INDENT>states = util.map_tensors(fn=tf.stop_gradient, tensors=states)<EOL>internals = util.map_tensors(fn=tf.stop_gradient, tensors=internals)<EOL>actions = util.map_tensors(fn=tf.stop_gradient, tensors=actions)<EOL>terminal = tf.stop_gradient(input=terminal)<EOL>reward = tf.stop_gradient(input=reward)<EOL>observation = self.fn_observe_timestep(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL><DEDENT>with tf.control_dependencies(control_inputs=(observation,)):<EOL><INDENT>self.unbuffered_episode_output = self.global_episode + <NUM_LIT:0><EOL><DEDENT>

Returns the tf op to fetch when unbuffered observations are passed in.

Args:
    states (any): One state (usually a value tuple) or dict of states if multiple states are expected.
    actions (any): One action (usually a value tuple) or dict of states if multiple actions are expected.
    internals (any): Internal list.
    terminal (bool): boolean indicating if the episode terminated after the observation.
    reward (float): scalar reward that resulted from executing the action.

Returns: Tf op to fetch when `observe()` is called.


def create_operations(self, states, internals, actions, terminal, reward, deterministic, independent, index):

Creates and stores tf operations for when `act()` and `observe()` are called.


def get_variables(self, include_submodules=False, include_nontrainable=False):

if include_nontrainable:<EOL><INDENT>model_variables = [self.all_variables[key] for key in sorted(self.all_variables)]<EOL>states_preprocessing_variables = [<EOL>variable for name in sorted(self.states_preprocessing)<EOL>for variable in self.states_preprocessing[name].get_variables()<EOL>]<EOL>model_variables += states_preprocessing_variables<EOL>actions_exploration_variables = [<EOL>variable for name in sorted(self.actions_exploration)<EOL>for variable in self.actions_exploration[name].get_variables()<EOL>]<EOL>model_variables += actions_exploration_variables<EOL>if self.reward_preprocessing is not None:<EOL><INDENT>reward_preprocessing_variables = self.reward_preprocessing.get_variables()<EOL>model_variables += reward_preprocessing_variables<EOL><DEDENT><DEDENT>else:<EOL><INDENT>model_variables = [self.variables[key] for key in sorted(self.variables)]<EOL><DEDENT>return model_variables<EOL>

Returns the TensorFlow variables used by the model.

Args:
    include_submodules: Includes variables of submodules (e.g. baseline, target network)
        if true.
    include_nontrainable: Includes non-trainable variables if true.

Returns:
    List of variables.


fetches = [self.global_episode, self.global_timestep]<EOL>for name in sorted(self.states_preprocessing):<EOL><INDENT>fetch = self.states_preprocessing[name].reset()<EOL>if fetch is not None:<EOL><INDENT>fetches.extend(fetch)<EOL><DEDENT><DEDENT>if self.flush_summarizer is not None:<EOL><INDENT>fetches.append(self.flush_summarizer)<EOL><DEDENT>fetch_list = self.monitored_session.run(fetches=fetches)<EOL>episode, timestep = fetch_list[:<NUM_LIT:2>]<EOL>return episode, timestep, self.internals_init<EOL>

Resets the model to its initial state on episode start. This should also reset all preprocessor(s).

Returns:
    tuple:
        Current episode, timestep counter and the shallow-copied list of internal state initialization Tensors.


feed_dict = dict()<EOL>batched = None<EOL>if states is not None:<EOL><INDENT>if batched is None:<EOL><INDENT>name = next(iter(states))<EOL>state = np.asarray(states[name])<EOL>batched = (state.ndim != len(self.states_spec[name]['<STR_LIT>']))<EOL><DEDENT>if batched:<EOL><INDENT>feed_dict.update({self.states_input[name]: states[name] for name in sorted(self.states_input)})<EOL><DEDENT>else:<EOL><INDENT>feed_dict.update({self.states_input[name]: (states[name],) for name in sorted(self.states_input)})<EOL><DEDENT><DEDENT>if internals is not None:<EOL><INDENT>if batched is None:<EOL><INDENT>name = next(iter(internals))<EOL>internal = np.asarray(internals[name])<EOL>batched = (internal.ndim != len(self.internals_spec[name]['<STR_LIT>']))<EOL><DEDENT>if batched:<EOL><INDENT>feed_dict.update({self.internals_input[name]: internals[name] for name in sorted(self.internals_input)})<EOL><DEDENT>else:<EOL><INDENT>feed_dict.update({self.internals_input[name]: (internals[name],) for name in sorted(self.internals_input)})<EOL><DEDENT><DEDENT>if actions is not None:<EOL><INDENT>if batched is None:<EOL><INDENT>name = next(iter(actions))<EOL>action = np.asarray(actions[name])<EOL>batched = (action.ndim != len(self.actions_spec[name]['<STR_LIT>']))<EOL><DEDENT>if batched:<EOL><INDENT>feed_dict.update({self.actions_input[name]: actions[name] for name in sorted(self.actions_input)})<EOL><DEDENT>else:<EOL><INDENT>feed_dict.update({self.actions_input[name]: (actions[name],) for name in sorted(self.actions_input)})<EOL><DEDENT><DEDENT>if terminal is not None:<EOL><INDENT>if batched is None:<EOL><INDENT>terminal = np.asarray(terminal)<EOL>batched = (terminal.ndim == <NUM_LIT:1>)<EOL><DEDENT>if batched:<EOL><INDENT>feed_dict[self.terminal_input] = terminal<EOL><DEDENT>else:<EOL><INDENT>feed_dict[self.terminal_input] = (terminal,)<EOL><DEDENT><DEDENT>if reward is not None:<EOL><INDENT>if batched is None:<EOL><INDENT>reward = np.asarray(reward)<EOL>batched = (reward.ndim == <NUM_LIT:1>)<EOL><DEDENT>if batched:<EOL><INDENT>feed_dict[self.reward_input] = reward<EOL><DEDENT>else:<EOL><INDENT>feed_dict[self.reward_input] = (reward,)<EOL><DEDENT><DEDENT>if deterministic is not None:<EOL><INDENT>feed_dict[self.deterministic_input] = deterministic<EOL><DEDENT>if independent is not None:<EOL><INDENT>feed_dict[self.independent_input] = independent<EOL><DEDENT>feed_dict[self.episode_index_input] = index<EOL>return feed_dict<EOL>

Returns the feed-dict for the model's acting and observing tf fetches.

Args:
    states (dict): Dict of state values (each key represents one state space component).
    internals (dict): Dict of internal state values (each key represents one internal state component).
    actions (dict): Dict of actions (each key represents one action space component).
    terminal (List[bool]): List of is-terminal signals.
    reward (List[float]): List of reward signals.
    deterministic (bool): Whether actions should be picked without exploration.
    independent (bool): Whether we are doing an independent act (not followed by call to observe;
        not to be stored in model's buffer).

Returns: The feed dict to use for the fetch.


def act(self, states, internals, deterministic=False, independent=False, fetch_tensors=None, index=<NUM_LIT:0>):

name = next(iter(states))<EOL>state = np.asarray(states[name])<EOL>batched = (state.ndim != len(self.states_spec[name]['<STR_LIT>']))<EOL>if batched:<EOL><INDENT>assert state.shape[<NUM_LIT:0>] <= self.batching_capacity<EOL><DEDENT>fetches = [self.actions_output, self.internals_output, self.timestep_output]<EOL>if self.network is not None and fetch_tensors is not None:<EOL><INDENT>for name in fetch_tensors:<EOL><INDENT>valid, tensor = self.network.get_named_tensor(name)<EOL>if valid:<EOL><INDENT>fetches.append(tensor)<EOL><DEDENT>else:<EOL><INDENT>keys = self.network.get_list_of_named_tensor()<EOL>raise TensorForceError('<STR_LIT>'.format(name, keys))<EOL><DEDENT><DEDENT><DEDENT>feed_dict = self.get_feed_dict(<EOL>states=states,<EOL>internals=internals,<EOL>deterministic=deterministic,<EOL>independent=independent,<EOL>index=index<EOL>)<EOL>fetch_list = self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)<EOL>actions, internals, timestep = fetch_list[<NUM_LIT:0>:<NUM_LIT:3>]<EOL>if not batched:<EOL><INDENT>actions = {name: actions[name][<NUM_LIT:0>] for name in sorted(actions)}<EOL>internals = {name: internals[name][<NUM_LIT:0>] for name in sorted(internals)}<EOL><DEDENT>if self.network is not None and fetch_tensors is not None:<EOL><INDENT>fetch_dict = dict()<EOL>for index_, tensor in enumerate(fetch_list[<NUM_LIT:3>:]):<EOL><INDENT>name = fetch_tensors[index_]<EOL>fetch_dict[name] = tensor<EOL><DEDENT>return actions, internals, timestep, fetch_dict<EOL><DEDENT>else:<EOL><INDENT>return actions, internals, timestep<EOL><DEDENT>

Does a forward pass through the model to retrieve action (outputs) given inputs for state (and internal
state, if applicable (e.g. RNNs))

Args:
    states (dict): Dict of state values (each key represents one state space component).
    internals (dict): Dict of internal state values (each key represents one internal state component).
    deterministic (bool): If True, will not apply exploration after actions are calculated.
    independent (bool): If true, action is not followed by observe (and hence not included
        in updates).
    fetch_tensors (list): List of names of additional tensors (from the model's network) to fetch (and return).
    index: (int) index of the episode we want to produce the next action

Returns:
    tuple:
        - Actual action-outputs (batched if state input is a batch).
        - Actual values of internal states (if applicable) (batched if state input is a batch).
        - The timestep (int) after calculating the (batch of) action(s).


def observe(self, terminal, reward, index=<NUM_LIT:0>):

fetches = self.episode_output<EOL>feed_dict = self.get_feed_dict(terminal=terminal, reward=reward, index=index)<EOL>episode = self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)<EOL>return episode<EOL>

Adds an observation (reward and is-terminal) to the model without updating its trainable variables.

Args:
    terminal (List[bool]): List of is-terminal signals.
    reward (List[float]): List of reward signals.
    index: (int) parallel episode you want to observe

Returns:
    The value of the model-internal episode counter.


def save(self, directory=None, append_timestep=True):

if self.flush_summarizer is not None:<EOL><INDENT>self.monitored_session.run(fetches=self.flush_summarizer)<EOL><DEDENT>return self.saver.save(<EOL>sess=self.session,<EOL>save_path=(self.saver_directory if directory is None else directory),<EOL>global_step=(self.global_timestep if append_timestep else None),<EOL>meta_graph_suffix='<STR_LIT>',<EOL>write_meta_graph=True,<EOL>write_state=True<EOL>)<EOL>

Save TensorFlow model. If no checkpoint directory is given, the model's default saver
directory is used. Optionally appends current timestep to prevent overwriting previous
checkpoint files. Turn off to be able to load model from the same given path argument as
given here.

Args:
    directory: Optional checkpoint directory.
    append_timestep: Appends the current timestep to the checkpoint file if true.

Returns:
    Checkpoint path where the model was saved.


def restore(self, directory=None, file=None):

if file is None:<EOL><INDENT>file = tf.train.latest_checkpoint(<EOL>checkpoint_dir=(self.saver_directory if directory is None else directory),<EOL>)<EOL><DEDENT>elif directory is None:<EOL><INDENT>file = os.path.join(self.saver_directory, file)<EOL><DEDENT>elif not os.path.isfile(file):<EOL><INDENT>file = os.path.join(directory, file)<EOL><DEDENT>self.saver.restore(sess=self.session, save_path=file)<EOL>self.session.run(fetches=self.list_buffer_index_reset_op)<EOL>

Restore TensorFlow model. If no checkpoint file is given, the latest checkpoint is
restored. If no checkpoint directory is given, the model's default saver directory is
used (unless file specifies the entire path).

Args:
    directory: Optional checkpoint directory.
    file: Optional checkpoint file, or path if directory not given.


return dict()<EOL>

Returns a dictionary of component name to component of all the components within this model.

Returns:
    (dict) The mapping of name to component.


components = self.get_components()<EOL>components = [components[name] for name in sorted(components)]<EOL>return set(filter(lambda x: isinstance(x, util.SavableComponent), components))<EOL>

Returns the list of all of the components this model consists of that can be individually saved and restored.
For instance the network or distribution.

Returns:
    List of util.SavableComponent


def save_component(self, component_name, save_path):

component = self.get_component(component_name=component_name)<EOL>self._validate_savable(component=component, component_name=component_name)<EOL>return component.save(sess=self.session, save_path=save_path)<EOL>

Saves a component of this model to the designated location.

Args:
    component_name: The component to save.
    save_path: The location to save to.
Returns:
    Checkpoint path where the component was saved.


def restore_component(self, component_name, save_path):

component = self.get_component(component_name=component_name)<EOL>self._validate_savable(component=component, component_name=component_name)<EOL>component.restore(sess=self.session, save_path=save_path)<EOL>

Restores a component's parameters from a save location.

Args:
    component_name: The component to restore.
    save_path: The save location.


mapping = self.get_components()<EOL>return mapping[component_name] if component_name in mapping else None<EOL>

Looks up a component by its name.

Args:
    component_name: The name of the component to look up.
Returns:
    The component for the provided name or None if there is no such component.


self.update_mode = update_mode<EOL>self.memory_spec = memory<EOL>self.optimizer_spec = optimizer<EOL>assert discount is None or discount >= <NUM_LIT:0.0><EOL>self.discount = discount<EOL>self.memory = None<EOL>self.optimizer = None<EOL>self.fn_discounted_cumulative_reward = None<EOL>self.fn_reference = None<EOL>self.fn_loss_per_instance = None<EOL>self.fn_regularization_losses = None<EOL>self.fn_loss = None<EOL>self.fn_optimization = None<EOL>self.fn_import_experience = None<EOL>super(MemoryModel, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>batching_capacity=batching_capacity,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing<EOL>)<EOL>

Memory model.

Args:
    states (spec): The state-space description dictionary.
    actions (spec): The action-space description dictionary.
    scope (str): The root scope str to use for tf variable scoping.
    device (str): The name of the device to run the graph of this model on.
    saver (spec): Dict specifying whether and how to save the model's parameters.
    summarizer (spec): Dict specifying which tensorboard summaries should be created and added to the graph.
    execution (spec): Dict specifying whether and how to do distributed training on the model's graph.
    batching_capacity (int): Batching capacity.
    variable_noise (float): The stddev value of a Normal distribution used for adding random
        noise to the model's output (for each batch, noise can be toggled and - if active - will be resampled).
        Use None for not adding any noise.
    states_preprocessing (spec / dict of specs): Dict specifying whether and how to preprocess state signals
        (e.g. normalization, greyscale, etc..).
    actions_exploration (spec / dict of specs): Dict specifying whether and how to add exploration to the model's
        "action outputs" (e.g. epsilon-greedy).
    reward_preprocessing (spec): Dict specifying whether and how to preprocess rewards coming
        from the Environment (e.g. reward normalization).
    update_mode (spec): Update mode.
    memory (spec): Memory.
    optimizer (spec): Dict specifying the tf optimizer to use for tuning the model's trainable parameters.
    discount (float): The RL reward discount factor (gamma).


super(MemoryModel, self).as_local_model()<EOL>self.optimizer_spec = dict(<EOL>type='<STR_LIT>',<EOL>optimizer=self.optimizer_spec<EOL>)<EOL>

Makes sure our optimizer is wrapped into the global_optimizer meta. This is only relevant for distributed RL.


Constructs the memory and the optimizer objects.
Generates and stores all template functions.


super(MemoryModel, self).tf_initialize()<EOL>self.memory.initialize()<EOL>

Also initializes our Memory object (self.memory).


def tf_discounted_cumulative_reward(self, terminal, reward, discount=None, final_reward=<NUM_LIT:0.0>, horizon=<NUM_LIT:0>):

<EOL>if discount is None:<EOL><INDENT>discount = self.discount<EOL><DEDENT>def cumulate(cumulative, reward_terminal_horizon_subtract):<EOL><INDENT>rew, is_terminal, is_over_horizon, sub = reward_terminal_horizon_subtract<EOL>return tf.where(<EOL>condition=is_terminal,<EOL>x=rew,<EOL>y=tf.where(<EOL>condition=is_over_horizon,<EOL>x=(rew + cumulative * discount - sub),<EOL>y=(rew + cumulative * discount)<EOL>)<EOL>)<EOL><DEDENT>def len_(cumulative, term):<EOL><INDENT>return tf.where(<EOL>condition=term,<EOL>x=tf.ones(shape=(), dtype=tf.int32),<EOL>y=cumulative + <NUM_LIT:1><EOL>)<EOL><DEDENT>reward = tf.reverse(tensor=reward, axis=(<NUM_LIT:0>,))<EOL>terminal = tf.reverse(tensor=terminal, axis=(<NUM_LIT:0>,))<EOL>lengths = tf.scan(fn=len_, elems=terminal, initializer=<NUM_LIT:0>)<EOL>off_horizon = tf.greater(lengths, tf.fill(dims=tf.shape(lengths), value=horizon))<EOL>if horizon > <NUM_LIT:0>:<EOL><INDENT>horizon_subtractions = tf.map_fn(lambda x: (discount ** horizon) * x, reward, dtype=tf.float32)<EOL>horizon_subtractions = tf.concat([np.zeros(shape=(horizon,)), horizon_subtractions], axis=<NUM_LIT:0>)<EOL>horizon_subtractions = tf.slice(horizon_subtractions, begin=(<NUM_LIT:0>,), size=tf.shape(reward))<EOL><DEDENT>else:<EOL><INDENT>horizon_subtractions = tf.zeros(shape=tf.shape(reward))<EOL><DEDENT>reward = tf.scan(<EOL>fn=cumulate,<EOL>elems=(reward, terminal, off_horizon, horizon_subtractions),<EOL>initializer=final_reward if horizon != <NUM_LIT:1> else <NUM_LIT:0.0><EOL>)<EOL>return tf.reverse(tensor=reward, axis=(<NUM_LIT:0>,))<EOL>

Creates and returns the TensorFlow operations for calculating the sequence of discounted cumulative rewards
for a given sequence of single rewards.

Example:
single rewards = 2.0 1.0 0.0 0.5 1.0 -1.0
terminal = False, False, False, False True False
gamma = 0.95
final_reward = 100.0 (only matters for last episode (r=-1.0) as this episode has no terminal signal)
horizon=3
output = 2.95 1.45 1.38 1.45 1.0 94.0

Args:
    terminal: Tensor (bool) holding the is-terminal sequence. This sequence may contain more than one
        True value. If its very last element is False (not terminating), the given `final_reward` value
        is assumed to follow the last value in the single rewards sequence (see below).
    reward: Tensor (float) holding the sequence of single rewards. If the last element of `terminal` is False,
        an assumed last reward of the value of `final_reward` will be used.
    discount (float): The discount factor (gamma). By default, take the Model's discount factor.
    final_reward (float): Reward value to use if last episode in sequence does not terminate (terminal sequence
        ends with False). This value will be ignored if horizon == 1 or discount == 0.0.
    horizon (int): The length of the horizon (e.g. for n-step cumulative rewards in continuous tasks
        without terminal signals). Use 0 (default) for an infinite horizon. Note that horizon=1 leads to the
        exact same results as a discount factor of 0.0.

Returns:
    Discounted cumulative reward tensor with the same shape as `reward`.


def tf_reference(self, states, internals, actions, terminal, reward, next_states, next_internals, update):

return None<EOL>

Creates the TensorFlow operations for obtaining the reference tensor(s), in case of a
comparative loss.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    actions: Dict of action tensors.
    terminal: Terminal boolean tensor.
    reward: Reward tensor.
    next_states: Dict of successor state tensors.
    next_internals: List of posterior internal state tensors.
    update: Boolean tensor indicating whether this call happens during an update.

Returns:
    Reference tensor(s).


def tf_loss_per_instance(self, states, internals, actions, terminal, reward,<EOL>next_states, next_internals, update, reference=None):

raise NotImplementedError<EOL>

Creates the TensorFlow operations for calculating the loss per batch instance.

Args:
    states: Dict of state tensors.
    internals: Dict of prior internal state tensors.
    actions: Dict of action tensors.
    terminal: Terminal boolean tensor.
    reward: Reward tensor.
    next_states: Dict of successor state tensors.
    next_internals: List of posterior internal state tensors.
    update: Boolean tensor indicating whether this call happens during an update.
    reference: Optional reference tensor(s), in case of a comparative loss.

Returns:
    Loss per instance tensor.


def tf_regularization_losses(self, states, internals, update):

return dict()<EOL>

Creates the TensorFlow operations for calculating the regularization losses for the given input states.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    update: Boolean tensor indicating whether this call happens during an update.

Returns:
    Dict of regularization loss tensors.


def tf_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):

<EOL>loss_per_instance = self.fn_loss_per_instance(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward,<EOL>next_states=next_states,<EOL>next_internals=next_internals,<EOL>update=update,<EOL>reference=reference<EOL>)<EOL>updated = self.memory.update_batch(loss_per_instance=loss_per_instance)<EOL>with tf.control_dependencies(control_inputs=(updated,)):<EOL><INDENT>loss = tf.reduce_mean(input_tensor=loss_per_instance, axis=<NUM_LIT:0>)<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.scalar(name='<STR_LIT>', tensor=loss)<EOL><DEDENT>losses = self.fn_regularization_losses(states=states, internals=internals, update=update)<EOL>if len(losses) > <NUM_LIT:0>:<EOL><INDENT>loss += tf.add_n(inputs=[losses[name] for name in sorted(losses)])<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>for name in sorted(losses):<EOL><INDENT>tf.contrib.summary.scalar(name=('<STR_LIT>' + name), tensor=losses[name])<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in self.summary_labels or '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.scalar(name='<STR_LIT>', tensor=loss)<EOL><DEDENT>return loss<EOL><DEDENT>

Creates the TensorFlow operations for calculating the full loss of a batch.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    actions: Dict of action tensors.
    terminal: Terminal boolean tensor.
    reward: Reward tensor.
    next_states: Dict of successor state tensors.
    next_internals: List of posterior internal state tensors.
    update: Boolean tensor indicating whether this call happens during an update.
    reference: Optional reference tensor(s), in case of a comparative loss.

Returns:
    Loss tensor.


def optimizer_arguments(self, states, internals, actions, terminal, reward, next_states, next_internals):

arguments = dict(<EOL>time=self.global_timestep,<EOL>variables=self.get_variables(),<EOL>arguments=dict(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward,<EOL>next_states=next_states,<EOL>next_internals=next_internals,<EOL>update=tf.constant(value=True)<EOL>),<EOL>fn_reference=self.fn_reference,<EOL>fn_loss=self.fn_loss<EOL>)<EOL>if self.global_model is not None:<EOL><INDENT>arguments['<STR_LIT>'] = self.global_model.get_variables()<EOL><DEDENT>return arguments<EOL>

Returns the optimizer arguments including the time, the list of variables to optimize,
and various functions which the optimizer might require to perform an update step.

Args:
    states (dict): Dict of state tensors.
    internals (dict): Dict of prior internal state tensors.
    actions (dict): Dict of action tensors.
    terminal: 1D boolean is-terminal tensor.
    reward: 1D (float) rewards tensor.
    next_states (dict): Dict of successor state tensors.
    next_internals (dict): Dict of posterior internal state tensors.

Returns:
    Optimizer arguments as dict to be used as **kwargs to the optimizer.


def tf_optimization(self, states, internals, actions, terminal, reward, next_states=None, next_internals=None):

Creates the TensorFlow operations for performing an optimization update step based
on the given input states and actions batch.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    actions: Dict of action tensors.
    terminal: Terminal boolean tensor.
    reward: Reward tensor.
    next_states: Dict of successor state tensors.
    next_internals: List of posterior internal state tensors.

Returns:
    The optimization operation.


def tf_observe_timestep(self, states, internals, actions, terminal, reward):

<EOL>stored = self.memory.store(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL>with tf.control_dependencies(control_inputs=(stored,)):<EOL><INDENT>unit = self.update_mode['<STR_LIT>']<EOL>batch_size = self.update_mode['<STR_LIT>']<EOL>frequency = self.update_mode.get('<STR_LIT>', batch_size)<EOL>first_update = self.update_mode.get('<STR_LIT>', <NUM_LIT:0>)<EOL>if unit == '<STR_LIT>':<EOL><INDENT>optimize = tf.logical_and(<EOL>x=tf.equal(x=(self.timestep % frequency), y=<NUM_LIT:0>),<EOL>y=tf.logical_and(<EOL>x=tf.greater_equal(x=self.timestep, y=batch_size),<EOL>y=tf.greater_equal(x=self.timestep, y=first_update)<EOL>)<EOL>)<EOL><DEDENT>elif unit == '<STR_LIT>':<EOL><INDENT>optimize = tf.logical_and(<EOL>x=tf.equal(x=(self.episode % frequency), y=<NUM_LIT:0>),<EOL>y=tf.logical_and(<EOL>x=tf.greater(x=tf.count_nonzero(input_tensor=terminal), y=<NUM_LIT:0>),<EOL>y=tf.logical_and(<EOL>x=tf.greater_equal(x=self.episode, y=batch_size),<EOL>y=tf.greater_equal(x=self.episode, y=first_update)<EOL>)<EOL>)<EOL>)<EOL><DEDENT>elif unit == '<STR_LIT>':<EOL><INDENT>sequence_length = self.update_mode.get('<STR_LIT>', <NUM_LIT:8>)<EOL>optimize = tf.logical_and(<EOL>x=tf.equal(x=(self.timestep % frequency), y=<NUM_LIT:0>),<EOL>y=tf.logical_and(<EOL>x=tf.greater_equal(x=self.timestep, y=(batch_size + sequence_length - <NUM_LIT:1>)),<EOL>y=tf.greater_equal(x=self.timestep, y=first_update)<EOL>)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>raise TensorForceError("<STR_LIT>".format(unit))<EOL><DEDENT>def true_fn():<EOL><INDENT>if unit == '<STR_LIT>':<EOL><INDENT>batch = self.memory.retrieve_timesteps(n=batch_size)<EOL><DEDENT>elif unit == '<STR_LIT>':<EOL><INDENT>batch = self.memory.retrieve_episodes(n=batch_size)<EOL><DEDENT>elif unit == '<STR_LIT>':<EOL><INDENT>batch = self.memory.retrieve_sequences(n=batch_size, sequence_length=sequence_length)<EOL><DEDENT>batch = util.map_tensors(<EOL>fn=(lambda tensor: tf.stop_gradient(input=tensor)),<EOL>tensors=batch<EOL>)<EOL>optimize = self.fn_optimization(**batch)<EOL>with tf.control_dependencies(control_inputs=(optimize,)):<EOL><INDENT>return tf.logical_and(x=True, y=True)<EOL><DEDENT><DEDENT>return tf.cond(pred=optimize, true_fn=true_fn, false_fn=tf.no_op)<EOL><DEDENT>

Creates and returns the op that - if frequency condition is hit - pulls a batch from the memory
and does one optimization step.


def tf_import_experience(self, states, internals, actions, terminal, reward):

return self.memory.store(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL>

Imports experiences into the TensorFlow memory structure. Can be used to import
off-policy data.

:param states: Dict of state values to import with keys as state names and values as values to set.
:param internals: Internal values to set, can be fetched from agent via agent.current_internals
    if no values available.
:param actions: Dict of action values to import with keys as action names and values as values to set.
:param terminal: Terminal value(s)
:param reward: Reward value(s)


def import_experience(self, states, internals, actions, terminal, reward):

fetches = self.import_experience_output<EOL>feed_dict = self.get_feed_dict(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL>self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)<EOL>


def tf_baseline_loss(self, states, internals, reward, update, reference=None):

if self.baseline_mode == '<STR_LIT>':<EOL><INDENT>loss = self.baseline.loss(<EOL>states=states,<EOL>internals=internals,<EOL>reward=reward,<EOL>update=update,<EOL>reference=reference<EOL>)<EOL><DEDENT>elif self.baseline_mode == '<STR_LIT>':<EOL><INDENT>loss = self.baseline.loss(<EOL>states=self.network.apply(x=states, internals=internals, update=update),<EOL>internals=internals,<EOL>reward=reward,<EOL>update=update,<EOL>reference=reference<EOL>)<EOL><DEDENT>regularization_loss = self.baseline.regularization_loss()<EOL>if regularization_loss is not None:<EOL><INDENT>loss += regularization_loss<EOL><DEDENT>return loss<EOL>

Creates the TensorFlow operations for calculating the baseline loss of a batch.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    reward: Reward tensor.
    update: Boolean tensor indicating whether this call happens during an update.
    reference: Optional reference tensor(s), in case of a comparative loss.

Returns:
    Loss tensor.


def baseline_optimizer_arguments(self, states, internals, reward):

arguments = dict(<EOL>time=self.global_timestep,<EOL>variables=self.baseline.get_variables(),<EOL>arguments=dict(<EOL>states=states,<EOL>internals=internals,<EOL>reward=reward,<EOL>update=tf.constant(value=True),<EOL>),<EOL>fn_reference=self.baseline.reference,<EOL>fn_loss=self.fn_baseline_loss,<EOL>)<EOL>if self.global_model is not None:<EOL><INDENT>arguments['<STR_LIT>'] = self.global_model.baseline.get_variables()<EOL><DEDENT>return arguments<EOL>

Returns the baseline optimizer arguments including the time, the list of variables to
optimize, and various functions which the optimizer might require to perform an update
step.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    reward: Reward tensor.

Returns:
    Baseline optimizer arguments as dict.


def tf_import_demo_experience(self, states, internals, actions, terminal, reward):

return self.demo_memory.store(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL>


def tf_demo_loss(self, states, actions, terminal, reward, internals, update, reference=None):

embedding = self.network.apply(x=states, internals=internals, update=update)<EOL>deltas = list()<EOL>for name in sorted(actions):<EOL><INDENT>action = actions[name]<EOL>distr_params = self.distributions[name].parameterize(x=embedding)<EOL>state_action_value = self.distributions[name].state_action_value(distr_params=distr_params, action=action)<EOL>if self.actions_spec[name]['<STR_LIT:type>'] == '<STR_LIT:bool>':<EOL><INDENT>num_actions = <NUM_LIT:2><EOL>action = tf.cast(x=action, dtype=util.tf_dtype('<STR_LIT:int>'))<EOL><DEDENT>else:<EOL><INDENT>num_actions = self.actions_spec[name]['<STR_LIT>']<EOL><DEDENT>one_hot = tf.one_hot(indices=action, depth=num_actions)<EOL>ones = tf.ones_like(tensor=one_hot, dtype=tf.float32)<EOL>inverted_one_hot = ones - one_hot<EOL>state_action_values = self.distributions[name].state_action_value(distr_params=distr_params)<EOL>state_action_values = state_action_values + inverted_one_hot * self.expert_margin<EOL>supervised_selector = tf.reduce_max(input_tensor=state_action_values, axis=-<NUM_LIT:1>)<EOL>delta = supervised_selector - state_action_value<EOL>action_size = util.prod(self.actions_spec[name]['<STR_LIT>'])<EOL>delta = tf.reshape(tensor=delta, shape=(-<NUM_LIT:1>, action_size))<EOL>deltas.append(delta)<EOL><DEDENT>loss_per_instance = tf.reduce_mean(input_tensor=tf.concat(values=deltas, axis=<NUM_LIT:1>), axis=<NUM_LIT:1>)<EOL>loss_per_instance = tf.square(x=loss_per_instance)<EOL>return tf.reduce_mean(input_tensor=loss_per_instance, axis=<NUM_LIT:0>)<EOL>

Extends the q-model loss via the dqfd large-margin loss.


def tf_combined_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):

q_model_loss = self.fn_loss(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward,<EOL>next_states=next_states,<EOL>next_internals=next_internals,<EOL>update=update,<EOL>reference=reference<EOL>)<EOL>demo_loss = self.fn_demo_loss(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward,<EOL>update=update,<EOL>reference=reference<EOL>)<EOL>return q_model_loss + self.supervised_weight * demo_loss<EOL>


def get_variables(self, include_submodules=False, include_nontrainable=False):

model_variables = super(QDemoModel, self).get_variables(<EOL>include_submodules=include_submodules,<EOL>include_nontrainable=include_nontrainable<EOL>)<EOL>if include_nontrainable:<EOL><INDENT>demo_memory_variables = self.demo_memory.get_variables()<EOL>model_variables += demo_memory_variables<EOL><DEDENT>return model_variables<EOL>

Returns the TensorFlow variables used by the model.

Returns:
    List of variables.


def import_demo_experience(self, states, internals, actions, terminal, reward):

fetches = self.import_demo_experience_output<EOL>feed_dict = self.get_feed_dict(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL>self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)<EOL>

Stores demonstrations in the demo memory.


fetches = self.demo_optimization_output<EOL>self.monitored_session.run(fetches=fetches)<EOL>

Performs a demonstration update by calling the demo optimization operation.
Note that the batch data does not have to be fetched from the demo memory as this is now part of
the TensorFlow operation of the demo update.


def tf_q_delta(self, q_value, next_q_value, terminal, reward):

for _ in range(util.rank(q_value) - <NUM_LIT:1>):<EOL><INDENT>terminal = tf.expand_dims(input=terminal, axis=<NUM_LIT:1>)<EOL>reward = tf.expand_dims(input=reward, axis=<NUM_LIT:1>)<EOL><DEDENT>multiples = (<NUM_LIT:1>,) + util.shape(q_value)[<NUM_LIT:1>:]<EOL>terminal = tf.tile(input=terminal, multiples=multiples)<EOL>reward = tf.tile(input=reward, multiples=multiples)<EOL>zeros = tf.zeros_like(tensor=next_q_value)<EOL>next_q_value = tf.where(condition=terminal, x=zeros, y=(self.discount * next_q_value))<EOL>return reward + next_q_value - q_value<EOL>

Creates the deltas (or advantage) of the Q values.

:return: A list of deltas per action


variables = self.target_network.get_variables() + [<EOL>variable for name in sorted(self.target_distributions)<EOL>for variable in self.target_distributions[name].get_variables()<EOL>]<EOL>source_variables = self.network.get_variables() + [<EOL>variable for name in sorted(self.distributions)<EOL>for variable in self.distributions[name].get_variables()<EOL>]<EOL>arguments = dict(<EOL>time=self.global_timestep,<EOL>variables=variables,<EOL>source_variables=source_variables<EOL>)<EOL>if self.global_model is not None:<EOL><INDENT>arguments['<STR_LIT>'] = self.global_model.target_network.get_variables() + [<EOL>variable for name in sorted(self.global_model.target_distributions)<EOL>for variable in self.global_model.target_distributions[name].get_variables()<EOL>]<EOL><DEDENT>return arguments<EOL>

Returns the target optimizer arguments including the time, the list of variables to
optimize, and various functions which the optimizer might require to perform an update
step.

Returns:
    Target optimizer arguments as dict.


self.scope = scope<EOL>self.device = device<EOL>self.saver = saver<EOL>self.summarizer = summarizer<EOL>self.execution = sanity_check_execution_spec(execution)<EOL>super(RandomAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity<EOL>)<EOL>

Initializes the random agent.

Args:
    scope (str): TensorFlow scope (default: name of agent).
    device: TensorFlow device (default: none)
    saver (spec): Saver specification, with the following attributes (default: none):
        - directory: model directory.
        - file: model filename (optional).
        - seconds or steps: save frequency (default: 600 seconds).
        - load: specifies whether model is loaded, if existent (default: true).
        - basename: optional file basename (default: 'model.ckpt').
    summarizer (spec): Summarizer specification, with the following attributes (default:
        none):
        - directory: summaries directory.
        - seconds or steps: summarize frequency (default: 120 seconds).
        - labels: list of summary labels to record (default: []).
        - meta_param_recorder_class: ???.
    execution (spec): Execution specification (see sanity_check_execution_spec for details).


<EOL>if update_mode is None:<EOL><INDENT>update_mode = dict(<EOL>unit='<STR_LIT>',<EOL>batch_size=<NUM_LIT:10><EOL>)<EOL><DEDENT>elif '<STR_LIT>' in update_mode:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>update_mode['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if memory is None:<EOL><INDENT>memory = dict(<EOL>type='<STR_LIT>',<EOL>include_next_states=False,<EOL>capacity=(<NUM_LIT:1000> * update_mode['<STR_LIT>'])<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert not memory['<STR_LIT>']<EOL><DEDENT>assert (update_mode['<STR_LIT>'] != '<STR_LIT>' or memory['<STR_LIT:type>'] == '<STR_LIT>')<EOL>optimizer = dict(<EOL>type='<STR_LIT>',<EOL>optimizer=dict(<EOL>type='<STR_LIT>',<EOL>learning_rate=learning_rate<EOL>),<EOL>ls_max_iterations=ls_max_iterations,<EOL>ls_accept_ratio=ls_accept_ratio,<EOL>ls_mode='<STR_LIT>',  <EOL>ls_parameter=<NUM_LIT:0.5>,  <EOL>ls_unroll_loop=ls_unroll_loop<EOL>)<EOL>self.baseline_mode = baseline_mode<EOL>self.baseline = baseline<EOL>self.baseline_optimizer = baseline_optimizer<EOL>self.gae_lambda = gae_lambda<EOL>self.likelihood_ratio_clipping = likelihood_ratio_clipping<EOL>super(ACKTRAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing,<EOL>update_mode=update_mode,<EOL>memory=memory,<EOL>optimizer=optimizer,<EOL>discount=discount,<EOL>network=network,<EOL>distributions=distributions,<EOL>entropy_regularization=entropy_regularization<EOL>)<EOL>

Initializes the ACKTR agent.

Args:
    update_mode (spec): Update mode specification, with the following attributes:
        - unit: 'episodes' if given (default: 'episodes').
        - batch_size: integer (default: 10).
        - frequency: integer (default: batch_size).
    memory (spec): Memory specification, see core.memories module for more information
        (default: {type='latest', include_next_states=false, capacity=1000*batch_size}).
    optimizer (spec): ACKTR agent implicitly defines an optimized-step KFAC optimizer.
    baseline_mode (str): One of 'states', 'network' (default: none).
    baseline (spec): Baseline specification, see core.baselines module for more information
        (default: none).
    baseline_optimizer (spec): Baseline optimizer specification, see core.optimizers module
        for more information (default: none).
    gae_lambda (float): Lambda factor for generalized advantage estimation (default: none).
    likelihood_ratio_clipping (float): Likelihood ratio clipping for policy gradient
        (default: none).
    learning_rate (float): Learning rate of natural-gradient optimizer (default: 1e-3).
    cg_max_iterations (int): Conjugate-gradient max iterations (default: 20).
    cg_damping (float): Conjugate-gradient damping (default: 1e-3).
    cg_unroll_loop (bool): Conjugate-gradient unroll loop (default: false).
    ls_max_iterations (int): Line-search max iterations (default: 10).
    ls_accept_ratio (float): Line-search accept ratio (default: 0.9).
    ls_unroll_loop (bool): Line-search unroll loop (default: false).


<EOL>if update_mode is None:<EOL><INDENT>update_mode = dict(<EOL>unit='<STR_LIT>',<EOL>batch_size=<NUM_LIT:32>,<EOL>frequency=<NUM_LIT:4><EOL>)<EOL><DEDENT>elif '<STR_LIT>' in update_mode:<EOL><INDENT>assert update_mode['<STR_LIT>'] == '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>update_mode['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if memory is None:<EOL><INDENT>memory = dict(<EOL>type='<STR_LIT>',<EOL>include_next_states=True,<EOL>capacity=(<NUM_LIT:1000> * update_mode['<STR_LIT>'])<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert memory['<STR_LIT>']<EOL><DEDENT>if optimizer is None:<EOL><INDENT>optimizer = dict(<EOL>type='<STR_LIT>',<EOL>learning_rate=<NUM_LIT><EOL>)<EOL><DEDENT>self.target_sync_frequency = target_sync_frequency<EOL>self.target_update_weight = target_update_weight<EOL>self.double_q_model = double_q_model<EOL>self.huber_loss = huber_loss<EOL>super(DQNAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing,<EOL>update_mode=update_mode,<EOL>memory=memory,<EOL>optimizer=optimizer,<EOL>discount=discount,<EOL>network=network,<EOL>distributions=distributions,<EOL>entropy_regularization=entropy_regularization<EOL>)<EOL>

Initializes the DQN agent.

Args:
    update_mode (spec): Update mode specification, with the following attributes:
        - unit: 'timesteps' if given (default: 'timesteps').
        - batch_size: integer (default: 32).
        - frequency: integer (default: 4).
    memory (spec): Memory specification, see core.memories module for more information
        (default: {type='replay', include_next_states=true, capacity=1000*batch_size}).
    optimizer (spec): Optimizer specification, see core.optimizers module for more
        information (default: {type='adam', learning_rate=1e-3}).
    target_sync_frequency (int): Target network sync frequency (default: 10000).
    target_update_weight (float): Target network update weight (default: 1.0).
    double_q_model (bool): Specifies whether double DQN mode is used (default: false).
    huber_loss (float): Huber loss clipping (default: none).


<EOL>if update_mode is None:<EOL><INDENT>update_mode = dict(<EOL>unit='<STR_LIT>',<EOL>batch_size=<NUM_LIT:10><EOL>)<EOL><DEDENT>elif '<STR_LIT>' in update_mode:<EOL><INDENT>assert update_mode['<STR_LIT>'] == '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>update_mode['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if memory is None:<EOL><INDENT>memory = dict(<EOL>type='<STR_LIT>',<EOL>include_next_states=True,<EOL>capacity=(<NUM_LIT:1000> * update_mode['<STR_LIT>'])<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert memory['<STR_LIT>']<EOL><DEDENT>if optimizer is None:<EOL><INDENT>optimizer = dict(<EOL>type='<STR_LIT>',<EOL>learning_rate=<NUM_LIT><EOL>)<EOL><DEDENT>self.target_sync_frequency = target_sync_frequency<EOL>self.target_update_weight = target_update_weight<EOL>self.double_q_model = double_q_model<EOL>self.huber_loss = huber_loss<EOL>super(DQNNstepAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing,<EOL>update_mode=update_mode,<EOL>memory=memory,<EOL>optimizer=optimizer,<EOL>discount=discount,<EOL>network=network,<EOL>distributions=distributions,<EOL>entropy_regularization=entropy_regularization<EOL>)<EOL>

Initializes the DQN n-step agent.

Args:
    update_mode (spec): Update mode specification, with the following attributes:
        - unit: 'episodes' if given (default: 'episodes').
        - batch_size: integer (default: 10).
        - frequency: integer (default: batch_size).
    memory (spec): Memory specification, see core.memories module for more information
        (default: {type='latest', include_next_states=true, capacity=1000*batch_size}).
    optimizer (spec): Optimizer specification, see core.optimizers module for more
        information (default: {type='adam', learning_rate=1e-3}).
    target_sync_frequency (int): Target network sync frequency (default: 10000).
    target_update_weight (float): Target network update weight (default: 1.0).
    double_q_model (bool): Specifies whether double DQN mode is used (default: false).
    huber_loss (float): Huber loss clipping (default: none).