def create_act_operations(self, states, internals, deterministic, independent, index):

<EOL>operations = list()<EOL>if self.variable_noise is not None and self.variable_noise > <NUM_LIT:0.0>:<EOL><INDENT>self.fn_actions_and_internals(<EOL>states=states,<EOL>internals=internals,<EOL>deterministic=deterministic<EOL>)<EOL>noise_deltas = list()<EOL>for variable in self.get_variables():<EOL><INDENT>noise_delta = tf.random_normal(shape=util.shape(variable), mean=<NUM_LIT:0.0>, stddev=self.variable_noise)<EOL>noise_deltas.append(noise_delta)<EOL>operations.append(variable.assign_add(delta=noise_delta))<EOL><DEDENT><DEDENT>with tf.control_dependencies(control_inputs=operations):<EOL><INDENT>self.actions_output, self.internals_output = self.fn_actions_and_internals(<EOL>states=states,<EOL>internals=internals,<EOL>deterministic=deterministic<EOL>)<EOL><DEDENT>with tf.control_dependencies(control_inputs=[self.actions_output[name] for name in sorted(self.actions_output)]):<EOL><INDENT>operations = list()<EOL>if self.variable_noise is not None and self.variable_noise > <NUM_LIT:0.0>:<EOL><INDENT>for variable, noise_delta in zip(self.get_variables(), noise_deltas):<EOL><INDENT>operations.append(variable.assign_sub(delta=noise_delta))<EOL><DEDENT><DEDENT><DEDENT>with tf.control_dependencies(control_inputs=operations):<EOL><INDENT>for name in sorted(self.actions_exploration):<EOL><INDENT>self.actions_output[name] = tf.cond(<EOL>pred=self.deterministic_input,<EOL>true_fn=(lambda: self.actions_output[name]),<EOL>false_fn=(lambda: self.fn_action_exploration(<EOL>action=self.actions_output[name],<EOL>exploration=self.actions_exploration[name],<EOL>action_spec=self.actions_spec[name]<EOL>))<EOL>)<EOL><DEDENT><DEDENT>def independent_act():<EOL><INDENT>"""<STR_LIT>"""<EOL>return self.global_timestep<EOL><DEDENT>def normal_act():<EOL><INDENT>"""<STR_LIT>"""<EOL>operations = list()<EOL>batch_size = tf.shape(input=states[next(iter(sorted(states)))])[<NUM_LIT:0>]<EOL>for name in sorted(states):<EOL><INDENT>operations.append(tf.assign(<EOL>ref=self.list_states_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],<EOL>value=states[name]<EOL>))<EOL><DEDENT>for name in sorted(internals):<EOL><INDENT>operations.append(tf.assign(<EOL>ref=self.list_internals_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],<EOL>value=internals[name]<EOL>))<EOL><DEDENT>for name in sorted(self.actions_output):<EOL><INDENT>operations.append(tf.assign(<EOL>ref=self.list_actions_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],<EOL>value=self.actions_output[name]<EOL>))<EOL><DEDENT>with tf.control_dependencies(control_inputs=operations):<EOL><INDENT>operations = list()<EOL>operations.append(tf.assign(<EOL>ref=self.list_buffer_index[index: index+<NUM_LIT:1>],<EOL>value=tf.add(self.list_buffer_index[index: index+<NUM_LIT:1>], tf.constant([<NUM_LIT:1>]))<EOL>))<EOL>operations.append(tf.assign_add(<EOL>ref=self.timestep,<EOL>value=tf.to_int64(x=batch_size)<EOL>))<EOL>operations.append(tf.assign_add(<EOL>ref=self.global_timestep,<EOL>value=tf.to_int64(x=batch_size)<EOL>))<EOL><DEDENT>with tf.control_dependencies(control_inputs=operations):<EOL><INDENT>return self.global_timestep + <NUM_LIT:0><EOL><DEDENT><DEDENT>self.timestep_output = tf.cond(<EOL>pred=independent,<EOL>true_fn=independent_act,<EOL>false_fn=normal_act<EOL>)<EOL>

Creates and stores tf operations that are fetched when calling act(): actions_output, internals_output and
timestep_output.

Args:
    states (dict): Dict of state tensors (each key represents one state space component).
    internals (dict): Dict of prior internal state tensors (each key represents one internal state component).
    deterministic: 0D (bool) tensor (whether to not use action exploration).
    independent (bool): 0D (bool) tensor (whether to store states/internals/action in local buffer).


def tf_preprocess(self, states, actions, reward):

<EOL>for name in sorted(self.states_preprocessing):<EOL><INDENT>states[name] = self.states_preprocessing[name].process(tensor=states[name])<EOL><DEDENT>if self.reward_preprocessing is not None:<EOL><INDENT>reward = self.reward_preprocessing.process(tensor=reward)<EOL><DEDENT>return states, actions, reward<EOL>

Applies preprocessing ops to the raw states/action/reward inputs.

Args:
    states (dict): Dict of raw state tensors.
    actions (dict): Dict or raw action tensors.
    reward: 1D (float) raw rewards tensor.

Returns: The preprocessed versions of the input tensors.


def get_variables(self, include_submodules=False, include_nontrainable=False):

if include_nontrainable:<EOL><INDENT>model_variables = [self.all_variables[key] for key in sorted(self.all_variables)]<EOL>states_preprocessing_variables = [<EOL>variable for name in sorted(self.states_preprocessing)<EOL>for variable in self.states_preprocessing[name].get_variables()<EOL>]<EOL>model_variables += states_preprocessing_variables<EOL>actions_exploration_variables = [<EOL>variable for name in sorted(self.actions_exploration)<EOL>for variable in self.actions_exploration[name].get_variables()<EOL>]<EOL>model_variables += actions_exploration_variables<EOL>if self.reward_preprocessing is not None:<EOL><INDENT>reward_preprocessing_variables = self.reward_preprocessing.get_variables()<EOL>model_variables += reward_preprocessing_variables<EOL><DEDENT><DEDENT>else:<EOL><INDENT>model_variables = [self.variables[key] for key in sorted(self.variables)]<EOL><DEDENT>return model_variables<EOL>

Returns the TensorFlow variables used by the model.

Args:
    include_submodules: Includes variables of submodules (e.g. baseline, target network)
        if true.
    include_nontrainable: Includes non-trainable variables if true.

Returns:
    List of variables.


<EOL>self.network = None<EOL>self.states_spec = states<EOL>self.internals_spec = dict()<EOL>self.actions_spec = actions<EOL>self.scope = scope<EOL>self.device = device<EOL>if saver is None or saver.get('<STR_LIT>') is None:<EOL><INDENT>self.saver_spec = None<EOL><DEDENT>else:<EOL><INDENT>self.saver_spec = saver<EOL><DEDENT>if summarizer is None or summarizer.get('<STR_LIT>') is None:<EOL><INDENT>self.summarizer_spec = None<EOL>self.summary_labels = set()<EOL><DEDENT>else:<EOL><INDENT>self.summarizer_spec = summarizer<EOL>self.summary_labels = set(self.summarizer_spec.get('<STR_LIT>', ()))<EOL><DEDENT>self.summarizer = None<EOL>self.graph_summary = None<EOL>self.summarizer_init_op = None<EOL>self.flush_summarizer = None<EOL>self.execution_spec = execution<EOL>self.execution_type = self.execution_spec["<STR_LIT:type>"]<EOL>self.session_config = self.execution_spec["<STR_LIT>"]<EOL>self.distributed_spec = self.execution_spec["<STR_LIT>"]<EOL>assert batching_capacity is None or (isinstance(batching_capacity, int) and batching_capacity > <NUM_LIT:0>)<EOL>self.batching_capacity = batching_capacity or <NUM_LIT:1>  <EOL>self.tf_session_dump_dir = tf_session_dump_dir<EOL>self.num_parallel = self.execution_spec.get('<STR_LIT>')<EOL>if self.num_parallel is None:<EOL><INDENT>self.num_parallel = <NUM_LIT:1><EOL><DEDENT>self.list_states_buffer = dict()<EOL>self.list_internals_buffer = dict()<EOL>self.list_actions_buffer = dict()<EOL>self.list_buffer_index = [None for _ in range(self.num_parallel)]<EOL>self.episode_output = None<EOL>self.episode_index_input = None<EOL>self.unbuffered_episode_output = None<EOL>assert variable_noise is None or variable_noise > <NUM_LIT:0.0><EOL>self.variable_noise = variable_noise<EOL>self.states_preprocessing_spec = states_preprocessing<EOL>self.actions_exploration_spec = actions_exploration<EOL>self.reward_preprocessing_spec = reward_preprocessing<EOL>self.variables = None<EOL>self.all_variables = None<EOL>self.registered_variables = None<EOL>self.list_timestep = [None for _ in range(self.num_parallel)]<EOL>self.episode = None<EOL>self.global_timestep = None<EOL>self.global_episode = None<EOL>self.states_input = dict()<EOL>self.states_preprocessing = dict()<EOL>self.internals_input = dict()<EOL>self.internals_init = dict()<EOL>self.actions_input = dict()<EOL>self.actions_exploration = dict()<EOL>self.terminal_input = None<EOL>self.reward_input = None<EOL>self.reward_preprocessing = None<EOL>self.deterministic_input = None<EOL>self.independent_input = None<EOL>self.update_input = None<EOL>self.fn_initialize = None<EOL>self.fn_preprocess = None<EOL>self.fn_actions_and_internals = None<EOL>self.fn_observe_timestep = None<EOL>self.fn_action_exploration = None<EOL>self.graph = None<EOL>self.global_model = None<EOL>self.is_local_model = True<EOL>self.server = None<EOL>self.summarizer = None<EOL>self.saver = None<EOL>self.saver_directory = None<EOL>self.scaffold = None<EOL>self.session = None<EOL>self.monitored_session = None<EOL>self.actions_output = None<EOL>self.internals_output = None<EOL>self.timestep_output = None<EOL>self.list_buffer_index_reset_op = None<EOL>self.setup()<EOL>

Model.

Args:
    states (spec): The state-space description dictionary.
    actions (spec): The action-space description dictionary.
    scope (str): The root scope str to use for tf variable scoping.
    device (str): The name of the device to run the graph of this model on.
    saver (spec): Dict specifying whether and how to save the model's parameters.
    summarizer (spec): Dict specifying which tensorboard summaries should be created and added to the graph.
    execution (spec): Dict specifying whether and how to do distributed training on the model's graph.
    batching_capacity (int): Batching capacity.
    variable_noise (float): The stddev value of a Normal distribution used for adding random
        noise to the model's output (for each batch, noise can be toggled and - if active - will be resampled).
        Use None for not adding any noise.
    states_preprocessing (spec / dict of specs): Dict specifying whether and how to preprocess state signals
        (e.g. normalization, greyscale, etc..).
    actions_exploration (spec / dict of specs): Dict specifying whether and how to add exploration to the model's
        "action outputs" (e.g. epsilon-greedy).
    reward_preprocessing (spec): Dict specifying whether and how to preprocess rewards coming
        from the Environment (e.g. reward normalization).
    tf_session_dump_dir (str): If non-empty string, all session.run calls will be dumped using the tensorflow
        offline-debug session into the given directory.
    execution: (dict)
        - num_parallel: (int) number of parallel episodes


if self.execution_type == "<STR_LIT>":<EOL><INDENT>global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)<EOL>init_op = tf.variables_initializer(var_list=global_variables)<EOL>if self.summarizer_init_op is not None:<EOL><INDENT>init_op = tf.group(init_op, self.summarizer_init_op)<EOL><DEDENT>if self.graph_summary is None:<EOL><INDENT>ready_op = tf.report_uninitialized_variables(var_list=global_variables)<EOL>ready_for_local_init_op = None<EOL>local_init_op = None<EOL><DEDENT>else:<EOL><INDENT>ready_op = None<EOL>ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)<EOL>local_init_op = self.graph_summary<EOL><DEDENT><DEDENT>else:<EOL><INDENT>global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)<EOL>local_variables = self.get_variables(include_submodules=True, include_nontrainable=True)<EOL>init_op = tf.variables_initializer(var_list=global_variables)<EOL>if self.summarizer_init_op is not None:<EOL><INDENT>init_op = tf.group(init_op, self.summarizer_init_op)<EOL><DEDENT>ready_op = tf.report_uninitialized_variables(var_list=(global_variables + local_variables))<EOL>ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)<EOL>if self.graph_summary is None:<EOL><INDENT>local_init_op = tf.group(<EOL>tf.variables_initializer(var_list=local_variables),<EOL>*(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(<EOL>self.get_variables(include_submodules=True),<EOL>self.global_model.get_variables(include_submodules=True)<EOL>))<EOL>)<EOL><DEDENT>else:<EOL><INDENT>local_init_op = tf.group(<EOL>tf.variables_initializer(var_list=local_variables),<EOL>self.graph_summary,<EOL>*(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(<EOL>self.get_variables(include_submodules=True),<EOL>self.global_model.get_variables(include_submodules=True)<EOL>))<EOL>)<EOL><DEDENT><DEDENT>def init_fn(scaffold, session):<EOL><INDENT>if self.saver_spec is not None and self.saver_spec.get('<STR_LIT>', True):<EOL><INDENT>directory = self.saver_spec['<STR_LIT>']<EOL>file = self.saver_spec.get('<STR_LIT:file>')<EOL>if file is None:<EOL><INDENT>file = tf.train.latest_checkpoint(<EOL>checkpoint_dir=directory,<EOL>latest_filename=None  <EOL>)<EOL><DEDENT>elif not os.path.isfile(file):<EOL><INDENT>file = os.path.join(directory, file)<EOL><DEDENT>if file is not None:<EOL><INDENT>try:<EOL><INDENT>scaffold.saver.restore(sess=session, save_path=file)<EOL>session.run(fetches=self.list_buffer_index_reset_op)<EOL><DEDENT>except tf.errors.NotFoundError:<EOL><INDENT>raise TensorForceError("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.scaffold = tf.train.Scaffold(<EOL>init_op=init_op,<EOL>init_feed_dict=None,<EOL>init_fn=init_fn,<EOL>ready_op=ready_op,<EOL>ready_for_local_init_op=ready_for_local_init_op,<EOL>local_init_op=local_init_op,<EOL>summary_op=None,<EOL>saver=self.saver,<EOL>copy_from_scaffold=None<EOL>)<EOL>

Creates the tf.train.Scaffold object and assigns it to self.scaffold.
Other fields of the Scaffold are generated automatically.


def act(self, states, internals, deterministic=False, independent=False, fetch_tensors=None, index=<NUM_LIT:0>):

name = next(iter(states))<EOL>state = np.asarray(states[name])<EOL>batched = (state.ndim != len(self.states_spec[name]['<STR_LIT>']))<EOL>if batched:<EOL><INDENT>assert state.shape[<NUM_LIT:0>] <= self.batching_capacity<EOL><DEDENT>fetches = [self.actions_output, self.internals_output, self.timestep_output]<EOL>if self.network is not None and fetch_tensors is not None:<EOL><INDENT>for name in fetch_tensors:<EOL><INDENT>valid, tensor = self.network.get_named_tensor(name)<EOL>if valid:<EOL><INDENT>fetches.append(tensor)<EOL><DEDENT>else:<EOL><INDENT>keys = self.network.get_list_of_named_tensor()<EOL>raise TensorForceError('<STR_LIT>'.format(name, keys))<EOL><DEDENT><DEDENT><DEDENT>feed_dict = self.get_feed_dict(<EOL>states=states,<EOL>internals=internals,<EOL>deterministic=deterministic,<EOL>independent=independent,<EOL>index=index<EOL>)<EOL>fetch_list = self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)<EOL>actions, internals, timestep = fetch_list[<NUM_LIT:0>:<NUM_LIT:3>]<EOL>if not batched:<EOL><INDENT>actions = {name: actions[name][<NUM_LIT:0>] for name in sorted(actions)}<EOL>internals = {name: internals[name][<NUM_LIT:0>] for name in sorted(internals)}<EOL><DEDENT>if self.network is not None and fetch_tensors is not None:<EOL><INDENT>fetch_dict = dict()<EOL>for index_, tensor in enumerate(fetch_list[<NUM_LIT:3>:]):<EOL><INDENT>name = fetch_tensors[index_]<EOL>fetch_dict[name] = tensor<EOL><DEDENT>return actions, internals, timestep, fetch_dict<EOL><DEDENT>else:<EOL><INDENT>return actions, internals, timestep<EOL><DEDENT>

Does a forward pass through the model to retrieve action (outputs) given inputs for state (and internal
state, if applicable (e.g. RNNs))

Args:
    states (dict): Dict of state values (each key represents one state space component).
    internals (dict): Dict of internal state values (each key represents one internal state component).
    deterministic (bool): If True, will not apply exploration after actions are calculated.
    independent (bool): If true, action is not followed by observe (and hence not included
        in updates).
    fetch_tensors (list): List of names of additional tensors (from the model's network) to fetch (and return).
    index: (int) index of the episode we want to produce the next action

Returns:
    tuple:
        - Actual action-outputs (batched if state input is a batch).
        - Actual values of internal states (if applicable) (batched if state input is a batch).
        - The timestep (int) after calculating the (batch of) action(s).


return dict()<EOL>

Returns a dictionary of component name to component of all the components within this model.

Returns:
    (dict) The mapping of name to component.


components = self.get_components()<EOL>components = [components[name] for name in sorted(components)]<EOL>return set(filter(lambda x: isinstance(x, util.SavableComponent), components))<EOL>

Returns the list of all of the components this model consists of that can be individually saved and restored.
For instance the network or distribution.

Returns:
    List of util.SavableComponent


fetches = [self.global_episode, self.global_timestep]<EOL>for name in sorted(self.states_preprocessing):<EOL><INDENT>fetch = self.states_preprocessing[name].reset()<EOL>if fetch is not None:<EOL><INDENT>fetches.extend(fetch)<EOL><DEDENT><DEDENT>if self.flush_summarizer is not None:<EOL><INDENT>fetches.append(self.flush_summarizer)<EOL><DEDENT>fetch_list = self.monitored_session.run(fetches=fetches)<EOL>episode, timestep = fetch_list[:<NUM_LIT:2>]<EOL>return episode, timestep, self.internals_init<EOL>

Resets the model to its initial state on episode start. This should also reset all preprocessor(s).

Returns:
    tuple:
        Current episode, timestep counter and the shallow-copied list of internal state initialization Tensors.


def create_observe_operations(self, terminal, reward, index):

<EOL>num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('<STR_LIT:int>'))<EOL>increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))<EOL>increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))<EOL>with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):<EOL><INDENT>fn = (lambda x: tf.stop_gradient(input=x[:self.list_buffer_index[index]]))<EOL>states = util.map_tensors(fn=fn, tensors=self.list_states_buffer, index=index)<EOL>internals = util.map_tensors(fn=fn, tensors=self.list_internals_buffer, index=index)<EOL>actions = util.map_tensors(fn=fn, tensors=self.list_actions_buffer, index=index)<EOL>terminal = tf.stop_gradient(input=terminal)<EOL>reward = tf.stop_gradient(input=reward)<EOL>observation = self.fn_observe_timestep(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL><DEDENT>with tf.control_dependencies(control_inputs=(observation,)):<EOL><INDENT>reset_index = tf.assign(ref=self.list_buffer_index[index], value=<NUM_LIT:0>)<EOL><DEDENT>with tf.control_dependencies(control_inputs=(reset_index,)):<EOL><INDENT>self.episode_output = self.global_episode + <NUM_LIT:0><EOL><DEDENT>self.list_buffer_index_reset_op = tf.group(<EOL>*(tf.assign(ref=self.list_buffer_index[n], value=<NUM_LIT:0>) for n in range(self.num_parallel))<EOL>)<EOL>

Returns the tf op to fetch when an observation batch is passed in (e.g. an episode's rewards and
terminals). Uses the filled tf buffers for states, actions and internals to run
the tf_observe_timestep (model-dependent), resets buffer index and increases counters (episodes,
timesteps).

Args:
    terminal: The 1D tensor (bool) of terminal signals to process (more than one True within that list is ok).
    reward: The 1D tensor (float) of rewards to process.

Returns: Tf op to fetch when `observe()` is called.


if self.execution_type == "<STR_LIT>":<EOL><INDENT>global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)<EOL><DEDENT>else:<EOL><INDENT>global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)<EOL><DEDENT>for c in self.get_savable_components():<EOL><INDENT>c.register_saver_ops()<EOL><DEDENT>self.saver = tf.train.Saver(<EOL>var_list=global_variables,  <EOL>reshape=False,<EOL>sharded=False,<EOL>max_to_keep=<NUM_LIT:5>,<EOL>keep_checkpoint_every_n_hours=<NUM_LIT>,<EOL>name=None,<EOL>restore_sequentially=False,<EOL>saver_def=None,<EOL>builder=None,<EOL>defer_build=False,<EOL>allow_empty=True,<EOL>write_version=tf.train.SaverDef.V2,<EOL>pad_step_number=False,<EOL>save_relative_paths=True<EOL>)<EOL>

Creates the tf.train.Saver object and stores it in self.saver.


if custom_getter is None:<EOL><INDENT>def custom_getter(getter, name, registered=False, **kwargs):<EOL><INDENT>"""<STR_LIT>"""<EOL>if registered:<EOL><INDENT>self.registered_variables.add(name)<EOL><DEDENT>elif name in self.registered_variables:<EOL><INDENT>registered = True<EOL><DEDENT>variable = getter(name=name, **kwargs)<EOL>if registered:<EOL><INDENT>pass<EOL><DEDENT>elif name in self.all_variables:<EOL><INDENT>assert variable is self.all_variables[name]<EOL>if kwargs.get('<STR_LIT>', True):<EOL><INDENT>assert variable is self.variables[name]<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name=name, tensor=variable)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self.all_variables[name] = variable<EOL>if kwargs.get('<STR_LIT>', True):<EOL><INDENT>self.variables[name] = variable<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name=name, tensor=variable)<EOL><DEDENT><DEDENT><DEDENT>return variable<EOL><DEDENT><DEDENT>self.fn_initialize = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_initialize,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.fn_preprocess = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_preprocess,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.fn_actions_and_internals = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_actions_and_internals,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.fn_observe_timestep = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_observe_timestep,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.fn_action_exploration = tf.make_template(<EOL>name_='<STR_LIT>',<EOL>func_=self.tf_action_exploration,<EOL>custom_getter_=custom_getter<EOL>)<EOL>return custom_getter<EOL>

Allows child models to create model's component objects, such as optimizer(s), memory(s), etc..
Creates all tensorflow functions via tf.make_template calls on all the class' "tf_"-methods.

Args:
    custom_getter: The `custom_getter_` object to use for `tf.make_template` when creating TensorFlow functions.
        If None, use a default custom_getter_.

Returns: The custom_getter passed in (or a default one if custom_getter was None).


def tf_actions_and_internals(self, states, internals, deterministic):

raise NotImplementedError<EOL>

Creates and returns the TensorFlow operations for retrieving the actions and - if applicable -
the posterior internal state Tensors in reaction to the given input states (and prior internal states).

Args:
    states (dict): Dict of state tensors (each key represents one state space component).
    internals (dict): Dict of internal state tensors (each key represents one internal space component).
    deterministic: Boolean tensor indicating whether action should be chosen
        deterministically.

Returns:
    tuple:
        1) dict of output actions (with or without exploration applied (see `deterministic`))
        2) list of posterior internal state Tensors (empty for non-internal state models)


def create_atomic_observe_operations(self, states, actions, internals, terminal, reward, index):

<EOL>num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('<STR_LIT:int>'))<EOL>increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))<EOL>increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))<EOL>with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):<EOL><INDENT>states = util.map_tensors(fn=tf.stop_gradient, tensors=states)<EOL>internals = util.map_tensors(fn=tf.stop_gradient, tensors=internals)<EOL>actions = util.map_tensors(fn=tf.stop_gradient, tensors=actions)<EOL>terminal = tf.stop_gradient(input=terminal)<EOL>reward = tf.stop_gradient(input=reward)<EOL>observation = self.fn_observe_timestep(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL><DEDENT>with tf.control_dependencies(control_inputs=(observation,)):<EOL><INDENT>self.unbuffered_episode_output = self.global_episode + <NUM_LIT:0><EOL><DEDENT>

Returns the tf op to fetch when unbuffered observations are passed in.

Args:
    states (any): One state (usually a value tuple) or dict of states if multiple states are expected.
    actions (any): One action (usually a value tuple) or dict of states if multiple actions are expected.
    internals (any): Internal list.
    terminal (bool): boolean indicating if the episode terminated after the observation.
    reward (float): scalar reward that resulted from executing the action.

Returns: Tf op to fetch when `observe()` is called.


def tf_optimization(self, states, internals, actions, terminal, reward, next_states=None, next_internals=None):

Creates the TensorFlow operations for performing an optimization update step based
on the given input states and actions batch.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    actions: Dict of action tensors.
    terminal: Terminal boolean tensor.
    reward: Reward tensor.
    next_states: Dict of successor state tensors.
    next_internals: List of posterior internal state tensors.

Returns:
    The optimization operation.


def tf_import_experience(self, states, internals, actions, terminal, reward):

return self.memory.store(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL>

Imports experiences into the TensorFlow memory structure. Can be used to import
off-policy data.

:param states: Dict of state values to import with keys as state names and values as values to set.
:param internals: Internal values to set, can be fetched from agent via agent.current_internals
    if no values available.
:param actions: Dict of action values to import with keys as action names and values as values to set.
:param terminal: Terminal value(s)
:param reward: Reward value(s)


def tf_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):

<EOL>loss_per_instance = self.fn_loss_per_instance(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward,<EOL>next_states=next_states,<EOL>next_internals=next_internals,<EOL>update=update,<EOL>reference=reference<EOL>)<EOL>updated = self.memory.update_batch(loss_per_instance=loss_per_instance)<EOL>with tf.control_dependencies(control_inputs=(updated,)):<EOL><INDENT>loss = tf.reduce_mean(input_tensor=loss_per_instance, axis=<NUM_LIT:0>)<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.scalar(name='<STR_LIT>', tensor=loss)<EOL><DEDENT>losses = self.fn_regularization_losses(states=states, internals=internals, update=update)<EOL>if len(losses) > <NUM_LIT:0>:<EOL><INDENT>loss += tf.add_n(inputs=[losses[name] for name in sorted(losses)])<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>for name in sorted(losses):<EOL><INDENT>tf.contrib.summary.scalar(name=('<STR_LIT>' + name), tensor=losses[name])<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in self.summary_labels or '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.scalar(name='<STR_LIT>', tensor=loss)<EOL><DEDENT>return loss<EOL><DEDENT>

Creates the TensorFlow operations for calculating the full loss of a batch.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    actions: Dict of action tensors.
    terminal: Terminal boolean tensor.
    reward: Reward tensor.
    next_states: Dict of successor state tensors.
    next_internals: List of posterior internal state tensors.
    update: Boolean tensor indicating whether this call happens during an update.
    reference: Optional reference tensor(s), in case of a comparative loss.

Returns:
    Loss tensor.


def tf_discounted_cumulative_reward(self, terminal, reward, discount=None, final_reward=<NUM_LIT:0.0>, horizon=<NUM_LIT:0>):

<EOL>if discount is None:<EOL><INDENT>discount = self.discount<EOL><DEDENT>def cumulate(cumulative, reward_terminal_horizon_subtract):<EOL><INDENT>rew, is_terminal, is_over_horizon, sub = reward_terminal_horizon_subtract<EOL>return tf.where(<EOL>condition=is_terminal,<EOL>x=rew,<EOL>y=tf.where(<EOL>condition=is_over_horizon,<EOL>x=(rew + cumulative * discount - sub),<EOL>y=(rew + cumulative * discount)<EOL>)<EOL>)<EOL><DEDENT>def len_(cumulative, term):<EOL><INDENT>return tf.where(<EOL>condition=term,<EOL>x=tf.ones(shape=(), dtype=tf.int32),<EOL>y=cumulative + <NUM_LIT:1><EOL>)<EOL><DEDENT>reward = tf.reverse(tensor=reward, axis=(<NUM_LIT:0>,))<EOL>terminal = tf.reverse(tensor=terminal, axis=(<NUM_LIT:0>,))<EOL>lengths = tf.scan(fn=len_, elems=terminal, initializer=<NUM_LIT:0>)<EOL>off_horizon = tf.greater(lengths, tf.fill(dims=tf.shape(lengths), value=horizon))<EOL>if horizon > <NUM_LIT:0>:<EOL><INDENT>horizon_subtractions = tf.map_fn(lambda x: (discount ** horizon) * x, reward, dtype=tf.float32)<EOL>horizon_subtractions = tf.concat([np.zeros(shape=(horizon,)), horizon_subtractions], axis=<NUM_LIT:0>)<EOL>horizon_subtractions = tf.slice(horizon_subtractions, begin=(<NUM_LIT:0>,), size=tf.shape(reward))<EOL><DEDENT>else:<EOL><INDENT>horizon_subtractions = tf.zeros(shape=tf.shape(reward))<EOL><DEDENT>reward = tf.scan(<EOL>fn=cumulate,<EOL>elems=(reward, terminal, off_horizon, horizon_subtractions),<EOL>initializer=final_reward if horizon != <NUM_LIT:1> else <NUM_LIT:0.0><EOL>)<EOL>return tf.reverse(tensor=reward, axis=(<NUM_LIT:0>,))<EOL>

Creates and returns the TensorFlow operations for calculating the sequence of discounted cumulative rewards
for a given sequence of single rewards.

Example:
single rewards = 2.0 1.0 0.0 0.5 1.0 -1.0
terminal = False, False, False, False True False
gamma = 0.95
final_reward = 100.0 (only matters for last episode (r=-1.0) as this episode has no terminal signal)
horizon=3
output = 2.95 1.45 1.38 1.45 1.0 94.0

Args:
    terminal: Tensor (bool) holding the is-terminal sequence. This sequence may contain more than one
        True value. If its very last element is False (not terminating), the given `final_reward` value
        is assumed to follow the last value in the single rewards sequence (see below).
    reward: Tensor (float) holding the sequence of single rewards. If the last element of `terminal` is False,
        an assumed last reward of the value of `final_reward` will be used.
    discount (float): The discount factor (gamma). By default, take the Model's discount factor.
    final_reward (float): Reward value to use if last episode in sequence does not terminate (terminal sequence
        ends with False). This value will be ignored if horizon == 1 or discount == 0.0.
    horizon (int): The length of the horizon (e.g. for n-step cumulative rewards in continuous tasks
        without terminal signals). Use 0 (default) for an infinite horizon. Note that horizon=1 leads to the
        exact same results as a discount factor of 0.0.

Returns:
    Discounted cumulative reward tensor with the same shape as `reward`.


self.update_mode = update_mode<EOL>self.memory_spec = memory<EOL>self.optimizer_spec = optimizer<EOL>assert discount is None or discount >= <NUM_LIT:0.0><EOL>self.discount = discount<EOL>self.memory = None<EOL>self.optimizer = None<EOL>self.fn_discounted_cumulative_reward = None<EOL>self.fn_reference = None<EOL>self.fn_loss_per_instance = None<EOL>self.fn_regularization_losses = None<EOL>self.fn_loss = None<EOL>self.fn_optimization = None<EOL>self.fn_import_experience = None<EOL>super(MemoryModel, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>batching_capacity=batching_capacity,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing<EOL>)<EOL>

Memory model.

Args:
    states (spec): The state-space description dictionary.
    actions (spec): The action-space description dictionary.
    scope (str): The root scope str to use for tf variable scoping.
    device (str): The name of the device to run the graph of this model on.
    saver (spec): Dict specifying whether and how to save the model's parameters.
    summarizer (spec): Dict specifying which tensorboard summaries should be created and added to the graph.
    execution (spec): Dict specifying whether and how to do distributed training on the model's graph.
    batching_capacity (int): Batching capacity.
    variable_noise (float): The stddev value of a Normal distribution used for adding random
        noise to the model's output (for each batch, noise can be toggled and - if active - will be resampled).
        Use None for not adding any noise.
    states_preprocessing (spec / dict of specs): Dict specifying whether and how to preprocess state signals
        (e.g. normalization, greyscale, etc..).
    actions_exploration (spec / dict of specs): Dict specifying whether and how to add exploration to the model's
        "action outputs" (e.g. epsilon-greedy).
    reward_preprocessing (spec): Dict specifying whether and how to preprocess rewards coming
        from the Environment (e.g. reward normalization).
    update_mode (spec): Update mode.
    memory (spec): Memory.
    optimizer (spec): Dict specifying the tf optimizer to use for tuning the model's trainable parameters.
    discount (float): The RL reward discount factor (gamma).


super(MemoryModel, self).tf_initialize()<EOL>self.memory.initialize()<EOL>

Also initializes our Memory object (self.memory).


def tf_baseline_loss(self, states, internals, reward, update, reference=None):

if self.baseline_mode == '<STR_LIT>':<EOL><INDENT>loss = self.baseline.loss(<EOL>states=states,<EOL>internals=internals,<EOL>reward=reward,<EOL>update=update,<EOL>reference=reference<EOL>)<EOL><DEDENT>elif self.baseline_mode == '<STR_LIT>':<EOL><INDENT>loss = self.baseline.loss(<EOL>states=self.network.apply(x=states, internals=internals, update=update),<EOL>internals=internals,<EOL>reward=reward,<EOL>update=update,<EOL>reference=reference<EOL>)<EOL><DEDENT>regularization_loss = self.baseline.regularization_loss()<EOL>if regularization_loss is not None:<EOL><INDENT>loss += regularization_loss<EOL><DEDENT>return loss<EOL>

Creates the TensorFlow operations for calculating the baseline loss of a batch.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    reward: Reward tensor.
    update: Boolean tensor indicating whether this call happens during an update.
    reference: Optional reference tensor(s), in case of a comparative loss.

Returns:
    Loss tensor.


def tf_import_demo_experience(self, states, internals, actions, terminal, reward):

return self.demo_memory.store(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL>


variables = self.target_network.get_variables() + [<EOL>variable for name in sorted(self.target_distributions)<EOL>for variable in self.target_distributions[name].get_variables()<EOL>]<EOL>source_variables = self.network.get_variables() + [<EOL>variable for name in sorted(self.distributions)<EOL>for variable in self.distributions[name].get_variables()<EOL>]<EOL>arguments = dict(<EOL>time=self.global_timestep,<EOL>variables=variables,<EOL>source_variables=source_variables<EOL>)<EOL>if self.global_model is not None:<EOL><INDENT>arguments['<STR_LIT>'] = self.global_model.target_network.get_variables() + [<EOL>variable for name in sorted(self.global_model.target_distributions)<EOL>for variable in self.global_model.target_distributions[name].get_variables()<EOL>]<EOL><DEDENT>return arguments<EOL>

Returns the target optimizer arguments including the time, the list of variables to
optimize, and various functions which the optimizer might require to perform an update
step.

Returns:
    Target optimizer arguments as dict.


self.scope = scope<EOL>self.device = device<EOL>self.saver = saver<EOL>self.summarizer = summarizer<EOL>self.execution = sanity_check_execution_spec(execution)<EOL>super(RandomAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity<EOL>)<EOL>

Initializes the random agent.

Args:
    scope (str): TensorFlow scope (default: name of agent).
    device: TensorFlow device (default: none)
    saver (spec): Saver specification, with the following attributes (default: none):
        - directory: model directory.
        - file: model filename (optional).
        - seconds or steps: save frequency (default: 600 seconds).
        - load: specifies whether model is loaded, if existent (default: true).
        - basename: optional file basename (default: 'model.ckpt').
    summarizer (spec): Summarizer specification, with the following attributes (default:
        none):
        - directory: summaries directory.
        - seconds or steps: summarize frequency (default: 120 seconds).
        - labels: list of summary labels to record (default: []).
        - meta_param_recorder_class: ???.
    execution (spec): Execution specification (see sanity_check_execution_spec for details).


<EOL>if update_mode is None:<EOL><INDENT>update_mode = dict(<EOL>unit='<STR_LIT>',<EOL>batch_size=<NUM_LIT:10><EOL>)<EOL><DEDENT>elif '<STR_LIT>' in update_mode:<EOL><INDENT>assert update_mode['<STR_LIT>'] == '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>update_mode['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if memory is None:<EOL><INDENT>memory = dict(<EOL>type='<STR_LIT>',<EOL>include_next_states=True,<EOL>capacity=(<NUM_LIT:1000> * update_mode['<STR_LIT>'])<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert memory['<STR_LIT>']<EOL><DEDENT>if optimizer is None:<EOL><INDENT>optimizer = dict(<EOL>type='<STR_LIT>',<EOL>learning_rate=<NUM_LIT><EOL>)<EOL><DEDENT>self.target_sync_frequency = target_sync_frequency<EOL>self.target_update_weight = target_update_weight<EOL>self.double_q_model = double_q_model<EOL>self.huber_loss = huber_loss<EOL>super(DQNNstepAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing,<EOL>update_mode=update_mode,<EOL>memory=memory,<EOL>optimizer=optimizer,<EOL>discount=discount,<EOL>network=network,<EOL>distributions=distributions,<EOL>entropy_regularization=entropy_regularization<EOL>)<EOL>

Initializes the DQN n-step agent.

Args:
    update_mode (spec): Update mode specification, with the following attributes:
        - unit: 'episodes' if given (default: 'episodes').
        - batch_size: integer (default: 10).
        - frequency: integer (default: batch_size).
    memory (spec): Memory specification, see core.memories module for more information
        (default: {type='latest', include_next_states=true, capacity=1000*batch_size}).
    optimizer (spec): Optimizer specification, see core.optimizers module for more
        information (default: {type='adam', learning_rate=1e-3}).
    target_sync_frequency (int): Target network sync frequency (default: 10000).
    target_update_weight (float): Target network update weight (default: 1.0).
    double_q_model (bool): Specifies whether double DQN mode is used (default: false).
    huber_loss (float): Huber loss clipping (default: none).


def observe(self, terminal, reward, index=<NUM_LIT:0>):

self.current_terminal = terminal<EOL>self.current_reward = reward<EOL>if self.batched_observe:<EOL><INDENT>self.observe_terminal[index].append(self.current_terminal)<EOL>self.observe_reward[index].append(self.current_reward)<EOL>if self.current_terminal or len(self.observe_terminal[index]) >= self.batching_capacity:<EOL><INDENT>self.episode = self.model.observe(<EOL>terminal=self.observe_terminal[index],<EOL>reward=self.observe_reward[index],<EOL>index=index<EOL>)<EOL>self.observe_terminal[index] = list()<EOL>self.observe_reward[index] = list()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.episode = self.model.observe(<EOL>terminal=self.current_terminal,<EOL>reward=self.current_reward<EOL>)<EOL><DEDENT>

Observe experience from the environment to learn from. Optionally pre-processes rewards
Child classes should call super to get the processed reward
EX: terminal, reward = super()...

Args:
    terminal (bool): boolean indicating if the episode terminated after the observation.
    reward (float): scalar reward that resulted from executing the action.


def act(self, states, deterministic=False, independent=False, fetch_tensors=None, buffered=True, index=<NUM_LIT:0>):

self.current_internals = self.next_internals<EOL>if self.unique_state:<EOL><INDENT>self.current_states = dict(state=np.asarray(states))<EOL><DEDENT>else:<EOL><INDENT>self.current_states = {name: np.asarray(states[name]) for name in sorted(states)}<EOL><DEDENT>if fetch_tensors is not None:<EOL><INDENT>self.current_actions, self.next_internals, self.timestep, self.fetched_tensors = self.model.act(<EOL>states=self.current_states,<EOL>internals=self.current_internals,<EOL>deterministic=deterministic,<EOL>independent=independent,<EOL>fetch_tensors=fetch_tensors,<EOL>index=index<EOL>)<EOL>if self.unique_action:<EOL><INDENT>return self.current_actions['<STR_LIT:action>'], self.fetched_tensors<EOL><DEDENT>else:<EOL><INDENT>return self.current_actions, self.fetched_tensors<EOL><DEDENT><DEDENT>self.current_actions, self.next_internals, self.timestep = self.model.act(<EOL>states=self.current_states,<EOL>internals=self.current_internals,<EOL>deterministic=deterministic,<EOL>independent=independent,<EOL>index=index<EOL>)<EOL>if buffered:<EOL><INDENT>if self.unique_action:<EOL><INDENT>return self.current_actions['<STR_LIT:action>']<EOL><DEDENT>else:<EOL><INDENT>return self.current_actions<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if self.unique_action:<EOL><INDENT>return self.current_actions['<STR_LIT:action>'], self.current_states, self.current_internals<EOL><DEDENT>else:<EOL><INDENT>return self.current_actions, self.current_states, self.current_internals<EOL><DEDENT><DEDENT>

Return action(s) for given state(s). States preprocessing and exploration are applied if
configured accordingly.

Args:
    states (any): One state (usually a value tuple) or dict of states if multiple states are expected.
    deterministic (bool): If true, no exploration and sampling is applied.
    independent (bool): If true, action is not followed by observe (and hence not included
        in updates).
    fetch_tensors (list): Optional String of named tensors to fetch
    buffered (bool): If true (default), states and internals are not returned but buffered
        with observes. Must be false for multi-threaded mode as we need atomic inserts.
Returns:
    Scalar value of the action or dict of multiple actions the agent wants to execute.
    (fetched_tensors) Optional dict() with named tensors fetched


def restore_model(self, directory=None, file=None):

self.model.restore(directory=directory, file=file)<EOL>

Restore TensorFlow model. If no checkpoint file is given, the latest checkpoint is
restored. If no checkpoint directory is given, the model's default saver directory is
used (unless file specifies the entire path).

Args:
    directory: Optional checkpoint directory.
    file: Optional checkpoint file, or path if directory not given.


<EOL>if update_mode is None:<EOL><INDENT>update_mode = dict(<EOL>unit='<STR_LIT>',<EOL>batch_size=<NUM_LIT:10><EOL>)<EOL><DEDENT>elif '<STR_LIT>' in update_mode:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>update_mode['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if memory is None:<EOL><INDENT>memory = dict(<EOL>type='<STR_LIT>',<EOL>include_next_states=False,<EOL>capacity=(<NUM_LIT:1000> * update_mode['<STR_LIT>'])<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert not memory['<STR_LIT>']<EOL><DEDENT>if optimizer is None:<EOL><INDENT>optimizer = dict(<EOL>type='<STR_LIT>',<EOL>learning_rate=<NUM_LIT><EOL>)<EOL><DEDENT>self.baseline_mode = baseline_mode<EOL>self.baseline = baseline<EOL>self.baseline_optimizer = baseline_optimizer<EOL>self.gae_lambda = gae_lambda<EOL>super(VPGAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing,<EOL>update_mode=update_mode,<EOL>memory=memory,<EOL>optimizer=optimizer,<EOL>discount=discount,<EOL>network=network,<EOL>distributions=distributions,<EOL>entropy_regularization=entropy_regularization<EOL>)<EOL>

Initializes the VPG agent.

Args:
    update_mode (spec): Update mode specification, with the following attributes:
        - unit: 'episodes' if given (default: 'episodes').
        - batch_size: integer (default: 10).
        - frequency: integer (default: batch_size).
    memory (spec): Memory specification, see core.memories module for more information
        (default: {type='latest', include_next_states=false, capacity=1000*batch_size}).
    optimizer (spec): Optimizer specification, see core.optimizers module for more
        information (default: {type='adam', learning_rate=1e-3}).
    baseline_mode (str): One of 'states', 'network' (default: none).
    baseline (spec): Baseline specification, see core.baselines module for more information
        (default: none).
    baseline_optimizer (spec): Baseline optimizer specification, see core.optimizers module
        for more information (default: none).
    gae_lambda (float): Lambda factor for generalized advantage estimation (default: none).


<EOL>if update_mode is None:<EOL><INDENT>update_mode = dict(<EOL>unit='<STR_LIT>',<EOL>batch_size=<NUM_LIT:10><EOL>)<EOL><DEDENT>elif '<STR_LIT>' in update_mode:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>update_mode['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if memory is None:<EOL><INDENT>memory = dict(<EOL>type='<STR_LIT>',<EOL>include_next_states=False,<EOL>capacity=(<NUM_LIT:1000> * update_mode['<STR_LIT>'])<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert not memory['<STR_LIT>']<EOL><DEDENT>assert (update_mode['<STR_LIT>'] != '<STR_LIT>' or memory['<STR_LIT:type>'] == '<STR_LIT>')<EOL>if step_optimizer is None:<EOL><INDENT>step_optimizer = dict(<EOL>type='<STR_LIT>',<EOL>learning_rate=<NUM_LIT><EOL>)<EOL><DEDENT>optimizer = dict(<EOL>type='<STR_LIT>',<EOL>optimizer=dict(<EOL>type='<STR_LIT>',<EOL>optimizer=step_optimizer,<EOL>fraction=subsampling_fraction<EOL>),<EOL>num_steps=optimization_steps<EOL>)<EOL>self.baseline_mode = baseline_mode<EOL>self.baseline = baseline<EOL>self.baseline_optimizer = baseline_optimizer<EOL>self.gae_lambda = gae_lambda<EOL>self.likelihood_ratio_clipping = likelihood_ratio_clipping<EOL>super(PPOAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing,<EOL>update_mode=update_mode,<EOL>memory=memory,<EOL>optimizer=optimizer,<EOL>discount=discount,<EOL>network=network,<EOL>distributions=distributions,<EOL>entropy_regularization=entropy_regularization<EOL>)<EOL>

Initializes the PPO agent.

Args:
    update_mode (spec): Update mode specification, with the following attributes:
        - unit: 'episodes' if given (default: 'episodes').
        - batch_size: integer (default: 10).
        - frequency: integer (default: batch_size).
    memory (spec): Memory specification, see core.memories module for more information
        (default: {type='latest', include_next_states=false, capacity=1000*batch_size}).
    optimizer (spec): PPO agent implicitly defines a multi-step subsampling optimizer.
    baseline_mode (str): One of 'states', 'network' (default: none).
    baseline (spec): Baseline specification, see core.baselines module for more information
        (default: none).
    baseline_optimizer (spec): Baseline optimizer specification, see core.optimizers module
        for more information (default: none).
    gae_lambda (float): Lambda factor for generalized advantage estimation (default: none).
    likelihood_ratio_clipping (float): Likelihood ratio clipping for policy gradient
        (default: 0.2).
    step_optimizer (spec): Step optimizer specification of implicit multi-step subsampling
        optimizer, see core.optimizers module for more information (default: {type='adam',
        learning_rate=1e-3}).
    subsampling_fraction (float): Subsampling fraction of implicit subsampling optimizer
        (default: 0.1).
    optimization_steps (int): Number of optimization steps for implicit multi-step
        optimizer (default: 50).


<EOL>if update_mode is None:<EOL><INDENT>update_mode = dict(<EOL>unit='<STR_LIT>',<EOL>batch_size=<NUM_LIT:10><EOL>)<EOL><DEDENT>elif '<STR_LIT>' in update_mode:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>update_mode['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if memory is None:<EOL><INDENT>memory = dict(<EOL>type='<STR_LIT>',<EOL>include_next_states=False,<EOL>capacity=(<NUM_LIT:1000> * update_mode['<STR_LIT>'])<EOL>)<EOL><DEDENT>else:<EOL><INDENT>assert not memory['<STR_LIT>']<EOL><DEDENT>assert (update_mode['<STR_LIT>'] != '<STR_LIT>' or memory['<STR_LIT:type>'] == '<STR_LIT>')<EOL>optimizer = dict(<EOL>type='<STR_LIT>',<EOL>optimizer=dict(<EOL>type='<STR_LIT>',<EOL>learning_rate=learning_rate,<EOL>cg_max_iterations=cg_max_iterations,<EOL>cg_damping=cg_damping,<EOL>cg_unroll_loop=cg_unroll_loop,<EOL>),<EOL>ls_max_iterations=ls_max_iterations,<EOL>ls_accept_ratio=ls_accept_ratio,<EOL>ls_mode='<STR_LIT>',  <EOL>ls_parameter=<NUM_LIT:0.5>,  <EOL>ls_unroll_loop=ls_unroll_loop<EOL>)<EOL>self.baseline_mode = baseline_mode<EOL>self.baseline = baseline<EOL>self.baseline_optimizer = baseline_optimizer<EOL>self.gae_lambda = gae_lambda<EOL>self.likelihood_ratio_clipping = likelihood_ratio_clipping<EOL>super(TRPOAgent, self).__init__(<EOL>states=states,<EOL>actions=actions,<EOL>batched_observe=batched_observe,<EOL>batching_capacity=batching_capacity,<EOL>scope=scope,<EOL>device=device,<EOL>saver=saver,<EOL>summarizer=summarizer,<EOL>execution=execution,<EOL>variable_noise=variable_noise,<EOL>states_preprocessing=states_preprocessing,<EOL>actions_exploration=actions_exploration,<EOL>reward_preprocessing=reward_preprocessing,<EOL>update_mode=update_mode,<EOL>memory=memory,<EOL>optimizer=optimizer,<EOL>discount=discount,<EOL>network=network,<EOL>distributions=distributions,<EOL>entropy_regularization=entropy_regularization<EOL>)<EOL>

Initializes the TRPO agent.

Args:
    update_mode (spec): Update mode specification, with the following attributes:
        - unit: 'episodes' if given (default: 'episodes').
        - batch_size: integer (default: 10).
        - frequency: integer (default: batch_size).
    memory (spec): Memory specification, see core.memories module for more information
        (default: {type='latest', include_next_states=false, capacity=1000*batch_size}).
    optimizer (spec): TRPO agent implicitly defines a optimized-step natural-gradient
        optimizer.
    baseline_mode (str): One of 'states', 'network' (default: none).
    baseline (spec): Baseline specification, see core.baselines module for more information
        (default: none).
    baseline_optimizer (spec): Baseline optimizer specification, see core.optimizers module
        for more information (default: none).
    gae_lambda (float): Lambda factor for generalized advantage estimation (default: none).
    likelihood_ratio_clipping (float): Likelihood ratio clipping for policy gradient
        (default: none).
    learning_rate (float): Learning rate of natural-gradient optimizer (default: 1e-3).
    cg_max_iterations (int): Conjugate-gradient max iterations (default: 20).
    cg_damping (float): Conjugate-gradient damping (default: 1e-3).
    cg_unroll_loop (bool): Conjugate-gradient unroll loop (default: false).
    ls_max_iterations (int): Line-search max iterations (default: 10).
    ls_accept_ratio (float): Line-search accept ratio (default: 0.9).
    ls_unroll_loop (bool): Line-search unroll loop (default: false).


self.sigma = sigma<EOL>self.mu = float(mu)  <EOL>self.theta = theta<EOL>super(OrnsteinUhlenbeckProcess, self).__init__(scope=scope, summary_labels=summary_labels)<EOL>

Initializes an Ornstein-Uhlenbeck process which is a mean reverting stochastic process
introducing time-correlated noise.


return dict()<EOL>

Returns the internal states specification.

Returns:
    Internal states specification


self.name = name<EOL>super(Output, self).__init__(named_tensors=named_tensors, scope=scope, summary_labels=summary_labels)<EOL>

Output layer.

Args:
    output: A string that names the tensor, will be added to available inputs


return None<EOL>

Creates the TensorFlow operations for the layer regularization loss.

Returns:
    Regularization loss tensor.


def __init__(self, layer, named_tensors=None, scope='<STR_LIT>', summary_labels=(), **kwargs):

self.layer_spec = layer<EOL>self.layer = util.get_object(obj=layer, predefined_objects=TFLayer.tf_layers, kwargs=kwargs)<EOL>self.first_scope = None<EOL>super(TFLayer, self).__init__(named_tensors=named_tensors, scope=scope, summary_labels=summary_labels)<EOL>

Creates a new layer instance of a TensorFlow layer.

Args:
    name: The name of the layer, one of 'dense'.
    **kwargs: Additional arguments passed on to the TensorFlow layer constructor.


def __init__(self, named_tensors=None, scope='<STR_LIT>', summary_labels=None):

self.scope = scope<EOL>self.summary_labels = set(summary_labels or ())<EOL>self.named_tensors = named_tensors<EOL>self.variables = dict()<EOL>self.all_variables = dict()<EOL>def custom_getter(getter, name, registered=False, **kwargs):<EOL><INDENT>variable = getter(name=name, registered=True, **kwargs)<EOL>if registered:<EOL><INDENT>pass<EOL><DEDENT>elif name in self.all_variables:<EOL><INDENT>assert variable is self.all_variables[name]<EOL>if kwargs.get('<STR_LIT>', True):<EOL><INDENT>assert variable is self.variables[name]<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name=name, tensor=variable)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self.all_variables[name] = variable<EOL>if kwargs.get('<STR_LIT>', True):<EOL><INDENT>self.variables[name] = variable<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name=name, tensor=variable)<EOL><DEDENT><DEDENT><DEDENT>return variable<EOL><DEDENT>self.apply = tf.make_template(<EOL>name_=(scope + '<STR_LIT>'),<EOL>func_=self.tf_apply,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.regularization_loss = tf.make_template(<EOL>name_=(scope + '<STR_LIT>'),<EOL>func_=self.tf_regularization_loss,<EOL>custom_getter_=custom_getter<EOL>)<EOL>


layer = util.get_object(<EOL>obj=spec,<EOL>predefined_objects=tensorforce.core.networks.layers,<EOL>kwargs=kwargs<EOL>)<EOL>assert isinstance(layer, Layer)<EOL>return layer<EOL>

Creates a layer from a specification dict.


def __init__(self, size, dropout=None, named_tensors=None, scope='<STR_LIT>', summary_labels=(), return_final_state=True):

self.size = size<EOL>self.dropout = dropout<EOL>self.return_final_state = return_final_state<EOL>super(Lstm, self).__init__(named_tensors=named_tensors, scope=scope, summary_labels=summary_labels)<EOL>

LSTM layer.

Args:
    size: LSTM size.
    dropout: Dropout rate.


<EOL>self.expectation_layer = Linear(<EOL>size=<NUM_LIT:1>, bias=bias,<EOL>l2_regularization=l2_regularization,<EOL>l1_regularization=l1_regularization,<EOL>summary_labels=summary_labels,<EOL>)<EOL>self.advantage_layer = Linear(<EOL>size=size,<EOL>bias=bias,<EOL>l2_regularization=l2_regularization,<EOL>l1_regularization=l1_regularization,<EOL>summary_labels=summary_labels,<EOL>)<EOL>self.output = output<EOL>self.nonlinearity = Nonlinearity(summary_labels=summary_labels, **util.prepare_kwargs(activation))<EOL>super(Dueling, self).__init__(named_tensors=named_tensors, scope=scope, summary_labels=summary_labels)<EOL>

Dueling layer.

[Dueling Networks] (https://arxiv.org/pdf/1511.06581.pdf)
Implement Y = Expectation[x] + (Advantage[x] - Mean(Advantage[x]))

Args:
    size: Layer size.
    bias: If true, bias is added.
    activation: Type of nonlinearity, or dict with name & arguments
    l2_regularization: L2 regularization weight.
    l1_regularization: L1 regularization weight.
    output: None or tuple of output names for ('expectation','advantage','mean_advantage')


self.indices = indices<EOL>self.size = size<EOL>self.l2_regularization = l2_regularization<EOL>self.l1_regularization = l1_regularization<EOL>super(Embedding, self).__init__(named_tensors=named_tensors, scope=scope, summary_labels=summary_labels)<EOL>

Embedding layer.

Args:
    indices: Number of embedding indices.
    size: Embedding size.
    l2_regularization: L2 regularization weight.
    l1_regularization: L1 regularization weight.


self.name = name<EOL>self.alpha = None<EOL>self.max = None<EOL>self.min = None<EOL>self.beta_learn = False<EOL>super(Nonlinearity, self).__init__(named_tensors=named_tensors, scope=scope, summary_labels=summary_labels)<EOL>if max is not None:<EOL><INDENT>self.max = float(max)<EOL><DEDENT>if min is not None:<EOL><INDENT>self.min = float(min)<EOL><DEDENT>if alpha is not None:<EOL><INDENT>self.alpha = float(alpha)<EOL><DEDENT>if beta == '<STR_LIT>':<EOL><INDENT>self.beta_learn = True<EOL>self.beta = None<EOL><DEDENT>else:<EOL><INDENT>self.beta = tf.constant(float(beta), dtype=util.tf_dtype('<STR_LIT:float>'))<EOL><DEDENT>

Non-linearity activation layer.

Args:
    name: Non-linearity name, one of 'elu', 'relu', 'selu', 'sigmoid', 'swish',
        'leaky_relu' (or 'lrelu'), 'crelu', 'softmax', 'softplus', 'softsign', 'tanh' or 'none'.
    alpha: (float|int) Alpha value for leaky Relu
    beta: (float|int|'learn') Beta value or 'learn' to train value (default 1.0)
    max: (float|int) maximum (beta * input) value passed to non-linearity function
    min: (float|int) minimum (beta * input) value passed to non-linearity function
    summary_labels: Requested summary labels for tensorboard export, add 'beta' to watch beta learning


network = util.get_object(<EOL>obj=spec,<EOL>default_object=LayeredNetwork,<EOL>kwargs=kwargs<EOL>)<EOL>assert isinstance(network, Network)<EOL>return network<EOL>

Creates a network from a specification dict.


if include_nontrainable:<EOL><INDENT>return [self.all_variables[key] for key in sorted(self.all_variables)]<EOL><DEDENT>else:<EOL><INDENT>return [self.variables[key] for key in sorted(self.variables)]<EOL><DEDENT>

Returns the TensorFlow variables used by the network.

Returns:
    List of variables


return dict()<EOL>


def tf_apply(self, x, internals, update, return_internals=False):

raise NotImplementedError<EOL>

Creates the TensorFlow operations for applying the network to the given input.

Args:
    x: Network input tensor or dict of input tensors.
    internals: List of prior internal state tensors
    update: Boolean tensor indicating whether this call happens during an update.
    return_internals: If true, also returns posterior internal state tensors

Returns:
    Network output tensor, plus optionally list of posterior internal state tensors


fetches = []<EOL>for processor in self.preprocessors:<EOL><INDENT>fetches.extend(processor.reset() or [])<EOL><DEDENT>return fetches<EOL>

Calls `reset` on all our Preprocessor objects.

Returns:
    A list of tensors to be fetched.


if isinstance(spec, dict):<EOL><INDENT>spec = [spec]<EOL><DEDENT>stack = PreprocessorStack()<EOL>for preprocessor_spec in spec:<EOL><INDENT>preprocessor_kwargs = copy.deepcopy(kwargs)<EOL>preprocessor = util.get_object(<EOL>obj=preprocessor_spec,<EOL>predefined_objects=tensorforce.core.preprocessors.preprocessors,<EOL>kwargs=preprocessor_kwargs<EOL>)<EOL>assert isinstance(preprocessor, Preprocessor)<EOL>stack.preprocessors.append(preprocessor)<EOL><DEDENT>return stack<EOL>

Creates a preprocessing stack from a specification dict.


return shape<EOL>

Shape of preprocessed state given original shape.

Args:
    shape (tuple): The original (unprocessed) shape.

Returns: The processed tensor shape.


return [self.variables[key] for key in sorted(self.variables)]<EOL>

Returns the TensorFlow variables used by the preprocessor.

Returns:
    List of variables.


def __init__(self, conv_sizes, dense_sizes, scope='<STR_LIT>', summary_labels=()):

network = []<EOL>for size in conv_sizes:<EOL><INDENT>network.append(dict(type='<STR_LIT>', size=size))<EOL><DEDENT>network[<NUM_LIT:0>]['<STR_LIT>'] = <NUM_LIT:5><EOL>network.append(dict(type='<STR_LIT>'))  <EOL>for size in dense_sizes:<EOL><INDENT>network.append(dict(type='<STR_LIT>', size=size))<EOL><DEDENT>super(CNNBaseline, self).__init__(network=network, scope=scope, summary_labels=summary_labels)<EOL>

CNN baseline.

Args:
    conv_sizes: List of convolutional layer sizes
    dense_sizes: List of dense layer sizes


baseline = util.get_object(<EOL>obj=spec,<EOL>predefined_objects=tensorforce.core.baselines.baselines,<EOL>kwargs=kwargs<EOL>)<EOL>assert isinstance(baseline, Baseline)<EOL>return baseline<EOL>

Creates a baseline from a specification dict.


def tf_reference(self, states, internals, reward, update):

return None<EOL>

Creates the TensorFlow operations for obtaining the reference tensor(s), in case of a
comparative loss.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    reward: Reward tensor.
    update: Boolean tensor indicating whether this call happens during an update.

Returns:
    Reference tensor(s).


def __init__(self, sizes, scope='<STR_LIT>', summary_labels=()):

network = []<EOL>for size in sizes:<EOL><INDENT>network.append(dict(type='<STR_LIT>', size=size))<EOL><DEDENT>super(MLPBaseline, self).__init__(network=network, scope=scope, summary_labels=summary_labels)<EOL>

Multi-layer perceptron baseline.

Args:
    sizes: List of dense layer sizes


def __init__(self, shape, min_value, max_value, alpha=<NUM_LIT:0.0>, beta=<NUM_LIT:0.0>, scope='<STR_LIT>', summary_labels=()):

assert min_value is None or max_value > min_value<EOL>self.shape = shape<EOL>self.min_value = min_value<EOL>self.max_value = max_value<EOL>action_size = util.prod(self.shape)<EOL>self.alpha = Linear(size=action_size, bias=alpha, scope='<STR_LIT>', summary_labels=summary_labels)<EOL>self.beta = Linear(size=action_size, bias=beta, scope='<STR_LIT>', summary_labels=summary_labels)<EOL>super(Beta, self).__init__(shape=shape, scope=scope, summary_labels=summary_labels)<EOL>

Beta distribution.

Args:
    shape: Action shape.
    min_value: Minimum value of continuous actions.
    max_value: Maximum value of continuous actions.
    alpha: Optional distribution bias for the alpha value.
    beta: Optional distribution bias for the beta value.


def tf_kl_divergence(self, distr_params1, distr_params2):

raise NotImplementedError<EOL>

Creates the tensorFlow operations for calculating the KL divergence between two
distributions.

Args:
    distr_params1: tuple of parameter tensors for first distribution.
    distr_params2: tuple of parameter tensors for second distribution.

Returns:
    KL divergence tensor.


raise NotImplementedError<EOL>

Creates the tensorFlow operations for calculating the entropy of a distribution.

Args:
    distr_params: tuple of distribution parameter tensors.

Returns:
    Entropy tensor.


return None<EOL>

Creates the tensorFlow operations for the distribution regularization loss.

Returns:
    Regularization loss tensor.


def __init__(self, shape, scope='<STR_LIT>', summary_labels=None):

self.shape = shape<EOL>self.scope = scope<EOL>self.summary_labels = set(summary_labels or ())<EOL>self.variables = dict()<EOL>self.all_variables = dict()<EOL>def custom_getter(getter, name, registered=False, **kwargs):<EOL><INDENT>variable = getter(name=name, registered=True, **kwargs)<EOL>if registered:<EOL><INDENT>pass<EOL><DEDENT>elif name in self.all_variables:<EOL><INDENT>assert variable is self.all_variables[name]<EOL>if kwargs.get('<STR_LIT>', True):<EOL><INDENT>assert variable is self.variables[name]<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name=name, tensor=variable)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self.all_variables[name] = variable<EOL>if kwargs.get('<STR_LIT>', True):<EOL><INDENT>self.variables[name] = variable<EOL>if '<STR_LIT>' in self.summary_labels:<EOL><INDENT>tf.contrib.summary.histogram(name=name, tensor=variable)<EOL><DEDENT><DEDENT><DEDENT>return variable<EOL><DEDENT>self.parameterize = tf.make_template(<EOL>name_=(scope + '<STR_LIT>'),<EOL>func_=self.tf_parameterize,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.sample = tf.make_template(<EOL>name_=(scope + '<STR_LIT>'),<EOL>func_=self.tf_sample,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.log_probability = tf.make_template(<EOL>name_=(scope + '<STR_LIT>'),<EOL>func_=self.tf_log_probability,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.entropy = tf.make_template(<EOL>name_=(scope + '<STR_LIT>'),<EOL>func_=self.tf_entropy,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.kl_divergence = tf.make_template(<EOL>name_=(scope + '<STR_LIT>'),<EOL>func_=self.tf_kl_divergence,<EOL>custom_getter_=custom_getter<EOL>)<EOL>self.regularization_loss = tf.make_template(<EOL>name_=(scope + '<STR_LIT>'),<EOL>func_=self.tf_regularization_loss,<EOL>custom_getter_=custom_getter<EOL>)<EOL>

Distribution.

Args:
    shape: Action shape.


raise NotImplementedError<EOL>

Creates the tensorFlow operations for parameterizing a distribution conditioned on the
given input.

Args:
    x: Input tensor which the distribution is conditioned on.

Returns:
    tuple of distribution parameter tensors.


def tf_sample(self, distr_params, deterministic):

raise NotImplementedError<EOL>

Creates the tensorFlow operations for sampling an action based on a distribution.

Args:
    distr_params: tuple of distribution parameter tensors.
    deterministic: Boolean input tensor indicating whether the maximum likelihood action
        should be returned.

Returns:
    Sampled action tensor.


Returns the TensorFlow variables used by the optimizer.

Returns:
    List of variables.


def apply_step(self, variables, deltas):

if len(variables) != len(deltas):<EOL><INDENT>raise TensorForceError("<STR_LIT>")<EOL><DEDENT>return tf.group(<EOL>*(tf.assign_add(ref=variable, value=delta) for variable, delta in zip(variables, deltas))<EOL>)<EOL>

Applies the given (and already calculated) step deltas to the variable values.

Args:
    variables: List of variables.
    deltas: List of deltas of same length.

Returns:
    The step-applied operation. A tf.group of tf.assign_add ops.


def tf_step(self, time, variables, **kwargs):

arguments = kwargs["<STR_LIT>"]<EOL>fn_loss = kwargs["<STR_LIT>"]<EOL>loss = fn_loss(**arguments)<EOL>with tf.control_dependencies(control_inputs=(loss,)):<EOL><INDENT>previous_variables = [variable + <NUM_LIT:0.0> for variable in variables]<EOL><DEDENT>with tf.control_dependencies(control_inputs=previous_variables):<EOL><INDENT>applied = self.tf_optimizer.minimize(loss=loss, var_list=variables)  <EOL><DEDENT>with tf.control_dependencies(control_inputs=(applied,)):<EOL><INDENT>return [<EOL>variable - previous_variable<EOL>for variable, previous_variable in zip(variables, previous_variables)<EOL>]<EOL><DEDENT>

Keyword Args:
    arguments: Dict of arguments for passing to fn_loss as **kwargs.
    fn_loss: A callable taking arguments as kwargs and returning the loss op of the current model.


def __init__(self, optimizer, scope=None, summary_labels=(), **kwargs):

self.tf_optimizer_type = optimizer<EOL>self.tf_optimizer = TFOptimizer.tf_optimizers[optimizer](**kwargs)<EOL>super(TFOptimizer, self).__init__(scope=(scope or optimizer), summary_labels=summary_labels)<EOL>

Creates a new optimizer instance of a TensorFlow optimizer.

Args:
    optimizer: The name of the optimizer. Must be one of the keys of the tf_optimizers dict.
    **kwargs: Arguments passed on to the TensorFlow optimizer constructor as **kwargs.


def __init__(self, optimizer, fraction=<NUM_LIT:0.1>, scope='<STR_LIT>', summary_labels=()):

assert isinstance(fraction, float) and fraction > <NUM_LIT:0.0><EOL>self.fraction = fraction<EOL>super(SubsamplingStep, self).__init__(optimizer=optimizer, scope=scope, summary_labels=summary_labels)<EOL>

Creates a new subsampling-step meta optimizer instance.

Args:
    optimizer: The optimizer which is modified by this meta optimizer.
    fraction: The fraction of instances of the batch to subsample.


def __init__(self, learning_rate, num_samples=<NUM_LIT:1>, unroll_loop=False, scope='<STR_LIT>', summary_labels=()):

assert isinstance(learning_rate, float) and learning_rate > <NUM_LIT:0.0><EOL>self.learning_rate = learning_rate<EOL>assert isinstance(num_samples, int) and num_samples > <NUM_LIT:0><EOL>self.num_samples = num_samples<EOL>assert isinstance(unroll_loop, bool)<EOL>self.unroll_loop = unroll_loop<EOL>super(Evolutionary, self).__init__(scope=scope, summary_labels=summary_labels)<EOL>

Creates a new evolutionary optimizer instance.

Args:
    learning_rate: Learning rate.
    num_samples: Number of sampled perturbations.


def __init__(self, sync_frequency=<NUM_LIT:1>, update_weight=<NUM_LIT:1.0>, scope='<STR_LIT>', summary_labels=()):

assert isinstance(sync_frequency, int) and sync_frequency > <NUM_LIT:0><EOL>self.sync_frequency = sync_frequency<EOL>assert isinstance(update_weight, float) and update_weight > <NUM_LIT:0.0><EOL>self.update_weight = update_weight<EOL>super(Synchronization, self).__init__(scope=scope, summary_labels=summary_labels)<EOL>

Creates a new synchronization optimizer instance.

Args:
    sync_frequency: The interval between optimization calls actually performing a
    synchronization step.
    update_weight: The update weight, 1.0 meaning a full assignment of the source
    variables values.


deltas = self.optimizer.step(time=time, variables=variables, **kwargs)<EOL>with tf.control_dependencies(control_inputs=deltas):<EOL><INDENT>clipped_deltas = list()<EOL>exceeding_deltas = list()<EOL>for delta in deltas:<EOL><INDENT>clipped_delta = tf.clip_by_value(<EOL>t=delta,<EOL>clip_value_min=-self.clipping_value,<EOL>clip_value_max=self.clipping_value<EOL>)<EOL>clipped_deltas.append(clipped_delta)<EOL>exceeding_deltas.append(clipped_delta - delta)<EOL><DEDENT><DEDENT>applied = self.apply_step(variables=variables, deltas=exceeding_deltas)<EOL>with tf.control_dependencies(control_inputs=(applied,)):<EOL><INDENT>return [delta + <NUM_LIT:0.0> for delta in clipped_deltas]<EOL><DEDENT>

Creates the TensorFlow operations for performing an optimization step.

Args:
    time: Time tensor.
    variables: List of variables to optimize.
    **kwargs: Additional arguments passed on to the internal optimizer.

Returns:
    List of delta tensors corresponding to the updates for each optimized variable.


def __init__(self, optimizer, clipping_value, scope='<STR_LIT>', summary_labels=()):

assert isinstance(clipping_value, float) and clipping_value > <NUM_LIT:0.0><EOL>self.clipping_value = clipping_value<EOL>super(ClippedStep, self).__init__(optimizer=optimizer, scope=scope, summary_labels=summary_labels)<EOL>

Creates a new multi-step meta optimizer instance.

Args:
    optimizer: The optimizer which is modified by this meta optimizer.
    clipping_value: Clip deltas at this value.


global_variables = kwargs["<STR_LIT>"]<EOL>assert all(<EOL>util.shape(global_variable) == util.shape(local_variable)<EOL>for global_variable, local_variable in zip(global_variables, variables)<EOL>)<EOL>local_deltas = self.optimizer.step(time=time, variables=variables, **kwargs)<EOL>with tf.control_dependencies(control_inputs=local_deltas):<EOL><INDENT>applied = self.optimizer.apply_step(variables=global_variables, deltas=local_deltas)<EOL><DEDENT>with tf.control_dependencies(control_inputs=(applied,)):<EOL><INDENT>update_deltas = list()<EOL>for global_variable, local_variable in zip(global_variables, variables):<EOL><INDENT>delta = global_variable - local_variable<EOL>update_deltas.append(delta)<EOL><DEDENT>applied = self.apply_step(variables=variables, deltas=update_deltas)<EOL><DEDENT>with tf.control_dependencies(control_inputs=(applied,)):<EOL><INDENT>return [local_delta + update_delta for local_delta, update_delta in zip(local_deltas, update_deltas)]<EOL><DEDENT>

Keyword Args:
    global_variables: List of global variables to apply the proposed optimization step to.

Returns:
    List of delta tensors corresponding to the updates for each optimized variable.


self.solver = LineSearch(<EOL>max_iterations=ls_max_iterations,<EOL>accept_ratio=ls_accept_ratio,<EOL>mode=ls_mode,<EOL>parameter=ls_parameter,<EOL>unroll_loop=ls_unroll_loop<EOL>)<EOL>super(OptimizedStep, self).__init__(optimizer=optimizer, scope=scope, summary_labels=summary_labels)<EOL>

Creates a new optimized step meta optimizer instance.

Args:
    optimizer: The optimizer which is modified by this meta optimizer.
    ls_max_iterations: Maximum number of line search iterations.
    ls_accept_ratio: Line search acceptance ratio.
    ls_mode: Line search mode, see LineSearch solver.
    ls_parameter: Line search parameter, see LineSearch solver.
    ls_unroll_loop: Unroll line search loop if true.


def minimize(self, time, variables, **kwargs):

loss = kwargs["<STR_LIT>"]<EOL>sampled_loss = kwargs["<STR_LIT>"]<EOL>min_op, _ = self.minimize_(loss, sampled_loss, var_list=variables)<EOL>return min_op<EOL>

Performs an optimization step.

Args:
    time: Time tensor. Not used for this
    variables: List of variables to optimize.
    **kwargs:
        fn_loss : loss function tensor that is differentiated
        sampled_loss : the sampled loss from running the model.

Returns:
    The optimization operation.


self.max_grad_norm = max_grad_norm<EOL>self._lr = learning_rate<EOL>self._momentum = momentum<EOL>self._clip_kl = clip_kl<EOL>self._channel_fac = channel_fac<EOL>self._kfac_update = kfac_update<EOL>self._async = async_<EOL>self._async_stats = async_stats<EOL>self._epsilon = epsilon<EOL>self._stats_decay = stats_decay<EOL>self._blockdiag_bias = blockdiag_bias<EOL>self._approxT2 = approxT2<EOL>self._use_float64 = use_float64<EOL>self._factored_damping = factored_damping<EOL>self._cold_iter = cold_iter<EOL>if cold_lr == None:<EOL><INDENT>self._cold_lr = self._lr<EOL><DEDENT>else:<EOL><INDENT>self._cold_lr = cold_lr<EOL><DEDENT>self._stats_accum_iter = stats_accum_iter<EOL>self._weight_decay_dict = weight_decay_dict<EOL>self._diag_init_coeff = <NUM_LIT:0.><EOL>self._full_stats_init = full_stats_init<EOL>if not self._full_stats_init:<EOL><INDENT>self._stats_accum_iter = self._cold_iter<EOL><DEDENT>self.sgd_step = tf.Variable(<NUM_LIT:0>, name='<STR_LIT>', trainable=False)<EOL>self.global_step = tf.Variable(<EOL><NUM_LIT:0>, name='<STR_LIT>', trainable=False)<EOL>self.cold_step = tf.Variable(<NUM_LIT:0>, name='<STR_LIT>', trainable=False)<EOL>self.factor_step = tf.Variable(<EOL><NUM_LIT:0>, name='<STR_LIT>', trainable=False)<EOL>self.stats_step = tf.Variable(<EOL><NUM_LIT:0>, name='<STR_LIT>', trainable=False)<EOL>self.vFv = tf.Variable(<NUM_LIT:0.>, name='<STR_LIT>', trainable=False)<EOL>self.factors = {}<EOL>self.param_vars = []<EOL>self.stats = {}<EOL>self.stats_eigen = {}<EOL>super(KFAC, self).__init__(scope=scope, summary_labels=summary_labels)<EOL>

Initializes a KFAC optimizer.

For more information on arguments, see the Kfac Optimization paper https://arxiv.org/pdf/1503.05671.pdf


fn_loss = kwargs["<STR_LIT>"]<EOL>if variables is None:<EOL><INDENT>variables = tf.trainable_variables<EOL><DEDENT>return tf.gradients(fn_loss, variables)<EOL>

Creates the TensorFlow operations for performing an optimization step on the given variables, including
actually changing the values of the variables.

Args:
    time: Time tensor. Not used for this optimizer.
    variables: List of variables to optimize.
    **kwargs:
        fn_loss : loss function tensor to differentiate.

Returns:
    List of delta tensors corresponding to the updates for each optimized variable.


def tf_solve(self, fn_x, x_init, base_value, target_value, estimated_improvement=None):

return super(LineSearch, self).tf_solve(fn_x, x_init, base_value, target_value, estimated_improvement)<EOL>

Iteratively optimizes $f(x)$ for $x$ on the line between $x'$ and $x_0$.

Args:
    fn_x: A callable returning the value $f(x)$ at $x$.
    x_init: Initial solution guess $x_0$.
    base_value: Value $f(x')$ at $x = x'$.
    target_value: Value $f(x_0)$ at $x = x_0$.
    estimated_improvement: Estimated improvement for $x = x_0$, $f(x')$ if None.

Returns:
    A solution $x$ to the problem as given by the solver.


def tf_step(self, x, iteration, deltas, improvement, last_improvement, estimated_improvement):

x, next_iteration, deltas, improvement, last_improvement, estimated_improvement = super(LineSearch, self).tf_step(<EOL>x, iteration, deltas, improvement, last_improvement, estimated_improvement<EOL>)<EOL>next_x = [t + delta for t, delta in zip(x, deltas)]<EOL>if self.mode == '<STR_LIT>':<EOL><INDENT>next_deltas = deltas<EOL>next_estimated_improvement = estimated_improvement + self.estimated_incr<EOL><DEDENT>elif self.mode == '<STR_LIT>':<EOL><INDENT>next_deltas = [delta * self.parameter for delta in deltas]<EOL>next_estimated_improvement = estimated_improvement * self.parameter<EOL><DEDENT>target_value = self.fn_x(next_deltas)<EOL>next_improvement = tf.divide(<EOL>x=(target_value - self.base_value),<EOL>y=tf.maximum(x=next_estimated_improvement, y=util.epsilon)<EOL>)<EOL>return next_x, next_iteration, next_deltas, next_improvement, improvement, next_estimated_improvement<EOL>

Iteration loop body of the line search algorithm.

Args:
    x: Current solution estimate $x_t$.
    iteration: Current iteration counter $t$.
    deltas: Current difference $x_t - x'$.
    improvement: Current improvement $(f(x_t) - f(x')) / v'$.
    last_improvement: Last improvement $(f(x_{t-1}) - f(x')) / v'$.
    estimated_improvement: Current estimated value $v'$.

Returns:
    Updated arguments for next iteration.


def tf_next_step(self, x, iteration, deltas, improvement, last_improvement, estimated_improvement):

next_step = super(LineSearch, self).tf_next_step(<EOL>x, iteration, deltas, improvement, last_improvement, estimated_improvement<EOL>)<EOL>def undo_deltas():<EOL><INDENT>value = self.fn_x([-delta for delta in deltas])<EOL>with tf.control_dependencies(control_inputs=(value,)):<EOL><INDENT>return tf.less(x=value, y=value)  <EOL><DEDENT><DEDENT>improved = tf.cond(<EOL>pred=(improvement > last_improvement),<EOL>true_fn=(lambda: True),<EOL>false_fn=undo_deltas<EOL>)<EOL>next_step = tf.logical_and(x=next_step, y=improved)<EOL>next_step = tf.logical_and(x=next_step, y=(improvement < self.accept_ratio))<EOL>return tf.logical_and(x=next_step, y=(estimated_improvement > util.epsilon))<EOL>

Termination condition: max number of iterations, or no improvement for last step, or
improvement less than acceptable ratio, or estimated value not positive.

Args:
    x: Current solution estimate $x_t$.
    iteration: Current iteration counter $t$.
    deltas: Current difference $x_t - x'$.
    improvement: Current improvement $(f(x_t) - f(x')) / v'$.
    last_improvement: Last improvement $(f(x_{t-1}) - f(x')) / v'$.
    estimated_improvement: Current estimated value $v'$.

Returns:
    True if another iteration should be performed.


def __init__(self, max_iterations, unroll_loop=False):

assert max_iterations >= <NUM_LIT:0><EOL>self.max_iterations = max_iterations<EOL>assert isinstance(unroll_loop, bool)<EOL>self.unroll_loop = unroll_loop<EOL>super(Iterative, self).__init__()<EOL>self.initialize = tf.make_template(name_='<STR_LIT>', func_=self.tf_initialize)<EOL>self.step = tf.make_template(name_='<STR_LIT>', func_=self.tf_step)<EOL>self.next_step = tf.make_template(name_='<STR_LIT>', func_=self.tf_next_step)<EOL>

Creates a new iterative solver instance.

Args:
    max_iterations: Maximum number of iterations before termination.
    unroll_loop: Unrolls the TensorFlow while loop if true.


def tf_solve(self, fn_x, x_init, *args):

self.fn_x = fn_x<EOL>args = self.initialize(x_init, *args)<EOL>if self.unroll_loop:<EOL><INDENT>for _ in range(self.max_iterations):<EOL><INDENT>next_step = self.next_step(*args)<EOL>step = (lambda: self.step(*args))<EOL>do_nothing = (lambda: args)<EOL>args = tf.cond(pred=next_step, true_fn=step, false_fn=do_nothing)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>args = tf.while_loop(cond=self.next_step, body=self.step, loop_vars=args)<EOL><DEDENT>return args[<NUM_LIT:0>]<EOL>

Iteratively solves an equation/optimization for $x$ involving an expression $f(x)$.

Args:
    fn_x: A callable returning an expression $f(x)$ given $x$.
    x_init: Initial solution guess $x_0$.
    *args: Additional solver-specific arguments.

Returns:
    A solution $x$ to the problem as given by the solver.


raise NotImplementedError<EOL>

Solves an equation/optimization for $x$ involving an expression $f(x)$.

Args:
    fn_x: A callable returning an expression $f(x)$ given $x$.
    *args: Additional solver-specific arguments.

Returns:
    A solution $x$ to the problem as given by the solver.


<EOL>self.solve = tf.make_template(name_='<STR_LIT>', func_=self.tf_solve)<EOL>


if x_init is None:<EOL><INDENT>x_init = [tf.zeros(shape=util.shape(t)) for t in b]<EOL><DEDENT>initial_args = super(ConjugateGradient, self).tf_initialize(x_init)<EOL>conjugate = residual = [t - fx for t, fx in zip(b, self.fn_x(x_init))]<EOL>squared_residual = tf.add_n(inputs=[tf.reduce_sum(input_tensor=(res * res)) for res in residual])<EOL>return initial_args + (conjugate, residual, squared_residual)<EOL>

Initialization step preparing the arguments for the first iteration of the loop body:
$x_0, 0, p_0, r_0, r_0^2$.

Args:
    x_init: Initial solution guess $x_0$, zero vector if None.
    b: The right-hand side $b$ of the system of linear equations.

Returns:
    Initial arguments for tf_step.


def tf_next_step(self, x, iteration, conjugate, residual, squared_residual):

next_step = super(ConjugateGradient, self).tf_next_step(x, iteration, conjugate, residual, squared_residual)<EOL>return tf.logical_and(x=next_step, y=(squared_residual >= util.epsilon))<EOL>

Termination condition: max number of iterations, or residual sufficiently small.

Args:
    x: Current solution estimate $x_t$.
    iteration: Current iteration counter $t$.
    conjugate: Current conjugate $c_t$.
    residual: Current residual $r_t$.
    squared_residual: Current squared residual $r_t^2$.

Returns:
    True if another iteration should be performed.


if self.batch_indices is None:<EOL><INDENT>raise TensorForceError("<STR_LIT>")<EOL><DEDENT>for index, loss in zip(self.batch_indices, loss_per_instance):<EOL><INDENT>new_priority = (np.abs(loss) + self.prioritization_constant) ** self.prioritization_weight<EOL>self.observations._move(index, new_priority)<EOL>self.none_priority_index += <NUM_LIT:1><EOL><DEDENT>

Computes priorities according to loss.

Args:
    loss_per_instance:


item, old_priority = self._memory[index]<EOL>old_priority = old_priority or <NUM_LIT:0><EOL>self._memory[index] = _SumRow(item, new_priority)<EOL>self._update_internal_nodes(index, new_priority - old_priority)<EOL>


start = self._capacity - <NUM_LIT:1><EOL>position = start + self._position<EOL>self._position = (self._position + <NUM_LIT:1>) % self._capacity<EOL>return position<EOL>


def move(self, external_index, new_priority):

index = external_index + (self._capacity - <NUM_LIT:1>)<EOL>return self._move(index, new_priority)<EOL>


return len(self._memory) - (self._capacity - <NUM_LIT:1>)<EOL>

Return the current number of transitions.


if not self._isfull():<EOL><INDENT>self._memory.append(None)<EOL><DEDENT>position = self._next_position_then_increment()<EOL>old_priority = <NUM_LIT:0> if self._memory[position] is Noneelse (self._memory[position].priority or <NUM_LIT:0>)<EOL>row = _SumRow(item, priority)<EOL>self._memory[position] = row<EOL>self._update_internal_nodes(<EOL>position, (row.priority or <NUM_LIT:0>) - old_priority)<EOL>

Stores a transition in replay memory.

If the memory is full, the oldest entry is replaced.


def __init__(self, states, internals, actions, include_next_states, capacity, scope='<STR_LIT>', summary_labels=None):

Replay memory.

Args:
    states (dict): States specification.
    internals (dict): Internal states specification.
    actions (dict): Actions specification.
    include_next_states (bool): Include subsequent state if true.
    capacity (int): Memory capacity (number of state/internals/action/(next-state)? records).


return [self.variables[key] for key in sorted(self.variables)]<EOL>

Returns the TensorFlow variables used by the memory.

Returns:
    List of variables.


def tf_store(self, states, internals, actions, terminal, reward):

raise NotImplementedError<EOL>

Stores experiences, i.e. a batch of timesteps.

Args:
    states: Dict of state tensors.
    internals: List of prior internal state tensors.
    actions: Dict of action tensors.
    terminal: Terminal boolean tensor.
    reward: Reward tensor.


def __init__(self, states, internals, actions, include_next_states, scope='<STR_LIT>', summary_labels=None):

self.states_spec = states<EOL>self.internals_spec = internals<EOL>self.actions_spec = actions<EOL>self.include_next_states = include_next_states<EOL>self.scope = scope<EOL>self.summary_labels = set(summary_labels or ())<EOL>self.variables = dict()<EOL>self.initialize = None  <EOL>self.store = None<EOL>self.retrieve_timesteps = None<EOL>self.retrieve_episodes = None<EOL>self.retrieve_sequences = None<EOL>self.update_batch = None<EOL>self.setup_template_funcs()<EOL>

Args:
    states (dict): States specification.
    internals (dict): Internal states specification.
    actions (dict): Actions specification.
    include_next_states (bool): Include subsequent state if true.
    scope (str): The tf variable scope to use when creating variables for this memory.
    summary_labels (list): List of summary labels.


memory = util.get_object(<EOL>obj=spec,<EOL>predefined_objects=tensorforce.core.memories.memories,<EOL>kwargs=kwargs<EOL>)<EOL>assert isinstance(memory, Memory)<EOL>return memory<EOL>

Creates a memory from a specification dict.


def tf_retrieve_indices(self, buffer_elements, priority_indices):

states = dict()<EOL>buffer_start = self.buffer_index - buffer_elements<EOL>buffer_end = self.buffer_index<EOL>for name in sorted(self.states_memory):<EOL><INDENT>buffer_state_memory = self.states_buffer[name]<EOL>buffer_states = buffer_state_memory[buffer_start:buffer_end]<EOL>memory_states = tf.gather(params=self.states_memory[name], indices=priority_indices)<EOL>states[name] = tf.concat(values=(buffer_states, memory_states), axis=<NUM_LIT:0>)<EOL><DEDENT>internals = dict()<EOL>for name in sorted(self.internals_memory):<EOL><INDENT>internal_buffer_memory = self.internals_buffer[name]<EOL>buffer_internals = internal_buffer_memory[buffer_start:buffer_end]<EOL>memory_internals = tf.gather(params=self.internals_memory[name], indices=priority_indices)<EOL>internals[name] = tf.concat(values=(buffer_internals, memory_internals), axis=<NUM_LIT:0>)<EOL><DEDENT>actions = dict()<EOL>for name in sorted(self.actions_memory):<EOL><INDENT>action_buffer_memory = self.actions_buffer[name]<EOL>buffer_action = action_buffer_memory[buffer_start:buffer_end]<EOL>memory_action = tf.gather(params=self.actions_memory[name], indices=priority_indices)<EOL>actions[name] = tf.concat(values=(buffer_action, memory_action), axis=<NUM_LIT:0>)<EOL><DEDENT>buffer_terminal = self.terminal_buffer[buffer_start:buffer_end]<EOL>priority_terminal = tf.gather(params=self.terminal_memory, indices=priority_indices)<EOL>terminal = tf.concat(values=(buffer_terminal, priority_terminal), axis=<NUM_LIT:0>)<EOL>buffer_reward = self.reward_buffer[buffer_start:buffer_end]<EOL>priority_reward = tf.gather(params=self.reward_memory, indices=priority_indices)<EOL>reward = tf.concat(values=(buffer_reward, priority_reward), axis=<NUM_LIT:0>)<EOL>if self.include_next_states:<EOL><INDENT>assert util.rank(priority_indices) == <NUM_LIT:1><EOL>next_priority_indices = (priority_indices + <NUM_LIT:1>) % self.capacity<EOL>next_buffer_start = (buffer_start + <NUM_LIT:1>) % self.buffer_size<EOL>next_buffer_end = (buffer_end + <NUM_LIT:1>) % self.buffer_size<EOL>next_states = dict()<EOL>for name in sorted(self.states_memory):<EOL><INDENT>buffer_state_memory = self.states_buffer[name]<EOL>buffer_next_states = buffer_state_memory[next_buffer_start:next_buffer_end]<EOL>memory_next_states = tf.gather(params=self.states_memory[name], indices=next_priority_indices)<EOL>next_states[name] = tf.concat(values=(buffer_next_states, memory_next_states), axis=<NUM_LIT:0>)<EOL><DEDENT>next_internals = dict()<EOL>for name in sorted(self.internals_memory):<EOL><INDENT>buffer_internal_memory = self.internals_buffer[name]<EOL>buffer_next_internals = buffer_internal_memory[next_buffer_start:next_buffer_end]<EOL>memory_next_internals = tf.gather(params=self.internals_memory[name], indices=next_priority_indices)<EOL>next_internals[name] = tf.concat(values=(buffer_next_internals, memory_next_internals), axis=<NUM_LIT:0>)<EOL><DEDENT>return dict(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward,<EOL>next_states=next_states,<EOL>next_internals=next_internals<EOL>)<EOL><DEDENT>else:<EOL><INDENT>return dict(<EOL>states=states,<EOL>internals=internals,<EOL>actions=actions,<EOL>terminal=terminal,<EOL>reward=reward<EOL>)<EOL><DEDENT>

Fetches experiences for given indices by combining entries from buffer
which have no priorities, and entries from priority memory.

Args:
    buffer_elements: Number of buffer elements to retrieve
    priority_indices: Index tensor for priority memory

Returns: Batch of experiences


<EOL>mask = tf.not_equal(<EOL>x=self.batch_indices,<EOL>y=tf.zeros(shape=tf.shape(input=self.batch_indices), dtype=tf.int32)<EOL>)<EOL>priority_indices = tf.reshape(tensor=tf.where(condition=mask), shape=[-<NUM_LIT:1>])<EOL>sampled_buffer_batch = self.tf_retrieve_indices(<EOL>buffer_elements=self.last_batch_buffer_elems,<EOL>priority_indices=priority_indices<EOL>)<EOL>states = sampled_buffer_batch['<STR_LIT>']<EOL>internals = sampled_buffer_batch['<STR_LIT>']<EOL>actions = sampled_buffer_batch['<STR_LIT>']<EOL>terminal = sampled_buffer_batch['<STR_LIT>']<EOL>reward = sampled_buffer_batch['<STR_LIT>']<EOL>priorities = loss_per_instance ** self.prioritization_weight<EOL>assignments = list()<EOL>memory_end_index = self.memory_index + self.last_batch_buffer_elems<EOL>memory_insert_indices = tf.range(<EOL>start=self.memory_index,<EOL>limit=memory_end_index<EOL>) % self.capacity<EOL>for name in sorted(states):<EOL><INDENT>assignments.append(tf.scatter_update(<EOL>ref=self.states_memory[name],<EOL>indices=memory_insert_indices,<EOL>updates=states[name][<NUM_LIT:0>:self.last_batch_buffer_elems])<EOL>)<EOL><DEDENT>for name in sorted(internals):<EOL><INDENT>assignments.append(tf.scatter_update(<EOL>ref=self.internals_buffer[name],<EOL>indices=memory_insert_indices,<EOL>updates=internals[name][<NUM_LIT:0>:self.last_batch_buffer_elems]<EOL>))<EOL><DEDENT>assignments.append(tf.scatter_update(<EOL>ref=self.priorities,<EOL>indices=memory_insert_indices,<EOL>updates=priorities[<NUM_LIT:0>:self.last_batch_buffer_elems]<EOL>))<EOL>assignments.append(tf.scatter_update(<EOL>ref=self.terminal_memory,<EOL>indices=memory_insert_indices,<EOL>updates=terminal[<NUM_LIT:0>:self.last_batch_buffer_elems])<EOL>)<EOL>assignments.append(tf.scatter_update(<EOL>ref=self.reward_memory,<EOL>indices=memory_insert_indices,<EOL>updates=reward[<NUM_LIT:0>:self.last_batch_buffer_elems])<EOL>)<EOL>for name in sorted(actions):<EOL><INDENT>assignments.append(tf.scatter_update(<EOL>ref=self.actions_memory[name],<EOL>indices=memory_insert_indices,<EOL>updates=actions[name][<NUM_LIT:0>:self.last_batch_buffer_elems]<EOL>))<EOL><DEDENT>main_memory_priorities = priorities[self.last_batch_buffer_elems:]<EOL>main_memory_priorities = main_memory_priorities[<NUM_LIT:0>:tf.shape(priority_indices)[<NUM_LIT:0>]]<EOL>assignments.append(tf.scatter_update(<EOL>ref=self.priorities,<EOL>indices=priority_indices,<EOL>updates=main_memory_priorities<EOL>))<EOL>with tf.control_dependencies(control_inputs=assignments):<EOL><INDENT>assignments = list()<EOL>sorted_priorities, sorted_indices = tf.nn.top_k(<EOL>input=self.priorities,<EOL>k=self.capacity,<EOL>sorted=True<EOL>)<EOL>assignments.append(tf.assign(ref=self.priorities, value=sorted_priorities))<EOL>assignments.append(tf.scatter_update(<EOL>ref=self.terminal_memory,<EOL>indices=sorted_indices,<EOL>updates=self.terminal_memory<EOL>))<EOL>for name in sorted(self.states_memory):<EOL><INDENT>assignments.append(tf.scatter_update(<EOL>ref=self.states_memory[name],<EOL>indices=sorted_indices,<EOL>updates=self.states_memory[name]<EOL>))<EOL><DEDENT>for name in sorted(self.actions_memory):<EOL><INDENT>assignments.append(tf.scatter_update(<EOL>ref=self.actions_memory[name],<EOL>indices=sorted_indices,<EOL>updates=self.actions_memory[name]<EOL>))<EOL><DEDENT>for name in sorted(self.internals_memory):<EOL><INDENT>assignments.append(tf.scatter_update(<EOL>ref=self.internals_memory[name],<EOL>indices=sorted_indices,<EOL>updates=self.internals_memory[name]<EOL>))<EOL><DEDENT>assignments.append(tf.scatter_update(<EOL>ref=self.reward_memory,<EOL>indices=sorted_indices,<EOL>updates=self.reward_memory<EOL>))<EOL><DEDENT>with tf.control_dependencies(control_inputs=assignments):<EOL><INDENT>assignments = list()<EOL>assignments.append(tf.assign_sub(ref=self.buffer_index, value=self.last_batch_buffer_elems))<EOL>total_inserted_elements = self.memory_size + self.last_batch_buffer_elems<EOL>assignments.append(tf.assign(<EOL>ref=self.memory_size,<EOL>value=tf.minimum(x=total_inserted_elements, y=self.capacity))<EOL>)<EOL>assignments.append(tf.assign(ref=self.memory_index, value=memory_end_index))<EOL>assignments.append(tf.assign(<EOL>ref=self.batch_indices,<EOL>value=tf.zeros(shape=tf.shape(self.batch_indices), dtype=tf.int32)<EOL>))<EOL><DEDENT>with tf.control_dependencies(control_inputs=assignments):<EOL><INDENT>return tf.no_op()<EOL><DEDENT>

Updates priority memory by performing the following steps:

1. Use saved indices from prior retrieval to reconstruct the batch
elements which will have their priorities updated.
2. Compute priorities for these elements.
3. Insert buffer elements to memory, potentially overwriting existing elements.
4. Update priorities of existing memory elements
5. Resort memory.
6. Update buffer insertion index.

Note that this implementation could be made more efficient by maintaining
a sorted version via sum trees.

:param loss_per_instance: Losses from recent batch to perform priority update


<EOL>if the_plot.get('<STR_LIT>') == the_plot.frame: return<EOL>the_plot['<STR_LIT>'] = the_plot.frame<EOL>col = np.random.choice(np.nonzero(layers['<STR_LIT:X>'].sum(axis=<NUM_LIT:0>))[<NUM_LIT:0>])<EOL>row = np.nonzero(layers['<STR_LIT:X>'][:, col])[<NUM_LIT:0>][-<NUM_LIT:1>] + <NUM_LIT:1><EOL>self._teleport((row, col))<EOL>

Launches a new bolt from a random Marauder.


def __init__(self, corner, position, character):

super(PlayerSprite, self).__init__(<EOL>corner, position, character, impassable='<STR_LIT>', confined_to_board=True)<EOL>

Simply indicates to the superclass that we can't walk off the board.


def _fly(self, board, layers, things, the_plot):

<EOL>if (self.character in the_plot['<STR_LIT>'] or<EOL>self.character in the_plot['<STR_LIT>']):<EOL><INDENT>return self._teleport((-<NUM_LIT:1>, -<NUM_LIT:1>))<EOL><DEDENT>self._north(board, the_plot)<EOL>

Handles the behaviour of visible bolts flying toward Marauders.


<EOL>if self.character in the_plot['<STR_LIT>']:<EOL><INDENT>return self._teleport((-<NUM_LIT:1>, -<NUM_LIT:1>))<EOL><DEDENT>if self.position == things['<STR_LIT:P>'].position: the_plot.terminate_episode()<EOL>self._south(board, the_plot)<EOL>

Handles the behaviour of visible bolts flying toward the player.


super(UpwardLaserBoltSprite, self).__init__(<EOL>corner, position, character, impassable='<STR_LIT>')<EOL>self._teleport((-<NUM_LIT:1>, -<NUM_LIT:1>))<EOL>

Starts the Sprite in a hidden position off of the board.


return ascii_art.ascii_art_to_game(<EOL>GAME_ART, what_lies_beneath='<STR_LIT:U+0020>',<EOL>sprites=dict(<EOL>[('<STR_LIT:P>', PlayerSprite)] +<EOL>[(c, UpwardLaserBoltSprite) for c in UPWARD_BOLT_CHARS] +<EOL>[(c, DownwardLaserBoltSprite) for c in DOWNWARD_BOLT_CHARS]),<EOL>drapes=dict(X=MarauderDrape,<EOL>B=BunkerDrape),<EOL>update_schedule=['<STR_LIT:P>', '<STR_LIT:B>', '<STR_LIT:X>'] + list(_ALL_BOLT_CHARS))<EOL>

Builds and returns an Extraterrestrial Marauders game.


tag = '<STR_LIT>'<EOL>if ordered:<EOL><INDENT>tag = '<STR_LIT>'<EOL><DEDENT>return '<STR_LIT>' % (tag, body, tag)<EOL>

Rendering list tags like ``<ul>`` and ``<ol>``.

        :param body: body contents of the list.
        :param ordered: whether this list is ordered or not.


if self.options.get('<STR_LIT>'):<EOL><INDENT>return text<EOL><DEDENT>return escape(text)<EOL>

Rendering unformatted text.

        :param text: text content.


if self.options.get('<STR_LIT>'):<EOL><INDENT>return escape(html)<EOL><DEDENT>return html<EOL>

Rendering span level pure html content.

        :param html: text content of the html snippet.


text = escape(text.rstrip(), smart_amp=False)<EOL>return '<STR_LIT>' % text<EOL>

Rendering inline `code` text.

        :param text: text content for inline code.


return escape(text)<EOL>

Rendering escape sequence.

        :param text: text content.


self.tokens.append({<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT:1> if m.group(<NUM_LIT:2>) == '<STR_LIT:=>' else <NUM_LIT:2>,<EOL>'<STR_LIT:text>': m.group(<NUM_LIT:1>),<EOL>})<EOL>


back = (<EOL>'<STR_LIT>'<EOL>) % escape(key)<EOL>text = text.rstrip()<EOL>if text.endswith('<STR_LIT>'):<EOL><INDENT>text = re.sub(r'<STR_LIT>', r'<STR_LIT>' % back, text)<EOL><DEDENT>else:<EOL><INDENT>text = '<STR_LIT>' % (text, back)<EOL><DEDENT>html = '<STR_LIT>' % (escape(key), text)<EOL>return html<EOL>

Rendering a footnote item.

        :param key: identity key for the footnote.
        :param text: text content of the footnote.