assert beta > <NUM_LIT:0><EOL>idxes = self._sample_proportional(batch_size)<EOL>weights = []<EOL>p_min = self._it_min.min() / self._it_sum.sum()<EOL>max_weight = (p_min * len(self._storage)) ** (-beta)<EOL>for idx in idxes:<EOL><INDENT>p_sample = self._it_sum[idx] / self._it_sum.sum()<EOL>weight = (p_sample * len(self._storage)) ** (-beta)<EOL>weights.append(weight / max_weight)<EOL><DEDENT>weights = np.array(weights)<EOL>encoded_sample = self._encode_sample(idxes)<EOL>return tuple(list(encoded_sample) + [weights, idxes])<EOL>

Sample a batch of experiences.

        compared to ReplayBuffer.sample
        it also returns importance weights and idxes
        of sampled experiences.

        Parameters
        ----------
        batch_size: int
            How many transitions to sample.
        beta: float
            To what degree to use importance weights
            (0 - no corrections, 1 - full correction)

        Returns
        -------
        obs_batch: np.array
            batch of observations
        act_batch: np.array
            batch of actions executed given obs_batch
        rew_batch: np.array
            rewards received as results of executing act_batch
        next_obs_batch: np.array
            next set of observations seen after executing act_batch
        done_mask: np.array
            done_mask[i] = 1 if executing act_batch[i] resulted in
            the end of an episode and 0 otherwise.
        weights: np.array
            Array of shape (batch_size,) and dtype np.float32
            denoting importance weight of each sampled transition
        idxes: np.array
            Array of shape (batch_size,) and dtype np.int32
            idexes in buffer of sampled experiences


def update_priorities(self, idxes, priorities):

assert len(idxes) == len(priorities)<EOL>for idx, priority in zip(idxes, priorities):<EOL><INDENT>assert priority > <NUM_LIT:0><EOL>assert <NUM_LIT:0> <= idx < len(self._storage)<EOL>self._it_sum[idx] = priority ** self._alpha<EOL>self._it_min[idx] = priority ** self._alpha<EOL>self._max_priority = max(self._max_priority, priority)<EOL><DEDENT>

Update priorities of sampled transitions.

        sets priority of transition at index idxes[i] in buffer
        to priorities[i].

        Parameters
        ----------
        idxes: [int]
            List of idxes of sampled transitions
        priorities: [float]
            List of updated priorities corresponding to
            transitions at the sampled idxes denoted by
            variable `idxes`.


return ActWrapper.load_act(path)<EOL>

Load act function that was returned by learn function.

    Parameters
    ----------
    path: str
        path to the act function pickle

    Returns
    -------
    act: ActWrapper
        function that takes a batch of observations
        and returns actions.


<EOL>sess = get_session()<EOL>set_global_seeds(seed)<EOL>q_func = build_q_func(network, **network_kwargs)<EOL>observation_space = env.observation_space<EOL>def make_obs_ph(name):<EOL><INDENT>return ObservationInput(observation_space, name=name)<EOL><DEDENT>act, train, update_target, debug = deepq.build_train(<EOL>make_obs_ph=make_obs_ph,<EOL>q_func=q_func,<EOL>num_actions=env.action_space.n,<EOL>optimizer=tf.train.AdamOptimizer(learning_rate=lr),<EOL>gamma=gamma,<EOL>grad_norm_clipping=<NUM_LIT:10>,<EOL>param_noise=param_noise<EOL>)<EOL>act_params = {<EOL>'<STR_LIT>': make_obs_ph,<EOL>'<STR_LIT>': q_func,<EOL>'<STR_LIT>': env.action_space.n,<EOL>}<EOL>act = ActWrapper(act, act_params)<EOL>if prioritized_replay:<EOL><INDENT>replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)<EOL>if prioritized_replay_beta_iters is None:<EOL><INDENT>prioritized_replay_beta_iters = total_timesteps<EOL><DEDENT>beta_schedule = LinearSchedule(prioritized_replay_beta_iters,<EOL>initial_p=prioritized_replay_beta0,<EOL>final_p=<NUM_LIT:1.0>)<EOL><DEDENT>else:<EOL><INDENT>replay_buffer = ReplayBuffer(buffer_size)<EOL>beta_schedule = None<EOL><DEDENT>exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * total_timesteps),<EOL>initial_p=<NUM_LIT:1.0>,<EOL>final_p=exploration_final_eps)<EOL>U.initialize()<EOL>update_target()<EOL>episode_rewards = [<NUM_LIT:0.0>]<EOL>saved_mean_reward = None<EOL>obs = env.reset()<EOL>reset = True<EOL>with tempfile.TemporaryDirectory() as td:<EOL><INDENT>td = checkpoint_path or td<EOL>model_file = os.path.join(td, "<STR_LIT>")<EOL>model_saved = False<EOL>if tf.train.latest_checkpoint(td) is not None:<EOL><INDENT>load_variables(model_file)<EOL>logger.log('<STR_LIT>'.format(model_file))<EOL>model_saved = True<EOL><DEDENT>elif load_path is not None:<EOL><INDENT>load_variables(load_path)<EOL>logger.log('<STR_LIT>'.format(load_path))<EOL><DEDENT>for t in range(total_timesteps):<EOL><INDENT>if callback is not None:<EOL><INDENT>if callback(locals(), globals()):<EOL><INDENT>break<EOL><DEDENT><DEDENT>kwargs = {}<EOL>if not param_noise:<EOL><INDENT>update_eps = exploration.value(t)<EOL>update_param_noise_threshold = <NUM_LIT:0.><EOL><DEDENT>else:<EOL><INDENT>update_eps = <NUM_LIT:0.><EOL>update_param_noise_threshold = -np.log(<NUM_LIT:1.> - exploration.value(t) + exploration.value(t) / float(env.action_space.n))<EOL>kwargs['<STR_LIT>'] = reset<EOL>kwargs['<STR_LIT>'] = update_param_noise_threshold<EOL>kwargs['<STR_LIT>'] = True<EOL><DEDENT>action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[<NUM_LIT:0>]<EOL>env_action = action<EOL>reset = False<EOL>new_obs, rew, done, _ = env.step(env_action)<EOL>replay_buffer.add(obs, action, rew, new_obs, float(done))<EOL>obs = new_obs<EOL>episode_rewards[-<NUM_LIT:1>] += rew<EOL>if done:<EOL><INDENT>obs = env.reset()<EOL>episode_rewards.append(<NUM_LIT:0.0>)<EOL>reset = True<EOL><DEDENT>if t > learning_starts and t % train_freq == <NUM_LIT:0>:<EOL><INDENT>if prioritized_replay:<EOL><INDENT>experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))<EOL>(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience<EOL><DEDENT>else:<EOL><INDENT>obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)<EOL>weights, batch_idxes = np.ones_like(rewards), None<EOL><DEDENT>td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)<EOL>if prioritized_replay:<EOL><INDENT>new_priorities = np.abs(td_errors) + prioritized_replay_eps<EOL>replay_buffer.update_priorities(batch_idxes, new_priorities)<EOL><DEDENT><DEDENT>if t > learning_starts and t % target_network_update_freq == <NUM_LIT:0>:<EOL><INDENT>update_target()<EOL><DEDENT>mean_100ep_reward = round(np.mean(episode_rewards[-<NUM_LIT>:-<NUM_LIT:1>]), <NUM_LIT:1>)<EOL>num_episodes = len(episode_rewards)<EOL>if done and print_freq is not None and len(episode_rewards) % print_freq == <NUM_LIT:0>:<EOL><INDENT>logger.record_tabular("<STR_LIT>", t)<EOL>logger.record_tabular("<STR_LIT>", num_episodes)<EOL>logger.record_tabular("<STR_LIT>", mean_100ep_reward)<EOL>logger.record_tabular("<STR_LIT>", int(<NUM_LIT:100> * exploration.value(t)))<EOL>logger.dump_tabular()<EOL><DEDENT>if (checkpoint_freq is not None and t > learning_starts and<EOL>num_episodes > <NUM_LIT:100> and t % checkpoint_freq == <NUM_LIT:0>):<EOL><INDENT>if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:<EOL><INDENT>if print_freq is not None:<EOL><INDENT>logger.log("<STR_LIT>".format(<EOL>saved_mean_reward, mean_100ep_reward))<EOL><DEDENT>save_variables(model_file)<EOL>model_saved = True<EOL>saved_mean_reward = mean_100ep_reward<EOL><DEDENT><DEDENT><DEDENT>if model_saved:<EOL><INDENT>if print_freq is not None:<EOL><INDENT>logger.log("<STR_LIT>".format(saved_mean_reward))<EOL><DEDENT>load_variables(model_file)<EOL><DEDENT><DEDENT>return act<EOL>

Train a deepq model.

    Parameters
    -------
    env: gym.Env
        environment to train on
    network: string or a function
        neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
        (mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
        will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
    seed: int or None
        prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
    lr: float
        learning rate for adam optimizer
    total_timesteps: int
        number of env steps to optimizer for
    buffer_size: int
        size of the replay buffer
    exploration_fraction: float
        fraction of entire training period over which the exploration rate is annealed
    exploration_final_eps: float
        final value of random action probability
    train_freq: int
        update the model every `train_freq` steps.
        set to None to disable printing
    batch_size: int
        size of a batched sampled from replay buffer for training
    print_freq: int
        how often to print out training progress
        set to None to disable printing
    checkpoint_freq: int
        how often to save the model. This is so that the best version is restored
        at the end of the training. If you do not wish to restore the best version at
        the end of the training set this variable to None.
    learning_starts: int
        how many steps of the model to collect transitions for before learning starts
    gamma: float
        discount factor
    target_network_update_freq: int
        update the target network every `target_network_update_freq` steps.
    prioritized_replay: True
        if True prioritized replay buffer will be used.
    prioritized_replay_alpha: float
        alpha parameter for prioritized replay buffer
    prioritized_replay_beta0: float
        initial value of beta for prioritized replay buffer
    prioritized_replay_beta_iters: int
        number of iterations over which beta will be annealed from initial value
        to 1.0. If set to None equals to total_timesteps.
    prioritized_replay_eps: float
        epsilon to add to the TD errors when updating priorities.
    param_noise: bool
        whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
    callback: (locals, globals) -> None
        function called at every steps with state of the algorithm.
        If callback returns true training stops.
    load_path: str
        path to load the model from. (default: None)
    **network_kwargs
        additional keyword arguments to pass to the network builder.

    Returns
    -------
    act: ActWrapper
        Wrapper over act function. Adds ability to save it and load it.
        See header of baselines/deepq/categorical.py for details on the act function.


if path is None:<EOL><INDENT>path = os.path.join(logger.get_dir(), "<STR_LIT>")<EOL><DEDENT>with tempfile.TemporaryDirectory() as td:<EOL><INDENT>save_variables(os.path.join(td, "<STR_LIT>"))<EOL>arc_name = os.path.join(td, "<STR_LIT>")<EOL>with zipfile.ZipFile(arc_name, '<STR_LIT:w>') as zipf:<EOL><INDENT>for root, dirs, files in os.walk(td):<EOL><INDENT>for fname in files:<EOL><INDENT>file_path = os.path.join(root, fname)<EOL>if file_path != arc_name:<EOL><INDENT>zipf.write(file_path, os.path.relpath(file_path, td))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>with open(arc_name, "<STR_LIT:rb>") as f:<EOL><INDENT>model_data = f.read()<EOL><DEDENT><DEDENT>with open(path, "<STR_LIT:wb>") as f:<EOL><INDENT>cloudpickle.dump((model_data, self._act_params), f)<EOL><DEDENT>

Save model to a pickle located at `path`


self.name = name<EOL>

Generalized Tensorflow placeholder. The main differences are:
            - possibly uses multiple placeholders internally and returns multiple values
            - can apply light postprocessing to the value feed to placeholder.


raise NotImplementedError<EOL>

Return the tf variable(s) representing the possibly postprocessed value
        of placeholder(s).


raise NotImplementedError<EOL>

Given data input it to the placeholder(s).


super().__init__(placeholder.name)<EOL>self._placeholder = placeholder<EOL>

Wrapper for regular tensorflow placeholder.


def __init__(self, observation_space, name=None):

inpt, self.processed_inpt = observation_input(observation_space, name=name)<EOL>super().__init__(inpt)<EOL>

Creates an input placeholder tailored to a specific observation space

        Parameters
        ----------

        observation_space:
                observation space of the environment. Should be one of the gym.spaces types
        name: str
                tensorflow name of the underlying placeholder


return tf.get_collection(<EOL>tf.GraphKeys.TRAINABLE_VARIABLES if trainable_only else tf.GraphKeys.GLOBAL_VARIABLES,<EOL>scope=scope if isinstance(scope, str) else scope.name<EOL>)<EOL>

Get variables inside a scope
The scope can be specified as a string
Parameters
----------
scope: str or VariableScope
    scope in which the variables reside.
trainable_only: bool
    whether or not to return only the variables that were marked as trainable.
Returns
-------
vars: [tf.Variable]
    list of variables in `scope`.


return tf.get_variable_scope().name<EOL>

Returns the name of current scope as a string, e.g. deepq/q_func


return scope_name() + "<STR_LIT:/>" + relative_scope_name<EOL>

Appends parent scope name to `relative_scope_name`


def build_act(make_obs_ph, q_func, num_actions, scope="<STR_LIT>", reuse=None):

with tf.variable_scope(scope, reuse=reuse):<EOL><INDENT>observations_ph = make_obs_ph("<STR_LIT>")<EOL>stochastic_ph = tf.placeholder(tf.bool, (), name="<STR_LIT>")<EOL>update_eps_ph = tf.placeholder(tf.float32, (), name="<STR_LIT>")<EOL>eps = tf.get_variable("<STR_LIT>", (), initializer=tf.constant_initializer(<NUM_LIT:0>))<EOL>q_values = q_func(observations_ph.get(), num_actions, scope="<STR_LIT>")<EOL>deterministic_actions = tf.argmax(q_values, axis=<NUM_LIT:1>)<EOL>batch_size = tf.shape(observations_ph.get())[<NUM_LIT:0>]<EOL>random_actions = tf.random_uniform(tf.stack([batch_size]), minval=<NUM_LIT:0>, maxval=num_actions, dtype=tf.int64)<EOL>chose_random = tf.random_uniform(tf.stack([batch_size]), minval=<NUM_LIT:0>, maxval=<NUM_LIT:1>, dtype=tf.float32) < eps<EOL>stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions)<EOL>output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions)<EOL>update_eps_expr = eps.assign(tf.cond(update_eps_ph >= <NUM_LIT:0>, lambda: update_eps_ph, lambda: eps))<EOL>_act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph],<EOL>outputs=output_actions,<EOL>givens={update_eps_ph: -<NUM_LIT:1.0>, stochastic_ph: True},<EOL>updates=[update_eps_expr])<EOL>def act(ob, stochastic=True, update_eps=-<NUM_LIT:1>):<EOL><INDENT>return _act(ob, stochastic, update_eps)<EOL><DEDENT>return act<EOL><DEDENT>

Creates the act function:

    Parameters
    ----------
    make_obs_ph: str -> tf.placeholder or TfInput
        a function that take a name and creates a placeholder of input with that name
    q_func: (tf.Variable, int, str, bool) -> tf.Variable
        the model that takes the following inputs:
            observation_in: object
                the output of observation placeholder
            num_actions: int
                number of actions
            scope: str
            reuse: bool
                should be passed to outer variable scope
        and returns a tensor of shape (batch_size, num_actions) with values of every action.
    num_actions: int
        number of actions.
    scope: str or VariableScope
        optional scope for variable_scope.
    reuse: bool or None
        whether or not the variables should be reused. To be able to reuse the scope must be given.

    Returns
    -------
    act: (tf.Variable, bool, float) -> tf.Variable
        function to select and action given observation.
`       See the top of the file for details.


def build_act_with_param_noise(make_obs_ph, q_func, num_actions, scope="<STR_LIT>", reuse=None, param_noise_filter_func=None):

if param_noise_filter_func is None:<EOL><INDENT>param_noise_filter_func = default_param_noise_filter<EOL><DEDENT>with tf.variable_scope(scope, reuse=reuse):<EOL><INDENT>observations_ph = make_obs_ph("<STR_LIT>")<EOL>stochastic_ph = tf.placeholder(tf.bool, (), name="<STR_LIT>")<EOL>update_eps_ph = tf.placeholder(tf.float32, (), name="<STR_LIT>")<EOL>update_param_noise_threshold_ph = tf.placeholder(tf.float32, (), name="<STR_LIT>")<EOL>update_param_noise_scale_ph = tf.placeholder(tf.bool, (), name="<STR_LIT>")<EOL>reset_ph = tf.placeholder(tf.bool, (), name="<STR_LIT>")<EOL>eps = tf.get_variable("<STR_LIT>", (), initializer=tf.constant_initializer(<NUM_LIT:0>))<EOL>param_noise_scale = tf.get_variable("<STR_LIT>", (), initializer=tf.constant_initializer(<NUM_LIT>), trainable=False)<EOL>param_noise_threshold = tf.get_variable("<STR_LIT>", (), initializer=tf.constant_initializer(<NUM_LIT>), trainable=False)<EOL>q_values = q_func(observations_ph.get(), num_actions, scope="<STR_LIT>")<EOL>q_values_perturbed = q_func(observations_ph.get(), num_actions, scope="<STR_LIT>")<EOL>def perturb_vars(original_scope, perturbed_scope):<EOL><INDENT>all_vars = scope_vars(absolute_scope_name(original_scope))<EOL>all_perturbed_vars = scope_vars(absolute_scope_name(perturbed_scope))<EOL>assert len(all_vars) == len(all_perturbed_vars)<EOL>perturb_ops = []<EOL>for var, perturbed_var in zip(all_vars, all_perturbed_vars):<EOL><INDENT>if param_noise_filter_func(perturbed_var):<EOL><INDENT>op = tf.assign(perturbed_var, var + tf.random_normal(shape=tf.shape(var), mean=<NUM_LIT:0.>, stddev=param_noise_scale))<EOL><DEDENT>else:<EOL><INDENT>op = tf.assign(perturbed_var, var)<EOL><DEDENT>perturb_ops.append(op)<EOL><DEDENT>assert len(perturb_ops) == len(all_vars)<EOL>return tf.group(*perturb_ops)<EOL><DEDENT>q_values_adaptive = q_func(observations_ph.get(), num_actions, scope="<STR_LIT>")<EOL>perturb_for_adaption = perturb_vars(original_scope="<STR_LIT>", perturbed_scope="<STR_LIT>")<EOL>kl = tf.reduce_sum(tf.nn.softmax(q_values) * (tf.log(tf.nn.softmax(q_values)) - tf.log(tf.nn.softmax(q_values_adaptive))), axis=-<NUM_LIT:1>)<EOL>mean_kl = tf.reduce_mean(kl)<EOL>def update_scale():<EOL><INDENT>with tf.control_dependencies([perturb_for_adaption]):<EOL><INDENT>update_scale_expr = tf.cond(mean_kl < param_noise_threshold,<EOL>lambda: param_noise_scale.assign(param_noise_scale * <NUM_LIT>),<EOL>lambda: param_noise_scale.assign(param_noise_scale / <NUM_LIT>),<EOL>)<EOL><DEDENT>return update_scale_expr<EOL><DEDENT>update_param_noise_threshold_expr = param_noise_threshold.assign(tf.cond(update_param_noise_threshold_ph >= <NUM_LIT:0>,<EOL>lambda: update_param_noise_threshold_ph, lambda: param_noise_threshold))<EOL>deterministic_actions = tf.argmax(q_values_perturbed, axis=<NUM_LIT:1>)<EOL>batch_size = tf.shape(observations_ph.get())[<NUM_LIT:0>]<EOL>random_actions = tf.random_uniform(tf.stack([batch_size]), minval=<NUM_LIT:0>, maxval=num_actions, dtype=tf.int64)<EOL>chose_random = tf.random_uniform(tf.stack([batch_size]), minval=<NUM_LIT:0>, maxval=<NUM_LIT:1>, dtype=tf.float32) < eps<EOL>stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions)<EOL>output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions)<EOL>update_eps_expr = eps.assign(tf.cond(update_eps_ph >= <NUM_LIT:0>, lambda: update_eps_ph, lambda: eps))<EOL>updates = [<EOL>update_eps_expr,<EOL>tf.cond(reset_ph, lambda: perturb_vars(original_scope="<STR_LIT>", perturbed_scope="<STR_LIT>"), lambda: tf.group(*[])),<EOL>tf.cond(update_param_noise_scale_ph, lambda: update_scale(), lambda: tf.Variable(<NUM_LIT:0.>, trainable=False)),<EOL>update_param_noise_threshold_expr,<EOL>]<EOL>_act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph, reset_ph, update_param_noise_threshold_ph, update_param_noise_scale_ph],<EOL>outputs=output_actions,<EOL>givens={update_eps_ph: -<NUM_LIT:1.0>, stochastic_ph: True, reset_ph: False, update_param_noise_threshold_ph: False, update_param_noise_scale_ph: False},<EOL>updates=updates)<EOL>def act(ob, reset=False, update_param_noise_threshold=False, update_param_noise_scale=False, stochastic=True, update_eps=-<NUM_LIT:1>):<EOL><INDENT>return _act(ob, stochastic, update_eps, reset, update_param_noise_threshold, update_param_noise_scale)<EOL><DEDENT>return act<EOL><DEDENT>

Creates the act function with support for parameter space noise exploration (https://arxiv.org/abs/1706.01905):

    Parameters
    ----------
    make_obs_ph: str -> tf.placeholder or TfInput
        a function that take a name and creates a placeholder of input with that name
    q_func: (tf.Variable, int, str, bool) -> tf.Variable
        the model that takes the following inputs:
            observation_in: object
                the output of observation placeholder
            num_actions: int
                number of actions
            scope: str
            reuse: bool
                should be passed to outer variable scope
        and returns a tensor of shape (batch_size, num_actions) with values of every action.
    num_actions: int
        number of actions.
    scope: str or VariableScope
        optional scope for variable_scope.
    reuse: bool or None
        whether or not the variables should be reused. To be able to reuse the scope must be given.
    param_noise_filter_func: tf.Variable -> bool
        function that decides whether or not a variable should be perturbed. Only applicable
        if param_noise is True. If set to None, default_param_noise_filter is used by default.

    Returns
    -------
    act: (tf.Variable, bool, float, bool, float, bool) -> tf.Variable
        function to select and action given observation.
`       See the top of the file for details.


def build_train(make_obs_ph, q_func, num_actions, optimizer, grad_norm_clipping=None, gamma=<NUM_LIT:1.0>,<EOL>double_q=True, scope="<STR_LIT>", reuse=None, param_noise=False, param_noise_filter_func=None):

if param_noise:<EOL><INDENT>act_f = build_act_with_param_noise(make_obs_ph, q_func, num_actions, scope=scope, reuse=reuse,<EOL>param_noise_filter_func=param_noise_filter_func)<EOL><DEDENT>else:<EOL><INDENT>act_f = build_act(make_obs_ph, q_func, num_actions, scope=scope, reuse=reuse)<EOL><DEDENT>with tf.variable_scope(scope, reuse=reuse):<EOL><INDENT>obs_t_input = make_obs_ph("<STR_LIT>")<EOL>act_t_ph = tf.placeholder(tf.int32, [None], name="<STR_LIT:action>")<EOL>rew_t_ph = tf.placeholder(tf.float32, [None], name="<STR_LIT>")<EOL>obs_tp1_input = make_obs_ph("<STR_LIT>")<EOL>done_mask_ph = tf.placeholder(tf.float32, [None], name="<STR_LIT>")<EOL>importance_weights_ph = tf.placeholder(tf.float32, [None], name="<STR_LIT>")<EOL>q_t = q_func(obs_t_input.get(), num_actions, scope="<STR_LIT>", reuse=True)  <EOL>q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=tf.get_variable_scope().name + "<STR_LIT>")<EOL>q_tp1 = q_func(obs_tp1_input.get(), num_actions, scope="<STR_LIT>")<EOL>target_q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=tf.get_variable_scope().name + "<STR_LIT>")<EOL>q_t_selected = tf.reduce_sum(q_t * tf.one_hot(act_t_ph, num_actions), <NUM_LIT:1>)<EOL>if double_q:<EOL><INDENT>q_tp1_using_online_net = q_func(obs_tp1_input.get(), num_actions, scope="<STR_LIT>", reuse=True)<EOL>q_tp1_best_using_online_net = tf.argmax(q_tp1_using_online_net, <NUM_LIT:1>)<EOL>q_tp1_best = tf.reduce_sum(q_tp1 * tf.one_hot(q_tp1_best_using_online_net, num_actions), <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>q_tp1_best = tf.reduce_max(q_tp1, <NUM_LIT:1>)<EOL><DEDENT>q_tp1_best_masked = (<NUM_LIT:1.0> - done_mask_ph) * q_tp1_best<EOL>q_t_selected_target = rew_t_ph + gamma * q_tp1_best_masked<EOL>td_error = q_t_selected - tf.stop_gradient(q_t_selected_target)<EOL>errors = U.huber_loss(td_error)<EOL>weighted_error = tf.reduce_mean(importance_weights_ph * errors)<EOL>if grad_norm_clipping is not None:<EOL><INDENT>gradients = optimizer.compute_gradients(weighted_error, var_list=q_func_vars)<EOL>for i, (grad, var) in enumerate(gradients):<EOL><INDENT>if grad is not None:<EOL><INDENT>gradients[i] = (tf.clip_by_norm(grad, grad_norm_clipping), var)<EOL><DEDENT><DEDENT>optimize_expr = optimizer.apply_gradients(gradients)<EOL><DEDENT>else:<EOL><INDENT>optimize_expr = optimizer.minimize(weighted_error, var_list=q_func_vars)<EOL><DEDENT>update_target_expr = []<EOL>for var, var_target in zip(sorted(q_func_vars, key=lambda v: v.name),<EOL>sorted(target_q_func_vars, key=lambda v: v.name)):<EOL><INDENT>update_target_expr.append(var_target.assign(var))<EOL><DEDENT>update_target_expr = tf.group(*update_target_expr)<EOL>train = U.function(<EOL>inputs=[<EOL>obs_t_input,<EOL>act_t_ph,<EOL>rew_t_ph,<EOL>obs_tp1_input,<EOL>done_mask_ph,<EOL>importance_weights_ph<EOL>],<EOL>outputs=td_error,<EOL>updates=[optimize_expr]<EOL>)<EOL>update_target = U.function([], [], updates=[update_target_expr])<EOL>q_values = U.function([obs_t_input], q_t)<EOL>return act_f, train, update_target, {'<STR_LIT>': q_values}<EOL><DEDENT>

Creates the train function:

    Parameters
    ----------
    make_obs_ph: str -> tf.placeholder or TfInput
        a function that takes a name and creates a placeholder of input with that name
    q_func: (tf.Variable, int, str, bool) -> tf.Variable
        the model that takes the following inputs:
            observation_in: object
                the output of observation placeholder
            num_actions: int
                number of actions
            scope: str
            reuse: bool
                should be passed to outer variable scope
        and returns a tensor of shape (batch_size, num_actions) with values of every action.
    num_actions: int
        number of actions
    reuse: bool
        whether or not to reuse the graph variables
    optimizer: tf.train.Optimizer
        optimizer to use for the Q-learning objective.
    grad_norm_clipping: float or None
        clip gradient norms to this value. If None no clipping is performed.
    gamma: float
        discount rate.
    double_q: bool
        if true will use Double Q Learning (https://arxiv.org/abs/1509.06461).
        In general it is a good idea to keep it enabled.
    scope: str or VariableScope
        optional scope for variable_scope.
    reuse: bool or None
        whether or not the variables should be reused. To be able to reuse the scope must be given.
    param_noise: bool
        whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
    param_noise_filter_func: tf.Variable -> bool
        function that decides whether or not a variable should be perturbed. Only applicable
        if param_noise is True. If set to None, default_param_noise_filter is used by default.

    Returns
    -------
    act: (tf.Variable, bool, float) -> tf.Variable
        function to select and action given observation.
`       See the top of the file for details.
    train: (object, np.array, np.array, object, np.array, np.array) -> np.array
        optimize the error in Bellman's equation.
`       See the top of the file for details.
    update_target: () -> ()
        copy the parameters from optimized Q function to the target Q function.
`       See the top of the file for details.
    debug: {str: function}
        a bunch of functions to print debug data like q_values.


return lambda *args, **kwargs: _mlp(hiddens, layer_norm=layer_norm, *args, **kwargs)<EOL>

This model takes as input an observation and returns values of all actions.

    Parameters
    ----------
    hiddens: [int]
        list of sizes of hidden layers
    layer_norm: bool
        if true applies layer normalization for every layer
        as described in https://arxiv.org/abs/1607.06450

    Returns
    -------
    q_func: function
        q_function for DQN algorithm.


def cnn_to_mlp(convs, hiddens, dueling=False, layer_norm=False):

return lambda *args, **kwargs: _cnn_to_mlp(convs, hiddens, dueling, layer_norm=layer_norm, *args, **kwargs)<EOL>

This model takes as input an observation and returns values of all actions.

    Parameters
    ----------
    convs: [(int, int, int)]
        list of convolutional layers in form of
        (num_outputs, kernel_size, stride)
    hiddens: [int]
        list of sizes of hidden layers
    dueling: bool
        if true double the output MLP to compute a baseline
        for action scores
    layer_norm: bool
        if true applies layer normalization for every layer
        as described in https://arxiv.org/abs/1607.06450

    Returns
    -------
    q_func: function
        q_function for DQN algorithm.


def model(inpt, num_actions, scope, reuse=False):

with tf.variable_scope(scope, reuse=reuse):<EOL><INDENT>out = inpt<EOL>out = layers.fully_connected(out, num_outputs=<NUM_LIT:64>, activation_fn=tf.nn.tanh)<EOL>out = layers.fully_connected(out, num_outputs=num_actions, activation_fn=None)<EOL>return out<EOL><DEDENT>

This model takes as input an observation and returns values of all actions.


def learn(*, network, env, total_timesteps, eval_env = None, seed=None, nsteps=<NUM_LIT>, ent_coef=<NUM_LIT:0.0>, lr=<NUM_LIT>,<EOL>vf_coef=<NUM_LIT:0.5>,  max_grad_norm=<NUM_LIT:0.5>, gamma=<NUM_LIT>, lam=<NUM_LIT>,<EOL>log_interval=<NUM_LIT:10>, nminibatches=<NUM_LIT:4>, noptepochs=<NUM_LIT:4>, cliprange=<NUM_LIT>,<EOL>save_interval=<NUM_LIT:0>, load_path=None, model_fn=None, **network_kwargs):

set_global_seeds(seed)<EOL>if isinstance(lr, float): lr = constfn(lr)<EOL>else: assert callable(lr)<EOL>if isinstance(cliprange, float): cliprange = constfn(cliprange)<EOL>else: assert callable(cliprange)<EOL>total_timesteps = int(total_timesteps)<EOL>policy = build_policy(env, network, **network_kwargs)<EOL>nenvs = env.num_envs<EOL>ob_space = env.observation_space<EOL>ac_space = env.action_space<EOL>nbatch = nenvs * nsteps<EOL>nbatch_train = nbatch // nminibatches<EOL>if model_fn is None:<EOL><INDENT>from baselines.ppo2.model import Model<EOL>model_fn = Model<EOL><DEDENT>model = model_fn(policy=policy, ob_space=ob_space, ac_space=ac_space, nbatch_act=nenvs, nbatch_train=nbatch_train,<EOL>nsteps=nsteps, ent_coef=ent_coef, vf_coef=vf_coef,<EOL>max_grad_norm=max_grad_norm)<EOL>if load_path is not None:<EOL><INDENT>model.load(load_path)<EOL><DEDENT>runner = Runner(env=env, model=model, nsteps=nsteps, gamma=gamma, lam=lam)<EOL>if eval_env is not None:<EOL><INDENT>eval_runner = Runner(env = eval_env, model = model, nsteps = nsteps, gamma = gamma, lam= lam)<EOL><DEDENT>epinfobuf = deque(maxlen=<NUM_LIT:100>)<EOL>if eval_env is not None:<EOL><INDENT>eval_epinfobuf = deque(maxlen=<NUM_LIT:100>)<EOL><DEDENT>tfirststart = time.perf_counter()<EOL>nupdates = total_timesteps//nbatch<EOL>for update in range(<NUM_LIT:1>, nupdates+<NUM_LIT:1>):<EOL><INDENT>assert nbatch % nminibatches == <NUM_LIT:0><EOL>tstart = time.perf_counter()<EOL>frac = <NUM_LIT:1.0> - (update - <NUM_LIT:1.0>) / nupdates<EOL>lrnow = lr(frac)<EOL>cliprangenow = cliprange(frac)<EOL>obs, returns, masks, actions, values, neglogpacs, states, epinfos = runner.run() <EOL>if eval_env is not None:<EOL><INDENT>eval_obs, eval_returns, eval_masks, eval_actions, eval_values, eval_neglogpacs, eval_states, eval_epinfos = eval_runner.run() <EOL><DEDENT>epinfobuf.extend(epinfos)<EOL>if eval_env is not None:<EOL><INDENT>eval_epinfobuf.extend(eval_epinfos)<EOL><DEDENT>mblossvals = []<EOL>if states is None: <EOL><INDENT>inds = np.arange(nbatch)<EOL>for _ in range(noptepochs):<EOL><INDENT>np.random.shuffle(inds)<EOL>for start in range(<NUM_LIT:0>, nbatch, nbatch_train):<EOL><INDENT>end = start + nbatch_train<EOL>mbinds = inds[start:end]<EOL>slices = (arr[mbinds] for arr in (obs, returns, masks, actions, values, neglogpacs))<EOL>mblossvals.append(model.train(lrnow, cliprangenow, *slices))<EOL><DEDENT><DEDENT><DEDENT>else: <EOL><INDENT>assert nenvs % nminibatches == <NUM_LIT:0><EOL>envsperbatch = nenvs // nminibatches<EOL>envinds = np.arange(nenvs)<EOL>flatinds = np.arange(nenvs * nsteps).reshape(nenvs, nsteps)<EOL>for _ in range(noptepochs):<EOL><INDENT>np.random.shuffle(envinds)<EOL>for start in range(<NUM_LIT:0>, nenvs, envsperbatch):<EOL><INDENT>end = start + envsperbatch<EOL>mbenvinds = envinds[start:end]<EOL>mbflatinds = flatinds[mbenvinds].ravel()<EOL>slices = (arr[mbflatinds] for arr in (obs, returns, masks, actions, values, neglogpacs))<EOL>mbstates = states[mbenvinds]<EOL>mblossvals.append(model.train(lrnow, cliprangenow, *slices, mbstates))<EOL><DEDENT><DEDENT><DEDENT>lossvals = np.mean(mblossvals, axis=<NUM_LIT:0>)<EOL>tnow = time.perf_counter()<EOL>fps = int(nbatch / (tnow - tstart))<EOL>if update % log_interval == <NUM_LIT:0> or update == <NUM_LIT:1>:<EOL><INDENT>ev = explained_variance(values, returns)<EOL>logger.logkv("<STR_LIT>", update*nsteps)<EOL>logger.logkv("<STR_LIT>", update)<EOL>logger.logkv("<STR_LIT>", update*nbatch)<EOL>logger.logkv("<STR_LIT>", fps)<EOL>logger.logkv("<STR_LIT>", float(ev))<EOL>logger.logkv('<STR_LIT>', safemean([epinfo['<STR_LIT:r>'] for epinfo in epinfobuf]))<EOL>logger.logkv('<STR_LIT>', safemean([epinfo['<STR_LIT:l>'] for epinfo in epinfobuf]))<EOL>if eval_env is not None:<EOL><INDENT>logger.logkv('<STR_LIT>', safemean([epinfo['<STR_LIT:r>'] for epinfo in eval_epinfobuf]) )<EOL>logger.logkv('<STR_LIT>', safemean([epinfo['<STR_LIT:l>'] for epinfo in eval_epinfobuf]) )<EOL><DEDENT>logger.logkv('<STR_LIT>', tnow - tfirststart)<EOL>for (lossval, lossname) in zip(lossvals, model.loss_names):<EOL><INDENT>logger.logkv(lossname, lossval)<EOL><DEDENT>if MPI is None or MPI.COMM_WORLD.Get_rank() == <NUM_LIT:0>:<EOL><INDENT>logger.dumpkvs()<EOL><DEDENT><DEDENT>if save_interval and (update % save_interval == <NUM_LIT:0> or update == <NUM_LIT:1>) and logger.get_dir() and (MPI is None or MPI.COMM_WORLD.Get_rank() == <NUM_LIT:0>):<EOL><INDENT>checkdir = osp.join(logger.get_dir(), '<STR_LIT>')<EOL>os.makedirs(checkdir, exist_ok=True)<EOL>savepath = osp.join(checkdir, '<STR_LIT>'%update)<EOL>print('<STR_LIT>', savepath)<EOL>model.save(savepath)<EOL><DEDENT><DEDENT>return model<EOL>

Learn policy using PPO algorithm (https://arxiv.org/abs/1707.06347)

Parameters:
----------

network:                          policy network architecture. Either string (mlp, lstm, lnlstm, cnn_lstm, cnn, cnn_small, conv_only - see baselines.common/models.py for full list)
                                  specifying the standard network architecture, or a function that takes tensorflow tensor as input and returns
                                  tuple (output_tensor, extra_feed) where output tensor is the last network layer output, extra_feed is None for feed-forward
                                  neural nets, and extra_feed is a dictionary describing how to feed state into the network for recurrent neural nets.
                                  See common/models.py/lstm for more details on using recurrent nets in policies

env: baselines.common.vec_env.VecEnv     environment. Needs to be vectorized for parallel environment simulation.
                                  The environments produced by gym.make can be wrapped using baselines.common.vec_env.DummyVecEnv class.

nsteps: int                       number of steps of the vectorized environment per update (i.e. batch size is nsteps * nenv where
                                  nenv is number of environment copies simulated in parallel)

total_timesteps: int              number of timesteps (i.e. number of actions taken in the environment)

ent_coef: float                   policy entropy coefficient in the optimization objective

lr: float or function             learning rate, constant or a schedule function [0,1] -> R+ where 1 is beginning of the
                                  training and 0 is the end of the training.

vf_coef: float                    value function loss coefficient in the optimization objective

max_grad_norm: float or None      gradient norm clipping coefficient

gamma: float                      discounting factor

lam: float                        advantage estimation discounting factor (lambda in the paper)

log_interval: int                 number of timesteps between logging events

nminibatches: int                 number of training minibatches per update. For recurrent policies,
                                  should be smaller or equal than number of environments run in parallel.

noptepochs: int                   number of training epochs per update

cliprange: float or function      clipping range, constant or schedule function [0,1] -> R+ where 1 is beginning of the training
                                  and 0 is the end of the training

save_interval: int                number of timesteps between saving events

load_path: str                    path to load the model from

**network_kwargs:                 keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
                                  For instance, 'mlp' network architecture has arguments num_hidden and num_layers.


s = arr.shape<EOL>return arr.swapaxes(<NUM_LIT:0>, <NUM_LIT:1>).reshape(s[<NUM_LIT:0>] * s[<NUM_LIT:1>], *s[<NUM_LIT:2>:])<EOL>


return next(filter(lambda task: task['<STR_LIT>'] == env_id, benchmark['<STR_LIT>']), None)<EOL>

Get a task by env_id. Return None if the benchmark doesn't have the env


if MPI is not None:<EOL><INDENT>nworkers = MPI.COMM_WORLD.Get_size()<EOL>rank = MPI.COMM_WORLD.Get_rank()<EOL><DEDENT>else:<EOL><INDENT>nworkers = <NUM_LIT:1><EOL>rank = <NUM_LIT:0><EOL><DEDENT>cpus_per_worker = <NUM_LIT:1><EOL>U.get_session(config=tf.ConfigProto(<EOL>allow_soft_placement=True,<EOL>inter_op_parallelism_threads=cpus_per_worker,<EOL>intra_op_parallelism_threads=cpus_per_worker<EOL>))<EOL>policy = build_policy(env, network, value_network='<STR_LIT>', **network_kwargs)<EOL>set_global_seeds(seed)<EOL>np.set_printoptions(precision=<NUM_LIT:3>)<EOL>ob_space = env.observation_space<EOL>ac_space = env.action_space<EOL>ob = observation_placeholder(ob_space)<EOL>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>pi = policy(observ_placeholder=ob)<EOL><DEDENT>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>oldpi = policy(observ_placeholder=ob)<EOL><DEDENT>atarg = tf.placeholder(dtype=tf.float32, shape=[None]) <EOL>ret = tf.placeholder(dtype=tf.float32, shape=[None]) <EOL>ac = pi.pdtype.sample_placeholder([None])<EOL>kloldnew = oldpi.pd.kl(pi.pd)<EOL>ent = pi.pd.entropy()<EOL>meankl = tf.reduce_mean(kloldnew)<EOL>meanent = tf.reduce_mean(ent)<EOL>entbonus = ent_coef * meanent<EOL>vferr = tf.reduce_mean(tf.square(pi.vf - ret))<EOL>ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) <EOL>surrgain = tf.reduce_mean(ratio * atarg)<EOL>optimgain = surrgain + entbonus<EOL>losses = [optimgain, meankl, entbonus, surrgain, meanent]<EOL>loss_names = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>dist = meankl<EOL>all_var_list = get_trainable_variables("<STR_LIT>")<EOL>var_list = get_pi_trainable_variables("<STR_LIT>")<EOL>vf_var_list = get_vf_trainable_variables("<STR_LIT>")<EOL>vfadam = MpiAdam(vf_var_list)<EOL>get_flat = U.GetFlat(var_list)<EOL>set_from_flat = U.SetFromFlat(var_list)<EOL>klgrads = tf.gradients(dist, var_list)<EOL>flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="<STR_LIT>")<EOL>shapes = [var.get_shape().as_list() for var in var_list]<EOL>start = <NUM_LIT:0><EOL>tangents = []<EOL>for shape in shapes:<EOL><INDENT>sz = U.intprod(shape)<EOL>tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))<EOL>start += sz<EOL><DEDENT>gvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) <EOL>fvp = U.flatgrad(gvp, var_list)<EOL>assign_old_eq_new = U.function([],[], updates=[tf.assign(oldv, newv)<EOL>for (oldv, newv) in zipsame(get_variables("<STR_LIT>"), get_variables("<STR_LIT>"))])<EOL>compute_losses = U.function([ob, ac, atarg], losses)<EOL>compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])<EOL>compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)<EOL>compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))<EOL>@contextmanager<EOL>def timed(msg):<EOL><INDENT>if rank == <NUM_LIT:0>:<EOL><INDENT>print(colorize(msg, color='<STR_LIT>'))<EOL>tstart = time.time()<EOL>yield<EOL>print(colorize("<STR_LIT>"%(time.time() - tstart), color='<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>yield<EOL><DEDENT><DEDENT>def allmean(x):<EOL><INDENT>assert isinstance(x, np.ndarray)<EOL>if MPI is not None:<EOL><INDENT>out = np.empty_like(x)<EOL>MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)<EOL>out /= nworkers<EOL><DEDENT>else:<EOL><INDENT>out = np.copy(x)<EOL><DEDENT>return out<EOL><DEDENT>U.initialize()<EOL>if load_path is not None:<EOL><INDENT>pi.load(load_path)<EOL><DEDENT>th_init = get_flat()<EOL>if MPI is not None:<EOL><INDENT>MPI.COMM_WORLD.Bcast(th_init, root=<NUM_LIT:0>)<EOL><DEDENT>set_from_flat(th_init)<EOL>vfadam.sync()<EOL>print("<STR_LIT>", th_init.sum(), flush=True)<EOL>seg_gen = traj_segment_generator(pi, env, timesteps_per_batch, stochastic=True)<EOL>episodes_so_far = <NUM_LIT:0><EOL>timesteps_so_far = <NUM_LIT:0><EOL>iters_so_far = <NUM_LIT:0><EOL>tstart = time.time()<EOL>lenbuffer = deque(maxlen=<NUM_LIT>) <EOL>rewbuffer = deque(maxlen=<NUM_LIT>) <EOL>if sum([max_iters><NUM_LIT:0>, total_timesteps><NUM_LIT:0>, max_episodes><NUM_LIT:0>])==<NUM_LIT:0>:<EOL><INDENT>return pi<EOL><DEDENT>assert sum([max_iters><NUM_LIT:0>, total_timesteps><NUM_LIT:0>, max_episodes><NUM_LIT:0>]) < <NUM_LIT:2>,'<STR_LIT>'<EOL>while True:<EOL><INDENT>if callback: callback(locals(), globals())<EOL>if total_timesteps and timesteps_so_far >= total_timesteps:<EOL><INDENT>break<EOL><DEDENT>elif max_episodes and episodes_so_far >= max_episodes:<EOL><INDENT>break<EOL><DEDENT>elif max_iters and iters_so_far >= max_iters:<EOL><INDENT>break<EOL><DEDENT>logger.log("<STR_LIT>"%iters_so_far)<EOL>with timed("<STR_LIT>"):<EOL><INDENT>seg = seg_gen.__next__()<EOL><DEDENT>add_vtarg_and_adv(seg, gamma, lam)<EOL>ob, ac, atarg, tdlamret = seg["<STR_LIT>"], seg["<STR_LIT>"], seg["<STR_LIT>"], seg["<STR_LIT>"]<EOL>vpredbefore = seg["<STR_LIT>"] <EOL>atarg = (atarg - atarg.mean()) / atarg.std() <EOL>if hasattr(pi, "<STR_LIT>"): pi.ret_rms.update(tdlamret)<EOL>if hasattr(pi, "<STR_LIT>"): pi.ob_rms.update(ob) <EOL>args = seg["<STR_LIT>"], seg["<STR_LIT>"], atarg<EOL>fvpargs = [arr[::<NUM_LIT:5>] for arr in args]<EOL>def fisher_vector_product(p):<EOL><INDENT>return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p<EOL><DEDENT>assign_old_eq_new() <EOL>with timed("<STR_LIT>"):<EOL><INDENT>*lossbefore, g = compute_lossandgrad(*args)<EOL><DEDENT>lossbefore = allmean(np.array(lossbefore))<EOL>g = allmean(g)<EOL>if np.allclose(g, <NUM_LIT:0>):<EOL><INDENT>logger.log("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>with timed("<STR_LIT>"):<EOL><INDENT>stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank==<NUM_LIT:0>)<EOL><DEDENT>assert np.isfinite(stepdir).all()<EOL>shs = <NUM_LIT>*stepdir.dot(fisher_vector_product(stepdir))<EOL>lm = np.sqrt(shs / max_kl)<EOL>fullstep = stepdir / lm<EOL>expectedimprove = g.dot(fullstep)<EOL>surrbefore = lossbefore[<NUM_LIT:0>]<EOL>stepsize = <NUM_LIT:1.0><EOL>thbefore = get_flat()<EOL>for _ in range(<NUM_LIT:10>):<EOL><INDENT>thnew = thbefore + fullstep * stepsize<EOL>set_from_flat(thnew)<EOL>meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))<EOL>improve = surr - surrbefore<EOL>logger.log("<STR_LIT>"%(expectedimprove, improve))<EOL>if not np.isfinite(meanlosses).all():<EOL><INDENT>logger.log("<STR_LIT>")<EOL><DEDENT>elif kl > max_kl * <NUM_LIT>:<EOL><INDENT>logger.log("<STR_LIT>")<EOL><DEDENT>elif improve < <NUM_LIT:0>:<EOL><INDENT>logger.log("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>logger.log("<STR_LIT>")<EOL>break<EOL><DEDENT>stepsize *= <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>logger.log("<STR_LIT>")<EOL>set_from_flat(thbefore)<EOL><DEDENT>if nworkers > <NUM_LIT:1> and iters_so_far % <NUM_LIT:20> == <NUM_LIT:0>:<EOL><INDENT>paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum())) <EOL>assert all(np.allclose(ps, paramsums[<NUM_LIT:0>]) for ps in paramsums[<NUM_LIT:1>:])<EOL><DEDENT><DEDENT>for (lossname, lossval) in zip(loss_names, meanlosses):<EOL><INDENT>logger.record_tabular(lossname, lossval)<EOL><DEDENT>with timed("<STR_LIT>"):<EOL><INDENT>for _ in range(vf_iters):<EOL><INDENT>for (mbob, mbret) in dataset.iterbatches((seg["<STR_LIT>"], seg["<STR_LIT>"]),<EOL>include_final_partial_batch=False, batch_size=<NUM_LIT:64>):<EOL><INDENT>g = allmean(compute_vflossandgrad(mbob, mbret))<EOL>vfadam.update(g, vf_stepsize)<EOL><DEDENT><DEDENT><DEDENT>logger.record_tabular("<STR_LIT>", explained_variance(vpredbefore, tdlamret))<EOL>lrlocal = (seg["<STR_LIT>"], seg["<STR_LIT>"]) <EOL>if MPI is not None:<EOL><INDENT>listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) <EOL><DEDENT>else:<EOL><INDENT>listoflrpairs = [lrlocal]<EOL><DEDENT>lens, rews = map(flatten_lists, zip(*listoflrpairs))<EOL>lenbuffer.extend(lens)<EOL>rewbuffer.extend(rews)<EOL>logger.record_tabular("<STR_LIT>", np.mean(lenbuffer))<EOL>logger.record_tabular("<STR_LIT>", np.mean(rewbuffer))<EOL>logger.record_tabular("<STR_LIT>", len(lens))<EOL>episodes_so_far += len(lens)<EOL>timesteps_so_far += sum(lens)<EOL>iters_so_far += <NUM_LIT:1><EOL>logger.record_tabular("<STR_LIT>", episodes_so_far)<EOL>logger.record_tabular("<STR_LIT>", timesteps_so_far)<EOL>logger.record_tabular("<STR_LIT>", time.time() - tstart)<EOL>if rank==<NUM_LIT:0>:<EOL><INDENT>logger.dump_tabular()<EOL><DEDENT><DEDENT>return pi<EOL>

learn a policy function with TRPO algorithm

Parameters:
----------

network                 neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
                        or function that takes input placeholder and returns tuple (output, None) for feedforward nets
                        or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets

env                     environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class

timesteps_per_batch     timesteps per gradient estimation batch

max_kl                  max KL divergence between old policy and new policy ( KL(pi_old || pi) )

ent_coef                coefficient of policy entropy term in the optimization objective

cg_iters                number of iterations of conjugate gradient algorithm

cg_damping              conjugate gradient damping

vf_stepsize             learning rate for adam optimizer used to optimie value function loss

vf_iters                number of iterations of value function optimization iterations per each policy optimization step

total_timesteps           max number of timesteps

max_episodes            max number of episodes

max_iters               maximum number of policy optimization iterations

callback                function to be called with (locals(), globals()) each policy optimization step

load_path               str, path to load the model from (default: None, i.e. no model is loaded)

**network_kwargs        keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network

Returns:
-------

learnt model


new = np.append(seg["<STR_LIT>"], <NUM_LIT:0>) <EOL>vpred = np.append(seg["<STR_LIT>"], seg["<STR_LIT>"])<EOL>T = len(seg["<STR_LIT>"])<EOL>seg["<STR_LIT>"] = gaelam = np.empty(T, '<STR_LIT>')<EOL>rew = seg["<STR_LIT>"]<EOL>lastgaelam = <NUM_LIT:0><EOL>for t in reversed(range(T)):<EOL><INDENT>nonterminal = <NUM_LIT:1>-new[t+<NUM_LIT:1>]<EOL>delta = rew[t] + gamma * vpred[t+<NUM_LIT:1>] * nonterminal - vpred[t]<EOL>gaelam[t] = lastgaelam = delta + gamma * lam * nonterminal * lastgaelam<EOL><DEDENT>seg["<STR_LIT>"] = seg["<STR_LIT>"] + seg["<STR_LIT>"]<EOL>

Compute target value using TD(lambda) estimator, and advantage with GAE(lambda)


def parse(v):<EOL><INDENT>assert isinstance(v, str)<EOL>try:<EOL><INDENT>return eval(v)<EOL><DEDENT>except (NameError, SyntaxError):<EOL><INDENT>return v<EOL><DEDENT><DEDENT>return {k: parse(v) for k,v in parse_unknown_args(args).items()}<EOL>

convert a list of '='-spaced command-line arguments to a dictionary, evaluating python objects when possible


def q_retrace(R, D, q_i, v, rho_i, nenvs, nsteps, gamma):

rho_bar = batch_to_seq(tf.minimum(<NUM_LIT:1.0>, rho_i), nenvs, nsteps, True)  <EOL>rs = batch_to_seq(R, nenvs, nsteps, True)  <EOL>ds = batch_to_seq(D, nenvs, nsteps, True)  <EOL>q_is = batch_to_seq(q_i, nenvs, nsteps, True)<EOL>vs = batch_to_seq(v, nenvs, nsteps + <NUM_LIT:1>, True)<EOL>v_final = vs[-<NUM_LIT:1>]<EOL>qret = v_final<EOL>qrets = []<EOL>for i in range(nsteps - <NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>check_shape([qret, ds[i], rs[i], rho_bar[i], q_is[i], vs[i]], [[nenvs]] * <NUM_LIT:6>)<EOL>qret = rs[i] + gamma * qret * (<NUM_LIT:1.0> - ds[i])<EOL>qrets.append(qret)<EOL>qret = (rho_bar[i] * (qret - q_is[i])) + vs[i]<EOL><DEDENT>qrets = qrets[::-<NUM_LIT:1>]<EOL>qret = seq_to_batch(qrets, flat=True)<EOL>return qret<EOL>

Calculates q_retrace targets

:param R: Rewards
:param D: Dones
:param q_i: Q values for actions taken
:param v: V values
:param rho_i: Importance weight for each action
:return: Q_retrace values


def learn(network, env, seed=None, nsteps=<NUM_LIT:20>, total_timesteps=int(<NUM_LIT>), q_coef=<NUM_LIT:0.5>, ent_coef=<NUM_LIT>,<EOL>max_grad_norm=<NUM_LIT:10>, lr=<NUM_LIT>, lrschedule='<STR_LIT>', rprop_epsilon=<NUM_LIT>, rprop_alpha=<NUM_LIT>, gamma=<NUM_LIT>,<EOL>log_interval=<NUM_LIT:100>, buffer_size=<NUM_LIT>, replay_ratio=<NUM_LIT:4>, replay_start=<NUM_LIT>, c=<NUM_LIT>,<EOL>trust_region=True, alpha=<NUM_LIT>, delta=<NUM_LIT:1>, load_path=None, **network_kwargs):

print("<STR_LIT>")<EOL>print(locals())<EOL>set_global_seeds(seed)<EOL>if not isinstance(env, VecFrameStack):<EOL><INDENT>env = VecFrameStack(env, <NUM_LIT:1>)<EOL><DEDENT>policy = build_policy(env, network, estimate_q=True, **network_kwargs)<EOL>nenvs = env.num_envs<EOL>ob_space = env.observation_space<EOL>ac_space = env.action_space<EOL>nstack = env.nstack<EOL>model = Model(policy=policy, ob_space=ob_space, ac_space=ac_space, nenvs=nenvs, nsteps=nsteps,<EOL>ent_coef=ent_coef, q_coef=q_coef, gamma=gamma,<EOL>max_grad_norm=max_grad_norm, lr=lr, rprop_alpha=rprop_alpha, rprop_epsilon=rprop_epsilon,<EOL>total_timesteps=total_timesteps, lrschedule=lrschedule, c=c,<EOL>trust_region=trust_region, alpha=alpha, delta=delta)<EOL>runner = Runner(env=env, model=model, nsteps=nsteps)<EOL>if replay_ratio > <NUM_LIT:0>:<EOL><INDENT>buffer = Buffer(env=env, nsteps=nsteps, size=buffer_size)<EOL><DEDENT>else:<EOL><INDENT>buffer = None<EOL><DEDENT>nbatch = nenvs*nsteps<EOL>acer = Acer(runner, model, buffer, log_interval)<EOL>acer.tstart = time.time()<EOL>for acer.steps in range(<NUM_LIT:0>, total_timesteps, nbatch): <EOL><INDENT>acer.call(on_policy=True)<EOL>if replay_ratio > <NUM_LIT:0> and buffer.has_atleast(replay_start):<EOL><INDENT>n = np.random.poisson(replay_ratio)<EOL>for _ in range(n):<EOL><INDENT>acer.call(on_policy=False)  <EOL><DEDENT><DEDENT><DEDENT>return model<EOL>

Main entrypoint for ACER (Actor-Critic with Experience Replay) algorithm (https://arxiv.org/pdf/1611.01224.pdf)
Train an agent with given network architecture on a given environment using ACER.

Parameters:
----------

network:            policy network architecture. Either string (mlp, lstm, lnlstm, cnn_lstm, cnn, cnn_small, conv_only - see baselines.common/models.py for full list)
                    specifying the standard network architecture, or a function that takes tensorflow tensor as input and returns
                    tuple (output_tensor, extra_feed) where output tensor is the last network layer output, extra_feed is None for feed-forward
                    neural nets, and extra_feed is a dictionary describing how to feed state into the network for recurrent neural nets.
                    See baselines.common/policies.py/lstm for more details on using recurrent nets in policies

env:                environment. Needs to be vectorized for parallel environment simulation.
                    The environments produced by gym.make can be wrapped using baselines.common.vec_env.DummyVecEnv class.

nsteps:             int, number of steps of the vectorized environment per update (i.e. batch size is nsteps * nenv where
                    nenv is number of environment copies simulated in parallel) (default: 20)

nstack:             int, size of the frame stack, i.e. number of the frames passed to the step model. Frames are stacked along channel dimension
                    (last image dimension) (default: 4)

total_timesteps:    int, number of timesteps (i.e. number of actions taken in the environment) (default: 80M)

q_coef:             float, value function loss coefficient in the optimization objective (analog of vf_coef for other actor-critic methods)

ent_coef:           float, policy entropy coefficient in the optimization objective (default: 0.01)

max_grad_norm:      float, gradient norm clipping coefficient. If set to None, no clipping. (default: 10),

lr:                 float, learning rate for RMSProp (current implementation has RMSProp hardcoded in) (default: 7e-4)

lrschedule:         schedule of learning rate. Can be 'linear', 'constant', or a function [0..1] -> [0..1] that takes fraction of the training progress as input and
                    returns fraction of the learning rate (specified as lr) as output

rprop_epsilon:      float, RMSProp epsilon (stabilizes square root computation in denominator of RMSProp update) (default: 1e-5)

rprop_alpha:        float, RMSProp decay parameter (default: 0.99)

gamma:              float, reward discounting factor (default: 0.99)

log_interval:       int, number of updates between logging events (default: 100)

buffer_size:        int, size of the replay buffer (default: 50k)

replay_ratio:       int, now many (on average) batches of data to sample from the replay buffer take after batch from the environment (default: 4)

replay_start:       int, the sampling from the replay buffer does not start until replay buffer has at least that many samples (default: 10k)

c:                  float, importance weight clipping factor (default: 10)

trust_region        bool, whether or not algorithms estimates the gradient KL divergence between the old and updated policy and uses it to determine step size  (default: True)

delta:              float, max KL divergence between the old policy and updated policy (default: 1)

alpha:              float, momentum factor in the Polyak (exponential moving average) averaging of the model parameters (default: 0.99)

load_path:          str, path to load the model from (default: None)

**network_kwargs:               keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
                                For instance, 'mlp' network architecture has arguments num_hidden and num_layers.


set_global_seeds(seed)<EOL>nenvs = env.num_envs<EOL>policy = build_policy(env, network, **network_kwargs)<EOL>model = Model(policy=policy, env=env, nsteps=nsteps, ent_coef=ent_coef, vf_coef=vf_coef,<EOL>max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule)<EOL>if load_path is not None:<EOL><INDENT>model.load(load_path)<EOL><DEDENT>runner = Runner(env, model, nsteps=nsteps, gamma=gamma)<EOL>epinfobuf = deque(maxlen=<NUM_LIT:100>)<EOL>nbatch = nenvs*nsteps<EOL>tstart = time.time()<EOL>for update in range(<NUM_LIT:1>, total_timesteps//nbatch+<NUM_LIT:1>):<EOL><INDENT>obs, states, rewards, masks, actions, values, epinfos = runner.run()<EOL>epinfobuf.extend(epinfos)<EOL>policy_loss, value_loss, policy_entropy = model.train(obs, states, rewards, masks, actions, values)<EOL>nseconds = time.time()-tstart<EOL>fps = int((update*nbatch)/nseconds)<EOL>if update % log_interval == <NUM_LIT:0> or update == <NUM_LIT:1>:<EOL><INDENT>ev = explained_variance(values, rewards)<EOL>logger.record_tabular("<STR_LIT>", update)<EOL>logger.record_tabular("<STR_LIT>", update*nbatch)<EOL>logger.record_tabular("<STR_LIT>", fps)<EOL>logger.record_tabular("<STR_LIT>", float(policy_entropy))<EOL>logger.record_tabular("<STR_LIT>", float(value_loss))<EOL>logger.record_tabular("<STR_LIT>", float(ev))<EOL>logger.record_tabular("<STR_LIT>", safemean([epinfo['<STR_LIT:r>'] for epinfo in epinfobuf]))<EOL>logger.record_tabular("<STR_LIT>", safemean([epinfo['<STR_LIT:l>'] for epinfo in epinfobuf]))<EOL>logger.dump_tabular()<EOL><DEDENT><DEDENT>return model<EOL>

Main entrypoint for A2C algorithm. Train a policy with given network architecture on a given environment using a2c algorithm.

Parameters:
-----------

network:            policy network architecture. Either string (mlp, lstm, lnlstm, cnn_lstm, cnn, cnn_small, conv_only - see baselines.common/models.py for full list)
                    specifying the standard network architecture, or a function that takes tensorflow tensor as input and returns
                    tuple (output_tensor, extra_feed) where output tensor is the last network layer output, extra_feed is None for feed-forward
                    neural nets, and extra_feed is a dictionary describing how to feed state into the network for recurrent neural nets.
                    See baselines.common/policies.py/lstm for more details on using recurrent nets in policies

env:                RL environment. Should implement interface similar to VecEnv (baselines.common/vec_env) or be wrapped with DummyVecEnv (baselines.common/vec_env/dummy_vec_env.py)

seed:               seed to make random number sequence in the alorightm reproducible. By default is None which means seed from system noise generator (not reproducible)

nsteps:             int, number of steps of the vectorized environment per update (i.e. batch size is nsteps * nenv where
                    nenv is number of environment copies simulated in parallel)

total_timesteps:    int, total number of timesteps to train on (default: 80M)

vf_coef:            float, coefficient in front of value function loss in the total loss function (default: 0.5)

ent_coef:           float, coeffictiant in front of the policy entropy in the total loss function (default: 0.01)

max_gradient_norm:  float, gradient is clipped to have global L2 norm no more than this value (default: 0.5)

lr:                 float, learning rate for RMSProp (current implementation has RMSProp hardcoded in) (default: 7e-4)

lrschedule:         schedule of learning rate. Can be 'linear', 'constant', or a function [0..1] -> [0..1] that takes fraction of the training progress as input and
                    returns fraction of the learning rate (specified as lr) as output

epsilon:            float, RMSProp epsilon (stabilizes square root computation in denominator of RMSProp update) (default: 1e-5)

alpha:              float, RMSProp decay parameter (default: 0.99)

gamma:              float, reward discounting parameter (default: 0.99)

log_interval:       int, specifies how frequently the logs are printed out (default: 100)

**network_kwargs:   keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
                    For instance, 'mlp' network architecture has arguments num_hidden and num_layers.


def wrap_deepmind(env, episode_life=True, clip_rewards=True, frame_stack=False, scale=False):

if episode_life:<EOL><INDENT>env = EpisodicLifeEnv(env)<EOL><DEDENT>if '<STR_LIT>' in env.unwrapped.get_action_meanings():<EOL><INDENT>env = FireResetEnv(env)<EOL><DEDENT>env = WarpFrame(env)<EOL>if scale:<EOL><INDENT>env = ScaledFloatFrame(env)<EOL><DEDENT>if clip_rewards:<EOL><INDENT>env = ClipRewardEnv(env)<EOL><DEDENT>if frame_stack:<EOL><INDENT>env = FrameStack(env, <NUM_LIT:4>)<EOL><DEDENT>return env<EOL>

Configure environment for DeepMind-style Atari.


def __init__(self, env, noop_max=<NUM_LIT:30>):

gym.Wrapper.__init__(self, env)<EOL>self.noop_max = noop_max<EOL>self.override_num_noops = None<EOL>self.noop_action = <NUM_LIT:0><EOL>assert env.unwrapped.get_action_meanings()[<NUM_LIT:0>] == '<STR_LIT>'<EOL>

Sample initial states by taking random number of no-ops on reset.
        No-op is assumed to be action 0.


self.env.reset(**kwargs)<EOL>if self.override_num_noops is not None:<EOL><INDENT>noops = self.override_num_noops<EOL><DEDENT>else:<EOL><INDENT>noops = self.unwrapped.np_random.randint(<NUM_LIT:1>, self.noop_max + <NUM_LIT:1>) <EOL><DEDENT>assert noops > <NUM_LIT:0><EOL>obs = None<EOL>for _ in range(noops):<EOL><INDENT>obs, _, done, _ = self.env.step(self.noop_action)<EOL>if done:<EOL><INDENT>obs = self.env.reset(**kwargs)<EOL><DEDENT><DEDENT>return obs<EOL>

Do no-op action for a number of steps in [1, noop_max].


gym.Wrapper.__init__(self, env)<EOL>assert env.unwrapped.get_action_meanings()[<NUM_LIT:1>] == '<STR_LIT>'<EOL>assert len(env.unwrapped.get_action_meanings()) >= <NUM_LIT:3><EOL>

Take action on reset for environments that are fixed until firing.


gym.Wrapper.__init__(self, env)<EOL>self.lives = <NUM_LIT:0><EOL>self.was_real_done  = True<EOL>

Make end-of-life == end-of-episode, but only reset on true game over.
        Done by DeepMind for the DQN and co. since it helps value estimation.


if self.was_real_done:<EOL><INDENT>obs = self.env.reset(**kwargs)<EOL><DEDENT>else:<EOL><INDENT>obs, _, _, _ = self.env.step(<NUM_LIT:0>)<EOL><DEDENT>self.lives = self.env.unwrapped.ale.lives()<EOL>return obs<EOL>

Reset only when lives are exhausted.
        This way all states are still reachable even though lives are episodic,
        and the learner need not know about any of this behind-the-scenes.


def __init__(self, env, skip=<NUM_LIT:4>):

gym.Wrapper.__init__(self, env)<EOL>self._obs_buffer = np.zeros((<NUM_LIT:2>,)+env.observation_space.shape, dtype=np.uint8)<EOL>self._skip       = skip<EOL>


total_reward = <NUM_LIT:0.0><EOL>done = None<EOL>for i in range(self._skip):<EOL><INDENT>obs, reward, done, info = self.env.step(action)<EOL>if i == self._skip - <NUM_LIT:2>: self._obs_buffer[<NUM_LIT:0>] = obs<EOL>if i == self._skip - <NUM_LIT:1>: self._obs_buffer[<NUM_LIT:1>] = obs<EOL>total_reward += reward<EOL>if done:<EOL><INDENT>break<EOL><DEDENT><DEDENT>max_frame = self._obs_buffer.max(axis=<NUM_LIT:0>)<EOL>return max_frame, total_reward, done, info<EOL>

Repeat action, sum reward, and max over last observations.


return np.sign(reward)<EOL>


def __init__(self, env, width=<NUM_LIT>, height=<NUM_LIT>, grayscale=True):

gym.ObservationWrapper.__init__(self, env)<EOL>self.width = width<EOL>self.height = height<EOL>self.grayscale = grayscale<EOL>if self.grayscale:<EOL><INDENT>self.observation_space = spaces.Box(low=<NUM_LIT:0>, high=<NUM_LIT:255>,<EOL>shape=(self.height, self.width, <NUM_LIT:1>), dtype=np.uint8)<EOL><DEDENT>else:<EOL><INDENT>self.observation_space = spaces.Box(low=<NUM_LIT:0>, high=<NUM_LIT:255>,<EOL>shape=(self.height, self.width, <NUM_LIT:3>), dtype=np.uint8)<EOL><DEDENT>

Warp frames to 84x84 as done in the Nature paper and later work.


gym.Wrapper.__init__(self, env)<EOL>self.k = k<EOL>self.frames = deque([], maxlen=k)<EOL>shp = env.observation_space.shape<EOL>self.observation_space = spaces.Box(low=<NUM_LIT:0>, high=<NUM_LIT:255>, shape=(shp[:-<NUM_LIT:1>] + (shp[-<NUM_LIT:1>] * k,)), dtype=env.observation_space.dtype)<EOL>

Stack k last frames.

        Returns lazy array, which is much more memory efficient.

        See Also
        --------
        baselines.common.atari_wrappers.LazyFrames


self._frames = frames<EOL>self._out = None<EOL>

This object ensures that common frames between the observations are only stored once.
        It exists purely to optimize memory usage which can be huge for DQN's 1M frames replay
        buffers.

        This object should only be converted to numpy array before being passed to the model.

        You'd not believe how complex the previous solution was.


assert x.ndim >= <NUM_LIT:1><EOL>return scipy.signal.lfilter([<NUM_LIT:1>],[<NUM_LIT:1>,-gamma],x[::-<NUM_LIT:1>], axis=<NUM_LIT:0>)[::-<NUM_LIT:1>]<EOL>

computes discounted sums along 0th dimension of x.

inputs
------
x: ndarray
gamma: float

outputs
-------
y: ndarray with same shape as x, satisfying

    y[t] = x[t] + gamma*x[t+1] + gamma^2*x[t+2] + ... + gamma^k x[t+k],
            where k = len(x) - t - 1


assert y.ndim == <NUM_LIT:1> and ypred.ndim == <NUM_LIT:1><EOL>vary = np.var(y)<EOL>return np.nan if vary==<NUM_LIT:0> else <NUM_LIT:1> - np.var(y-ypred)/vary<EOL>

Computes fraction of variance that ypred explains about y.
Returns 1 - Var[y-ypred] / Var[y]

interpretation:
    ev=0  =>  might as well have predicted zero
    ev=1  =>  perfect prediction
    ev<0  =>  worse than just predicting zero


def discount_with_boundaries(X, New, gamma):

Y = np.zeros_like(X)<EOL>T = X.shape[<NUM_LIT:0>]<EOL>Y[T-<NUM_LIT:1>] = X[T-<NUM_LIT:1>]<EOL>for t in range(T-<NUM_LIT:2>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>Y[t] = X[t] + gamma * Y[t+<NUM_LIT:1>] * (<NUM_LIT:1> - New[t+<NUM_LIT:1>])<EOL><DEDENT>return Y<EOL>

X: 2d array of floats, time x features
New: 2d array of bools, indicating when a new episode has started


wrapper_kwargs = wrapper_kwargs or {}<EOL>mpi_rank = MPI.COMM_WORLD.Get_rank() if MPI else <NUM_LIT:0><EOL>seed = seed + <NUM_LIT> * mpi_rank if seed is not None else None<EOL>logger_dir = logger.get_dir()<EOL>def make_thunk(rank):<EOL><INDENT>return lambda: make_env(<EOL>env_id=env_id,<EOL>env_type=env_type,<EOL>mpi_rank=mpi_rank,<EOL>subrank=rank,<EOL>seed=seed,<EOL>reward_scale=reward_scale,<EOL>gamestate=gamestate,<EOL>flatten_dict_observations=flatten_dict_observations,<EOL>wrapper_kwargs=wrapper_kwargs,<EOL>logger_dir=logger_dir<EOL>)<EOL><DEDENT>set_global_seeds(seed)<EOL>if num_env > <NUM_LIT:1>:<EOL><INDENT>return SubprocVecEnv([make_thunk(i + start_index) for i in range(num_env)])<EOL><DEDENT>else:<EOL><INDENT>return DummyVecEnv([make_thunk(start_index)])<EOL><DEDENT>

Create a wrapped, monitored SubprocVecEnv for Atari and MuJoCo.


def make_mujoco_env(env_id, seed, reward_scale=<NUM_LIT:1.0>):

rank = MPI.COMM_WORLD.Get_rank()<EOL>myseed = seed  + <NUM_LIT:1000> * rank if seed is not None else None<EOL>set_global_seeds(myseed)<EOL>env = gym.make(env_id)<EOL>logger_path = None if logger.get_dir() is None else os.path.join(logger.get_dir(), str(rank))<EOL>env = Monitor(env, logger_path, allow_early_resets=True)<EOL>env.seed(seed)<EOL>if reward_scale != <NUM_LIT:1.0>:<EOL><INDENT>from baselines.common.retro_wrappers import RewardScaler<EOL>env = RewardScaler(env, reward_scale)<EOL><DEDENT>return env<EOL>

Create a wrapped, monitored gym.Env for MuJoCo.


def make_robotics_env(env_id, seed, rank=<NUM_LIT:0>):

set_global_seeds(seed)<EOL>env = gym.make(env_id)<EOL>env = FlattenDictWrapper(env, ['<STR_LIT>', '<STR_LIT>'])<EOL>env = Monitor(<EOL>env, logger.get_dir() and os.path.join(logger.get_dir(), str(rank)),<EOL>info_keywords=('<STR_LIT>',))<EOL>env.seed(seed)<EOL>return env<EOL>


import argparse<EOL>return argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)<EOL>

Create an empty argparse.ArgumentParser.


print('<STR_LIT>')<EOL>return common_arg_parser()<EOL>

Create an argparse.ArgumentParser for run_atari.py.


parser = arg_parser()<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', type=str, default='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', type=str)<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', type=int, default=None)<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', type=str, default='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', type=float, default=<NUM_LIT>),<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', default=None)<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', default=None)<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', default=None, type=int)<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', default=<NUM_LIT:1.0>, type=float)<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', default=None, type=str)<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', default=<NUM_LIT:0>, type=int)<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', default=<NUM_LIT:200>, type=int)<EOL>parser.add_argument('<STR_LIT>', default=False, action='<STR_LIT:store_true>')<EOL>return parser<EOL>

Create an argparse.ArgumentParser for run_mujoco.py.


parser = arg_parser()<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', type=str, default='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', help='<STR_LIT>', type=int, default=None)<EOL>parser.add_argument('<STR_LIT>', type=int, default=int(<NUM_LIT>))<EOL>return parser<EOL>


retval = {}<EOL>preceded_by_key = False<EOL>for arg in args:<EOL><INDENT>if arg.startswith('<STR_LIT>'):<EOL><INDENT>if '<STR_LIT:=>' in arg:<EOL><INDENT>key = arg.split('<STR_LIT:=>')[<NUM_LIT:0>][<NUM_LIT:2>:]<EOL>value = arg.split('<STR_LIT:=>')[<NUM_LIT:1>]<EOL>retval[key] = value<EOL><DEDENT>else:<EOL><INDENT>key = arg[<NUM_LIT:2>:]<EOL>preceded_by_key = True<EOL><DEDENT><DEDENT>elif preceded_by_key:<EOL><INDENT>retval[key] = arg<EOL>preceded_by_key = False<EOL><DEDENT><DEDENT>return retval<EOL>

Parse arguments not consumed by arg parser into a dicitonary


return {k: np.copy(v) for k, v in obs.items()}<EOL>


if set(obs_dict.keys()) == {None}:<EOL><INDENT>return obs_dict[None]<EOL><DEDENT>return obs_dict<EOL>

Convert an observation dict into a raw array if the
original observation space was not a Dict space.


if isinstance(obs_space, gym.spaces.Dict):<EOL><INDENT>assert isinstance(obs_space.spaces, OrderedDict)<EOL>subspaces = obs_space.spaces<EOL><DEDENT>else:<EOL><INDENT>subspaces = {None: obs_space}<EOL><DEDENT>keys = []<EOL>shapes = {}<EOL>dtypes = {}<EOL>for key, box in subspaces.items():<EOL><INDENT>keys.append(key)<EOL>shapes[key] = box.shape<EOL>dtypes[key] = box.dtype<EOL><DEDENT>return keys, shapes, dtypes<EOL>

Get dict-structured information about a gym.Space.

Returns:
  A tuple (keys, shapes, dtypes):
    keys: a list of dict keys.
    shapes: a dict mapping keys to shapes.
    dtypes: a dict mapping keys to dtypes.


if isinstance(obs, dict):<EOL><INDENT>return obs<EOL><DEDENT>return {None: obs}<EOL>


def __init__(self, env_fns, spaces=None, context='<STR_LIT>'):

self.waiting = False<EOL>self.closed = False<EOL>nenvs = len(env_fns)<EOL>ctx = mp.get_context(context)<EOL>self.remotes, self.work_remotes = zip(*[ctx.Pipe() for _ in range(nenvs)])<EOL>self.ps = [ctx.Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))<EOL>for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]<EOL>for p in self.ps:<EOL><INDENT>p.daemon = True  <EOL>with clear_mpi_env_vars():<EOL><INDENT>p.start()<EOL><DEDENT><DEDENT>for remote in self.work_remotes:<EOL><INDENT>remote.close()<EOL><DEDENT>self.remotes[<NUM_LIT:0>].send(('<STR_LIT>', None))<EOL>observation_space, action_space, self.spec = self.remotes[<NUM_LIT:0>].recv()<EOL>self.viewer = None<EOL>VecEnv.__init__(self, len(env_fns), observation_space, action_space)<EOL>

Arguments:

env_fns: iterable of callables -  functions that create environments to run in subprocesses. Need to be cloud-pickleable


def _subproc_worker(pipe, parent_pipe, env_fn_wrapper, obs_bufs, obs_shapes, obs_dtypes, keys):

def _write_obs(maybe_dict_obs):<EOL><INDENT>flatdict = obs_to_dict(maybe_dict_obs)<EOL>for k in keys:<EOL><INDENT>dst = obs_bufs[k].get_obj()<EOL>dst_np = np.frombuffer(dst, dtype=obs_dtypes[k]).reshape(obs_shapes[k])  <EOL>np.copyto(dst_np, flatdict[k])<EOL><DEDENT><DEDENT>env = env_fn_wrapper.x()<EOL>parent_pipe.close()<EOL>try:<EOL><INDENT>while True:<EOL><INDENT>cmd, data = pipe.recv()<EOL>if cmd == '<STR_LIT>':<EOL><INDENT>pipe.send(_write_obs(env.reset()))<EOL><DEDENT>elif cmd == '<STR_LIT>':<EOL><INDENT>obs, reward, done, info = env.step(data)<EOL>if done:<EOL><INDENT>obs = env.reset()<EOL><DEDENT>pipe.send((_write_obs(obs), reward, done, info))<EOL><DEDENT>elif cmd == '<STR_LIT>':<EOL><INDENT>pipe.send(env.render(mode='<STR_LIT>'))<EOL><DEDENT>elif cmd == '<STR_LIT>':<EOL><INDENT>pipe.send(None)<EOL>break<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>' % cmd)<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>finally:<EOL><INDENT>env.close()<EOL><DEDENT>

Control a single environment instance using IPC and
shared memory.


ctx = mp.get_context(context)<EOL>if spaces:<EOL><INDENT>observation_space, action_space = spaces<EOL><DEDENT>else:<EOL><INDENT>logger.log('<STR_LIT>')<EOL>with logger.scoped_configure(format_strs=[]):<EOL><INDENT>dummy = env_fns[<NUM_LIT:0>]()<EOL>observation_space, action_space = dummy.observation_space, dummy.action_space<EOL>dummy.close()<EOL>del dummy<EOL><DEDENT><DEDENT>VecEnv.__init__(self, len(env_fns), observation_space, action_space)<EOL>self.obs_keys, self.obs_shapes, self.obs_dtypes = obs_space_info(observation_space)<EOL>self.obs_bufs = [<EOL>{k: ctx.Array(_NP_TO_CT[self.obs_dtypes[k].type], int(np.prod(self.obs_shapes[k]))) for k in self.obs_keys}<EOL>for _ in env_fns]<EOL>self.parent_pipes = []<EOL>self.procs = []<EOL>with clear_mpi_env_vars():<EOL><INDENT>for env_fn, obs_buf in zip(env_fns, self.obs_bufs):<EOL><INDENT>wrapped_fn = CloudpickleWrapper(env_fn)<EOL>parent_pipe, child_pipe = ctx.Pipe()<EOL>proc = ctx.Process(target=_subproc_worker,<EOL>args=(child_pipe, parent_pipe, wrapped_fn, obs_buf, self.obs_shapes, self.obs_dtypes, self.obs_keys))<EOL>proc.daemon = True<EOL>self.procs.append(proc)<EOL>self.parent_pipes.append(parent_pipe)<EOL>proc.start()<EOL>child_pipe.close()<EOL><DEDENT><DEDENT>self.waiting_step = False<EOL>self.viewer = None<EOL>

If you don't specify observation_space, we'll have to create a dummy
environment to get it.


def assert_venvs_equal(venv1, venv2, num_steps):

assert venv1.num_envs == venv2.num_envs<EOL>assert venv1.observation_space.shape == venv2.observation_space.shape<EOL>assert venv1.observation_space.dtype == venv2.observation_space.dtype<EOL>assert venv1.action_space.shape == venv2.action_space.shape<EOL>assert venv1.action_space.dtype == venv2.action_space.dtype<EOL>try:<EOL><INDENT>obs1, obs2 = venv1.reset(), venv2.reset()<EOL>assert np.array(obs1).shape == np.array(obs2).shape<EOL>assert np.array(obs1).shape == (venv1.num_envs,) + venv1.observation_space.shape<EOL>assert np.allclose(obs1, obs2)<EOL>venv1.action_space.seed(<NUM_LIT>)<EOL>for _ in range(num_steps):<EOL><INDENT>actions = np.array([venv1.action_space.sample() for _ in range(venv1.num_envs)])<EOL>for venv in [venv1, venv2]:<EOL><INDENT>venv.step_async(actions)<EOL><DEDENT>outs1 = venv1.step_wait()<EOL>outs2 = venv2.step_wait()<EOL>for out1, out2 in zip(outs1[:<NUM_LIT:3>], outs2[:<NUM_LIT:3>]):<EOL><INDENT>assert np.array(out1).shape == np.array(out2).shape<EOL>assert np.allclose(out1, out2)<EOL><DEDENT>assert list(outs1[<NUM_LIT:3>]) == list(outs2[<NUM_LIT:3>])<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>venv1.close()<EOL>venv2.close()<EOL><DEDENT>

Compare two environments over num_steps steps and make sure
that the observations produced by each are the same when given
the same actions.


self.envs = [fn() for fn in env_fns]<EOL>env = self.envs[<NUM_LIT:0>]<EOL>VecEnv.__init__(self, len(env_fns), env.observation_space, env.action_space)<EOL>obs_space = env.observation_space<EOL>self.keys, shapes, dtypes = obs_space_info(obs_space)<EOL>self.buf_obs = { k: np.zeros((self.num_envs,) + tuple(shapes[k]), dtype=dtypes[k]) for k in self.keys }<EOL>self.buf_dones = np.zeros((self.num_envs,), dtype=np.bool)<EOL>self.buf_rews  = np.zeros((self.num_envs,), dtype=np.float32)<EOL>self.buf_infos = [{} for _ in range(self.num_envs)]<EOL>self.actions = None<EOL>self.spec = self.envs[<NUM_LIT:0>].spec<EOL>

Arguments:

env_fns: iterable of callables      functions that build environments


def __init__(self, venv, directory, record_video_trigger, video_length=<NUM_LIT:200>):

VecEnvWrapper.__init__(self, venv)<EOL>self.record_video_trigger = record_video_trigger<EOL>self.video_recorder = None<EOL>self.directory = os.path.abspath(directory)<EOL>if not os.path.exists(self.directory): os.mkdir(self.directory)<EOL>self.file_prefix = "<STR_LIT>"<EOL>self.file_infix = '<STR_LIT:{}>'.format(os.getpid())<EOL>self.step_id = <NUM_LIT:0><EOL>self.video_length = video_length<EOL>self.recording = False<EOL>self.recorded_frames = <NUM_LIT:0><EOL>

# Arguments
    venv: VecEnv to wrap
    directory: Where to save videos
    record_video_trigger:
        Function that defines when to start recording.
        The function takes the current number of step,
        and returns whether we should start recording or not.
    video_length: Length of recorded video


removed_environment = {}<EOL>for k, v in list(os.environ.items()):<EOL><INDENT>for prefix in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if k.startswith(prefix):<EOL><INDENT>removed_environment[k] = v<EOL>del os.environ[k]<EOL><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>yield<EOL><DEDENT>finally:<EOL><INDENT>os.environ.update(removed_environment)<EOL><DEDENT>

from mpi4py import MPI will call MPI_Init by default.  If the child process has MPI environment variables, MPI will think that the child process is an MPI process just like the parent and do bad things such as hang.
This context manager is a hacky way to clear those environment variables temporarily such as when we are starting multiprocessing
Processes.


pass<EOL>

Reset all the environments and return an array of
observations, or a dict of observation arrays.

If step_async is still doing work, that work will
be cancelled and step_wait() should not be called
until step_async() is invoked again.


pass<EOL>

Tell all the environments to start taking a step
with the given actions.
Call step_wait() to get the results of the step.

You should not call this if a step_async run is
already pending.


pass<EOL>

Wait for the step taken with step_async().

Returns (obs, rews, dones, infos):
 - obs: an array of observations, or a dict of
        arrays of observations.
 - rews: an array of rewards
 - dones: an array of "episode done" booleans
 - infos: a sequence of info objects


pass<EOL>

Clean up the  extra resources, beyond what's in this base class.
Only runs when not self.closed.


self.step_async(actions)<EOL>return self.step_wait()<EOL>

Step the environments synchronously.

This is available for backwards compatibility.


raise NotImplementedError<EOL>


def observation_placeholder(ob_space, batch_size=None, name='<STR_LIT>'):

assert isinstance(ob_space, Discrete) or isinstance(ob_space, Box) or isinstance(ob_space, MultiDiscrete),'<STR_LIT>'<EOL>dtype = ob_space.dtype<EOL>if dtype == np.int8:<EOL><INDENT>dtype = np.uint8<EOL><DEDENT>return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=dtype, name=name)<EOL>

Create placeholder to feed observations into of the size appropriate to the observation space

Parameters:
----------

ob_space: gym.Space     observation space

batch_size: int         size of the batch to be fed into input. Can be left None in most cases.

name: str               name of the placeholder

Returns:
-------

tensorflow placeholder tensor


def observation_input(ob_space, batch_size=None, name='<STR_LIT>'):

placeholder = observation_placeholder(ob_space, batch_size, name)<EOL>return placeholder, encode_observation(ob_space, placeholder)<EOL>

Create placeholder to feed observations into of the size appropriate to the observation space, and add input
encoder of the appropriate type.


if isinstance(ob_space, Discrete):<EOL><INDENT>return tf.to_float(tf.one_hot(placeholder, ob_space.n))<EOL><DEDENT>elif isinstance(ob_space, Box):<EOL><INDENT>return tf.to_float(placeholder)<EOL><DEDENT>elif isinstance(ob_space, MultiDiscrete):<EOL><INDENT>placeholder = tf.cast(placeholder, tf.int32)<EOL>one_hots = [tf.to_float(tf.one_hot(placeholder[..., i], ob_space.nvec[i])) for i in range(placeholder.shape[-<NUM_LIT:1>])]<EOL>return tf.concat(one_hots, axis=-<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError<EOL><DEDENT>

Encode input in the way that is appropriate to the observation space

Parameters:
----------

ob_space: gym.Space             observation space

placeholder: tf.placeholder     observation input placeholder


def __init__(self, capacity, operation, neutral_element):

assert capacity > <NUM_LIT:0> and capacity & (capacity - <NUM_LIT:1>) == <NUM_LIT:0>, "<STR_LIT>"<EOL>self._capacity = capacity<EOL>self._value = [neutral_element for _ in range(<NUM_LIT:2> * capacity)]<EOL>self._operation = operation<EOL>

Build a Segment Tree data structure.

        https://en.wikipedia.org/wiki/Segment_tree

        Can be used as regular array, but with two
        important differences:

            a) setting item's value is slightly slower.
               It is O(lg capacity) instead of O(1).
            b) user has access to an efficient ( O(log segment size) )
               `reduce` operation which reduces `operation` over
               a contiguous subsequence of items in the array.

        Paramters
        ---------
        capacity: int
            Total size of the array - must be a power of two.
        operation: lambda obj, obj -> obj
            and operation for combining elements (eg. sum, max)
            must form a mathematical group together with the set of
            possible values for array elements (i.e. be associative)
        neutral_element: obj
            neutral element for the operation above. eg. float('-inf')
            for max and 0 for sum.


def reduce(self, start=<NUM_LIT:0>, end=None):

if end is None:<EOL><INDENT>end = self._capacity<EOL><DEDENT>if end < <NUM_LIT:0>:<EOL><INDENT>end += self._capacity<EOL><DEDENT>end -= <NUM_LIT:1><EOL>return self._reduce_helper(start, end, <NUM_LIT:1>, <NUM_LIT:0>, self._capacity - <NUM_LIT:1>)<EOL>

Returns result of applying `self.operation`
        to a contiguous subsequence of the array.

            self.operation(arr[start], operation(arr[start+1], operation(... arr[end])))

        Parameters
        ----------
        start: int
            beginning of the subsequence
        end: int
            end of the subsequences

        Returns
        -------
        reduced: obj
            result of reducing self.operation over the specified range of array elements.


def sum(self, start=<NUM_LIT:0>, end=None):

return super(SumSegmentTree, self).reduce(start, end)<EOL>


assert <NUM_LIT:0> <= prefixsum <= self.sum() + <NUM_LIT><EOL>idx = <NUM_LIT:1><EOL>while idx < self._capacity:  <EOL><INDENT>if self._value[<NUM_LIT:2> * idx] > prefixsum:<EOL><INDENT>idx = <NUM_LIT:2> * idx<EOL><DEDENT>else:<EOL><INDENT>prefixsum -= self._value[<NUM_LIT:2> * idx]<EOL>idx = <NUM_LIT:2> * idx + <NUM_LIT:1><EOL><DEDENT><DEDENT>return idx - self._capacity<EOL>

Find the highest index `i` in the array such that
            sum(arr[0] + arr[1] + ... + arr[i - i]) <= prefixsum

        if array values are probabilities, this function
        allows to sample indexes according to the discrete
        probability efficiently.

        Parameters
        ----------
        perfixsum: float
            upperbound on the sum of array prefix

        Returns
        -------
        idx: int
            highest index satisfying the prefixsum constraint


def min(self, start=<NUM_LIT:0>, end=None):

return super(MinSegmentTree, self).reduce(start, end)<EOL>


if n<=<NUM_LIT:1>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>if os.getenv("<STR_LIT>") is None:<EOL><INDENT>env = os.environ.copy()<EOL>env.update(<EOL>MKL_NUM_THREADS="<STR_LIT:1>",<EOL>OMP_NUM_THREADS="<STR_LIT:1>",<EOL>IN_MPI="<STR_LIT:1>"<EOL>)<EOL>args = ["<STR_LIT>", "<STR_LIT>", str(n)]<EOL>if bind_to_core:<EOL><INDENT>args += ["<STR_LIT>", "<STR_LIT>"]<EOL><DEDENT>args += [sys.executable] + sys.argv<EOL>subprocess.check_call(args, env=env)<EOL>return "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>

Re-launches the current script with workers
    Returns "parent" for original parent, "child" for MPI children


comm = comm or MPI.COMM_WORLD<EOL>vals = comm.gather(localval)<EOL>if comm.rank == <NUM_LIT:0>:<EOL><INDENT>assert all(val==vals[<NUM_LIT:0>] for val in vals[<NUM_LIT:1>:])<EOL><DEDENT>

It's common to forget to initialize your variables to the same values, or
(less commonly) if you update them in some other way than adam, to get them out of sync.
This function checks that variables on all MPI workers are the same, and raises
an AssertionError otherwise

Arguments:
    comm: MPI communicator
    localval: list of local variables (list of variables on current worker to be compared with the other workers)


def switch(condition, then_expression, else_expression):

x_shape = copy.copy(then_expression.get_shape())<EOL>x = tf.cond(tf.cast(condition, '<STR_LIT:bool>'),<EOL>lambda: then_expression,<EOL>lambda: else_expression)<EOL>x.set_shape(x_shape)<EOL>return x<EOL>

Switches between two operations depending on a scalar value (int or bool).
    Note that both `then_expression` and `else_expression`
    should be symbolic tensors of the *same shape*.

    # Arguments
        condition: scalar tensor.
        then_expression: TensorFlow operation.
        else_expression: TensorFlow operation.


return tf.where(<EOL>tf.abs(x) < delta,<EOL>tf.square(x) * <NUM_LIT:0.5>,<EOL>delta * (tf.abs(x) - <NUM_LIT:0.5> * delta)<EOL>)<EOL>


sess = tf.get_default_session()<EOL>if sess is None:<EOL><INDENT>sess = make_session(config=config, make_default=True)<EOL><DEDENT>return sess<EOL>

Get default session or create one with a given config


def make_session(config=None, num_cpu=None, make_default=False, graph=None):

if num_cpu is None:<EOL><INDENT>num_cpu = int(os.getenv('<STR_LIT>', multiprocessing.cpu_count()))<EOL><DEDENT>if config is None:<EOL><INDENT>config = tf.ConfigProto(<EOL>allow_soft_placement=True,<EOL>inter_op_parallelism_threads=num_cpu,<EOL>intra_op_parallelism_threads=num_cpu)<EOL>config.gpu_options.allow_growth = True<EOL><DEDENT>if make_default:<EOL><INDENT>return tf.InteractiveSession(config=config, graph=graph)<EOL><DEDENT>else:<EOL><INDENT>return tf.Session(config=config, graph=graph)<EOL><DEDENT>

Returns a session that will use <num_cpu> CPU's only


return make_session(num_cpu=<NUM_LIT:1>)<EOL>

Returns a session which will only use a single CPU


new_variables = set(tf.global_variables()) - ALREADY_INITIALIZED<EOL>get_session().run(tf.variables_initializer(new_variables))<EOL>ALREADY_INITIALIZED.update(new_variables)<EOL>

Initialize all the uninitialized variables in the global scope.


def function(inputs, outputs, updates=None, givens=None):

if isinstance(outputs, list):<EOL><INDENT>return _Function(inputs, outputs, updates, givens=givens)<EOL><DEDENT>elif isinstance(outputs, (dict, collections.OrderedDict)):<EOL><INDENT>f = _Function(inputs, outputs.values(), updates, givens=givens)<EOL>return lambda *args, **kwargs: type(outputs)(zip(outputs.keys(), f(*args, **kwargs)))<EOL><DEDENT>else:<EOL><INDENT>f = _Function(inputs, [outputs], updates, givens=givens)<EOL>return lambda *args, **kwargs: f(*args, **kwargs)[<NUM_LIT:0>]<EOL><DEDENT>

Just like Theano function. Take a bunch of tensorflow placeholders and expressions
    computed based on those placeholders and produces f(inputs) -> outputs. Function f takes
    values to be fed to the input's placeholders and produces the values of the expressions
    in outputs.

    Input values can be passed in the same order as inputs or can be provided as kwargs based
    on placeholder name (passed to constructor or accessible via placeholder.op.name).

    Example:
        x = tf.placeholder(tf.int32, (), name="x")
        y = tf.placeholder(tf.int32, (), name="y")
        z = 3 * x + 2 * y
        lin = function([x, y], z, givens={y: 0})

        with single_threaded_session():
            initialize()

            assert lin(2) == 6
            assert lin(x=3) == 9
            assert lin(2, 2) == 10
            assert lin(x=2, y=3) == 12

    Parameters
    ----------
    inputs: [tf.placeholder, tf.constant, or object with make_feed_dict method]
        list of input arguments
    outputs: [tf.Variable] or tf.Variable
        list of outputs or a single output to be returned from function. Returned
        value will also have the same shape.
    updates: [tf.Operation] or tf.Operation
        list of update functions or single update function that will be run whenever
        the function is called. The return is ignored.


if not isinstance(data, np.ndarray) and not isinstance(data, list):<EOL><INDENT>return data<EOL><DEDENT>if isinstance(data, list):<EOL><INDENT>data = np.array(data)<EOL><DEDENT>placeholder_shape = [x or -<NUM_LIT:1> for x in placeholder.shape.as_list()]<EOL>assert _check_shape(placeholder_shape, data.shape),'<STR_LIT>'.format(data.shape, placeholder_shape)<EOL>return np.reshape(data, placeholder_shape)<EOL>

adjust shape of the data to the shape of the placeholder if possible.
If shape is incompatible, AssertionError is thrown

Parameters:
    placeholder     tensorflow input placeholder

    data            input data to be (potentially) reshaped to be fed into placeholder

Returns:
    reshaped data


return True<EOL>squeezed_placeholder_shape = _squeeze_shape(placeholder_shape)<EOL>squeezed_data_shape = _squeeze_shape(data_shape)<EOL>for i, s_data in enumerate(squeezed_data_shape):<EOL><INDENT>s_placeholder = squeezed_placeholder_shape[i]<EOL>if s_placeholder != -<NUM_LIT:1> and s_data != s_placeholder:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

check if two shapes are compatible (i.e. differ only by dimensions of size 1, or by the batch dimension)


import subprocess<EOL>subprocess.Popen(['<STR_LIT>', '<STR_LIT>', log_dir])<EOL>

To log the Tensorflow graph when using rl-algs
algorithms, you can run the following code
in your main script:
    import threading, time
    def start_tensorboard(session):
        time.sleep(10) # Wait until graph is setup
        tb_path = osp.join(logger.get_dir(), 'tb')
        summary_writer = tf.summary.FileWriter(tb_path, graph=session.graph)
        summary_op = tf.summary.merge_all()
        launch_tensorboard_in_background(tb_path)
    session = tf.get_default_session()
    t = threading.Thread(target=start_tensorboard, args=([session]))
    t.start()


def smooth(y, radius, mode='<STR_LIT>', valid_only=False):

assert mode in ('<STR_LIT>', '<STR_LIT>')<EOL>if len(y) < <NUM_LIT:2>*radius+<NUM_LIT:1>:<EOL><INDENT>return np.ones_like(y) * y.mean()<EOL><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>convkernel = np.ones(<NUM_LIT:2> * radius+<NUM_LIT:1>)<EOL>out = np.convolve(y, convkernel,mode='<STR_LIT>') / np.convolve(np.ones_like(y), convkernel, mode='<STR_LIT>')<EOL>if valid_only:<EOL><INDENT>out[:radius] = out[-radius:] = np.nan<EOL><DEDENT><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>convkernel = np.ones(radius)<EOL>out = np.convolve(y, convkernel,mode='<STR_LIT>') / np.convolve(np.ones_like(y), convkernel, mode='<STR_LIT>')<EOL>out = out[:-radius+<NUM_LIT:1>]<EOL>if valid_only:<EOL><INDENT>out[:radius] = np.nan<EOL><DEDENT><DEDENT>return out<EOL>

Smooth signal y, where radius is determines the size of the window

mode='twosided':
    average over the window [max(index - radius, 0), min(index + radius, len(y)-1)]
mode='causal':
    average over the window [max(index - radius, 0), index]

valid_only: put nan in entries where the full-sized window is not available


def one_sided_ema(xolds, yolds, low=None, high=None, n=<NUM_LIT>, decay_steps=<NUM_LIT:1.>, low_counts_threshold=<NUM_LIT>):

low = xolds[<NUM_LIT:0>] if low is None else low<EOL>high = xolds[-<NUM_LIT:1>] if high is None else high<EOL>assert xolds[<NUM_LIT:0>] <= low, '<STR_LIT>'.format(low, xolds[<NUM_LIT:0>])<EOL>assert xolds[-<NUM_LIT:1>] >= high, '<STR_LIT>'.format(high, xolds[-<NUM_LIT:1>])<EOL>assert len(xolds) == len(yolds), '<STR_LIT>'.format(len(xolds), len(yolds))<EOL>xolds = xolds.astype('<STR_LIT>')<EOL>yolds = yolds.astype('<STR_LIT>')<EOL>luoi = <NUM_LIT:0> <EOL>sum_y = <NUM_LIT:0.><EOL>count_y = <NUM_LIT:0.><EOL>xnews = np.linspace(low, high, n)<EOL>decay_period = (high - low) / (n - <NUM_LIT:1>) * decay_steps<EOL>interstep_decay = np.exp(- <NUM_LIT:1.> / decay_steps)<EOL>sum_ys = np.zeros_like(xnews)<EOL>count_ys = np.zeros_like(xnews)<EOL>for i in range(n):<EOL><INDENT>xnew = xnews[i]<EOL>sum_y *= interstep_decay<EOL>count_y *= interstep_decay<EOL>while True:<EOL><INDENT>xold = xolds[luoi]<EOL>if xold <= xnew:<EOL><INDENT>decay = np.exp(- (xnew - xold) / decay_period)<EOL>sum_y += decay * yolds[luoi]<EOL>count_y += decay<EOL>luoi += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT>if luoi >= len(xolds):<EOL><INDENT>break<EOL><DEDENT><DEDENT>sum_ys[i] = sum_y<EOL>count_ys[i] = count_y<EOL><DEDENT>ys = sum_ys / count_ys<EOL>ys[count_ys < low_counts_threshold] = np.nan<EOL>return xnews, ys, count_ys<EOL>

perform one-sided (causal) EMA (exponential moving average)
smoothing and resampling to an even grid with n points.
Does not do extrapolation, so we assume
xolds[0] <= low && high <= xolds[-1]

Arguments:

xolds: array or list  - x values of data. Needs to be sorted in ascending order
yolds: array of list  - y values of data. Has to have the same length as xolds

low: float            - min value of the new x grid. By default equals to xolds[0]
high: float           - max value of the new x grid. By default equals to xolds[-1]

n: int                - number of points in new x grid

decay_steps: float    - EMA decay factor, expressed in new x grid steps.

low_counts_threshold: float or int
                      - y values with counts less than this value will be set to NaN

Returns:
    tuple sum_ys, count_ys where
        xs        - array with new x grid
        ys        - array of EMA of y at each point of the new x grid
        count_ys  - array of EMA of y counts at each point of the new x grid


def symmetric_ema(xolds, yolds, low=None, high=None, n=<NUM_LIT>, decay_steps=<NUM_LIT:1.>, low_counts_threshold=<NUM_LIT>):

xs, ys1, count_ys1 = one_sided_ema(xolds, yolds, low, high, n, decay_steps, low_counts_threshold=<NUM_LIT:0>)<EOL>_,  ys2, count_ys2 = one_sided_ema(-xolds[::-<NUM_LIT:1>], yolds[::-<NUM_LIT:1>], -high, -low, n, decay_steps, low_counts_threshold=<NUM_LIT:0>)<EOL>ys2 = ys2[::-<NUM_LIT:1>]<EOL>count_ys2 = count_ys2[::-<NUM_LIT:1>]<EOL>count_ys = count_ys1 + count_ys2<EOL>ys = (ys1 * count_ys1 + ys2 * count_ys2) / count_ys<EOL>ys[count_ys < low_counts_threshold] = np.nan<EOL>return xs, ys, count_ys<EOL>

perform symmetric EMA (exponential moving average)
smoothing and resampling to an even grid with n points.
Does not do extrapolation, so we assume
xolds[0] <= low && high <= xolds[-1]

Arguments:

xolds: array or list  - x values of data. Needs to be sorted in ascending order
yolds: array of list  - y values of data. Has to have the same length as xolds

low: float            - min value of the new x grid. By default equals to xolds[0]
high: float           - max value of the new x grid. By default equals to xolds[-1]

n: int                - number of points in new x grid

decay_steps: float    - EMA decay factor, expressed in new x grid steps.

low_counts_threshold: float or int
                      - y values with counts less than this value will be set to NaN

Returns:
    tuple sum_ys, count_ys where
        xs        - array with new x grid
        ys        - array of EMA of y at each point of the new x grid
        count_ys  - array of EMA of y counts at each point of the new x grid


def load_results(root_dir_or_dirs, enable_progress=True, enable_monitor=True, verbose=False):

import re<EOL>if isinstance(root_dir_or_dirs, str):<EOL><INDENT>rootdirs = [osp.expanduser(root_dir_or_dirs)]<EOL><DEDENT>else:<EOL><INDENT>rootdirs = [osp.expanduser(d) for d in root_dir_or_dirs]<EOL><DEDENT>allresults = []<EOL>for rootdir in rootdirs:<EOL><INDENT>assert osp.exists(rootdir), "<STR_LIT>"%rootdir<EOL>for dirname, dirs, files in os.walk(rootdir):<EOL><INDENT>if '<STR_LIT>' in dirname:<EOL><INDENT>files[:] = []<EOL>continue<EOL><DEDENT>monitor_re = re.compile(r'<STR_LIT>')<EOL>if set(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']).intersection(files) orany([f for f in files if monitor_re.match(f)]):  <EOL><INDENT>result = {'<STR_LIT>' : dirname}<EOL>if "<STR_LIT>" in files:<EOL><INDENT>with open(osp.join(dirname, "<STR_LIT>"), "<STR_LIT:r>") as fh:<EOL><INDENT>result['<STR_LIT>'] = json.load(fh)<EOL><DEDENT><DEDENT>progjson = osp.join(dirname, "<STR_LIT>")<EOL>progcsv = osp.join(dirname, "<STR_LIT>")<EOL>if enable_progress:<EOL><INDENT>if osp.exists(progjson):<EOL><INDENT>result['<STR_LIT>'] = pandas.DataFrame(read_json(progjson))<EOL><DEDENT>elif osp.exists(progcsv):<EOL><INDENT>try:<EOL><INDENT>result['<STR_LIT>'] = read_csv(progcsv)<EOL><DEDENT>except pandas.errors.EmptyDataError:<EOL><INDENT>print('<STR_LIT>', dirname, '<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if verbose: print('<STR_LIT>'%dirname)<EOL><DEDENT><DEDENT>if enable_monitor:<EOL><INDENT>try:<EOL><INDENT>result['<STR_LIT>'] = pandas.DataFrame(monitor.load_results(dirname))<EOL><DEDENT>except monitor.LoadMonitorResultsError:<EOL><INDENT>print('<STR_LIT>'%dirname)<EOL><DEDENT>except Exception as e:<EOL><INDENT>print('<STR_LIT>'%(dirname, e))<EOL><DEDENT><DEDENT>if result.get('<STR_LIT>') is not None or result.get('<STR_LIT>') is not None:<EOL><INDENT>allresults.append(Result(**result))<EOL>if verbose:<EOL><INDENT>print('<STR_LIT>'%dirname)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if verbose: print('<STR_LIT>'%len(allresults))<EOL>return allresults<EOL>

load summaries of runs from a list of directories (including subdirectories)
Arguments:

enable_progress: bool - if True, will attempt to load data from progress.csv files (data saved by logger). Default: True

enable_monitor: bool - if True, will attempt to load data from monitor.csv files (data saved by Monitor environment wrapper). Default: True

verbose: bool - if True, will print out list of directories from which the data is loaded. Default: False

Returns:
List of Result objects with the following fields:
     - dirname - path to the directory data was loaded from
     - metadata - run metadata (such as command-line arguments and anything else in metadata.json file
     - monitor - if enable_monitor is True, this field contains pandas dataframe with loaded monitor.csv file (or aggregate of all *.monitor.csv files in the directory)
     - progress - if enable_progress is True, this field contains pandas dataframe with loaded progress.csv file