def _bfgs_inv_hessian_update(grad_delta, position_delta, normalization_factor,<EOL>inv_hessian_estimate):

<EOL>conditioned_grad_delta = _mul_right(inv_hessian_estimate, grad_delta)<EOL>conditioned_grad_delta_norm = tf.reduce_sum(<EOL>input_tensor=conditioned_grad_delta * grad_delta, axis=-<NUM_LIT:1>)<EOL>cross_term = _tensor_product(position_delta, conditioned_grad_delta)<EOL>def _expand_scalar(s):<EOL><INDENT>return s[..., tf.newaxis, tf.newaxis]<EOL><DEDENT>cross_term += _tensor_product(conditioned_grad_delta, position_delta)<EOL>position_term = _tensor_product(position_delta, position_delta)<EOL>with tf.control_dependencies([position_term]):<EOL><INDENT>position_term *= _expand_scalar(<EOL><NUM_LIT:1> + conditioned_grad_delta_norm / normalization_factor)<EOL><DEDENT>return (inv_hessian_estimate +<EOL>(position_term - cross_term) / _expand_scalar(normalization_factor))<EOL>

Applies the BFGS update to the inverse Hessian estimate.

    The BFGS update rule is (note A^T denotes the transpose of a vector/matrix A).

    ```None
      rho = 1/(grad_delta^T * position_delta)
      U = (I - rho * position_delta * grad_delta^T)
      H_1 =  U * H_0 * U^T + rho * position_delta * position_delta^T
    ```

    Here, `H_0` is the inverse Hessian estimate at the previous iteration and
    `H_1` is the next estimate. Note that `*` should be interpreted as the
    matrix multiplication (with the understanding that matrix multiplication for
    scalars is usual multiplication and for matrix with vector is the action of
    the matrix on the vector.).

    The implementation below utilizes an expanded version of the above formula
    to avoid the matrix multiplications that would be needed otherwise. By
    expansion it is easy to see that one only needs matrix-vector or
    vector-vector operations. The expanded version is:

    ```None
      f = 1 + rho * (grad_delta^T * H_0 * grad_delta)
      H_1 - H_0 = - rho * [position_delta * (H_0 * grad_delta)^T +
                          (H_0 * grad_delta) * position_delta^T] +
                    rho * f * [position_delta * position_delta^T]
    ```

    All the terms in square brackets are matrices and are constructed using
    vector outer products. All the other terms on the right hand side are scalars.
    Also worth noting that the first and second lines are both rank 1 updates
    applied to the current inverse Hessian estimate.

    Args:
      grad_delta: Real `Tensor` of shape `[..., n]`. The difference between the
        gradient at the new position and the old position.
      position_delta: Real `Tensor` of shape `[..., n]`. The change in position
        from the previous iteration to the current one.
      normalization_factor: Real `Tensor` of shape `[...]`. Should be equal to
        `grad_delta^T * position_delta`, i.e. `1/rho` as defined above.
      inv_hessian_estimate: Real `Tensor` of shape `[..., n, n]`. The previous
        estimate of the inverse Hessian. Should be positive definite and
        symmetric.

    Returns:
      A tuple containing the following fields
        is_valid: A Boolean `Tensor` of shape `[...]` indicating batch members
          where the update succeeded. The update can fail if the position change
          becomes orthogonal to the gradient change.
        next_inv_hessian_estimate: A `Tensor` of shape `[..., n, n]`. The next
          Hessian estimate updated using the BFGS update scheme. If the
          `inv_hessian_estimate` is symmetric and positive definite, the
          `next_inv_hessian_estimate` is guaranteed to satisfy the same
          conditions.


return tf.squeeze(tf.matmul(mat, tf.expand_dims(vec, axis=-<NUM_LIT:1>)), axis=-<NUM_LIT:1>)<EOL>


return tf.matmul(tf.expand_dims(t1, axis=-<NUM_LIT:1>), tf.expand_dims(t2, axis=-<NUM_LIT:2>))<EOL>

Computes the outer product of two possibly batched vectors.

    Args:
      t1: A `tf.Tensor` of shape `[..., n]`.
      t2: A `tf.Tensor` of shape `[..., m]`.

    Returns:
      A tensor of shape `[..., n, m]` with matching batch dimensions, let's call
      it `r`, whose components are:

      ```None
        r[..., i, j] = t1[..., i] * t2[..., j]
      ```


n = distribution_util.prefer_static_rank(mat)<EOL>perm = tf.range(n)<EOL>perm = tf.concat([perm[:-<NUM_LIT:2>], [perm[-<NUM_LIT:1>], perm[-<NUM_LIT:2>]]], axis=<NUM_LIT:0>)<EOL>return tf.transpose(a=mat, perm=perm)<EOL>

Transpose a possibly batched matrix.

    Args:
      mat: A `tf.Tensor` of shape `[..., n, m]`.

    Returns:
      A tensor of shape `[..., m, n]` with matching batch dimensions.


if isinstance(tval, tf.Tensor):<EOL><INDENT>return tf.where(cond, tval, fval)<EOL><DEDENT>elif isinstance(tval, tuple):<EOL><INDENT>cls = type(tval)<EOL>return cls(*(val_where(cond, t, f) for t, f in zip(tval, fval)))<EOL><DEDENT>else:<EOL><INDENT>raise Exception(TypeError)<EOL><DEDENT>


with tf.compat.v1.name_scope(name, '<STR_LIT>', [<EOL>val_0, search_interval, f_lim, sufficient_decrease_param,<EOL>curvature_param]):<EOL><INDENT>val_c = value_and_gradients_function(<EOL>_secant(search_interval.left, search_interval.right))<EOL>failed = search_interval.failed | ~is_finite(val_c)<EOL>converged = search_interval.converged | (~failed & _satisfies_wolfe(<EOL>val_0, val_c, f_lim, sufficient_decrease_param, curvature_param))<EOL>new_converged = converged & ~search_interval.converged<EOL>val_left = val_where(new_converged, val_c, search_interval.left)<EOL>val_right = val_where(new_converged, val_c, search_interval.right)<EOL>initial_args = _Secant2Result(<EOL>active=~failed & ~converged,<EOL>converged=converged,<EOL>failed=failed,<EOL>num_evals=search_interval.func_evals + <NUM_LIT:1>,<EOL>left=val_left,<EOL>right=val_right)<EOL>def _apply_secant2_inner():<EOL><INDENT>return _secant2_inner(<EOL>value_and_gradients_function,<EOL>initial_args,<EOL>val_0,<EOL>val_c,<EOL>f_lim,<EOL>sufficient_decrease_param,<EOL>curvature_param)<EOL><DEDENT>return prefer_static.cond(<EOL>tf.reduce_any(input_tensor=initial_args.active),<EOL>_apply_secant2_inner,<EOL>lambda: initial_args)<EOL><DEDENT>

Performs the secant square procedure of Hager Zhang.

    Given an interval that brackets a root, this procedure performs an update of
    both end points using two intermediate points generated using the secant
    interpolation. For details see the steps S1-S4 in [Hager and Zhang (2006)][2].

    The interval [a, b] must satisfy the opposite slope conditions described in
    the documentation for `update`.

    Args:
      value_and_gradients_function: A Python callable that accepts a real scalar
        tensor and returns an object that can be converted to a namedtuple.
        The namedtuple should have fields 'f' and 'df' that correspond to scalar
        tensors of real dtype containing the value of the function and its
        derivative at that point. The other namedtuple fields, if present,
        should be tensors or sequences (possibly nested) of tensors.
        In usual optimization application, this function would be generated by
        projecting the multivariate objective function along some specific
        direction. The direction is determined by some other procedure but should
        be a descent direction (i.e. the derivative of the projected univariate
        function must be negative at 0.).
        Alternatively, the function may represent the batching of `n` such line
        functions (e.g. projecting a single multivariate objective function along
        `n` distinct directions at once) accepting n points as input, i.e. a
        tensor of shape [n], and the fields 'f' and 'df' in the returned
        namedtuple should each be a tensor of shape [n], with the corresponding
        function values and derivatives at the input points.
      val_0: A namedtuple, as returned by value_and_gradients_function evaluated
        at `0.`.
      search_interval: A namedtuple describing the current search interval,
        must include the fields:
        - converged: Boolean `Tensor` of shape [n], indicating batch members
            where search has already converged. Interval for these batch members
            won't be modified.
        - failed: Boolean `Tensor` of shape [n], indicating batch members
            where search has already failed. Interval for these batch members
            wont be modified.
        - iterations: Scalar int32 `Tensor`. Number of line search iterations
            so far.
        - func_evals: Scalar int32 `Tensor`. Number of function evaluations
            so far.
        - left: A namedtuple, as returned by value_and_gradients_function,
            of the left end point of the current search interval.
        - right: A namedtuple, as returned by value_and_gradients_function,
            of the right end point of the current search interval.
      f_lim: Scalar `Tensor` of real dtype. The function value threshold for
        the approximate Wolfe conditions to be checked.
      sufficient_decrease_param: Positive scalar `Tensor` of real dtype.
        Bounded above by the curvature param. Corresponds to 'delta' in the
        terminology of [Hager and Zhang (2006)][2].
      curvature_param: Positive scalar `Tensor` of real dtype. Bounded above
        by `1.`. Corresponds to 'sigma' in the terminology of
        [Hager and Zhang (2006)][2].
      name: (Optional) Python str. The name prefixed to the ops created by this
        function. If not supplied, the default name 'secant2' is used.

    Returns:
      A namedtuple containing the following fields.
        active: A boolean `Tensor` of shape [n]. Used internally by the procedure
          to indicate batch members on which there is work left to do.
        converged: A boolean `Tensor` of shape [n]. Indicates whether a point
          satisfying the Wolfe conditions has been found. If this is True, the
          interval will be degenerate (i.e. `left` and `right` below will be
          identical).
        failed: A boolean `Tensor` of shape [n]. Indicates if invalid function or
          gradient values were encountered (i.e. infinity or NaNs).
        num_evals: A scalar int32 `Tensor`. The total number of function
          evaluations made.
        left: Return value of value_and_gradients_function at the updated left
          end point of the interval.
        right: Return value of value_and_gradients_function at the updated right
          end point of the interval.


<EOL>update_result = update(value_and_gradients_function,<EOL>initial_args.left,<EOL>initial_args.right,<EOL>val_c,<EOL>f_lim,<EOL>active=initial_args.active)<EOL>active = initial_args.active & ~update_result.failed<EOL>failed = initial_args.failed | update_result.failed<EOL>val_left = val_where(active, update_result.left, initial_args.left)<EOL>val_right = val_where(active, update_result.right, initial_args.right)<EOL>updated_left = active & tf.equal(val_left.x, val_c.x)<EOL>updated_right = active & tf.equal(val_right.x, val_c.x)<EOL>is_new = updated_left | updated_right<EOL>next_c = tf.where(updated_left,<EOL>_secant(initial_args.left, val_left),<EOL>val_c.x)<EOL>next_c = tf.where(updated_right,<EOL>_secant(initial_args.right, val_right),<EOL>next_c)<EOL>in_range = (val_left.x <= next_c) & (next_c <= val_right.x)<EOL>needs_extra_eval = tf.reduce_any(input_tensor=in_range & is_new)<EOL>num_evals = initial_args.num_evals + update_result.num_evals<EOL>num_evals = num_evals + tf.cast(needs_extra_eval, num_evals.dtype)<EOL>next_args = _Secant2Result(<EOL>active=active & in_range,  <EOL>converged=initial_args.converged,<EOL>failed=failed,<EOL>num_evals=num_evals,<EOL>left=val_left,<EOL>right=val_right)<EOL>def _apply_inner_update():<EOL><INDENT>next_val_c = prefer_static.cond(<EOL>needs_extra_eval,<EOL>(lambda: value_and_gradients_function(next_c)),<EOL>(lambda: val_c))<EOL>return _secant2_inner_update(<EOL>value_and_gradients_function, next_args, val_0, next_val_c, f_lim,<EOL>sufficient_decrease_param, curvature_param)<EOL><DEDENT>return prefer_static.cond(<EOL>tf.reduce_any(input_tensor=next_args.active),<EOL>_apply_inner_update,<EOL>lambda: next_args)<EOL>


<EOL>new_failed = initial_args.active & ~is_finite(val_c)<EOL>active = initial_args.active & ~new_failed<EOL>failed = initial_args.failed | new_failed<EOL>found_wolfe = active & _satisfies_wolfe(<EOL>val_0, val_c, f_lim, sufficient_decrease_param, curvature_param)<EOL>val_left = val_where(found_wolfe, val_c, initial_args.left)<EOL>val_right = val_where(found_wolfe, val_c, initial_args.right)<EOL>converged = initial_args.converged | found_wolfe<EOL>active = active & ~found_wolfe<EOL>def _apply_update():<EOL><INDENT>update_result = update(<EOL>value_and_gradients_function, val_left, val_right, val_c, f_lim,<EOL>active=active)<EOL>return _Secant2Result(<EOL>active=tf.zeros_like(active),  <EOL>converged=converged,<EOL>failed=failed | update_result.failed,<EOL>num_evals=initial_args.num_evals + update_result.num_evals,<EOL>left=update_result.left,<EOL>right=update_result.right)<EOL><DEDENT>def _default():<EOL><INDENT>return _Secant2Result(<EOL>active=active,<EOL>converged=converged,<EOL>failed=failed,<EOL>num_evals=initial_args.num_evals,<EOL>left=val_left,<EOL>right=val_right)<EOL><DEDENT>return prefer_static.cond(<EOL>tf.reduce_any(input_tensor=active), _apply_update, _default)<EOL>


def update(value_and_gradients_function, val_left, val_right, val_trial, f_lim,<EOL>active=None):

<EOL>within_range = (val_left.x < val_trial.x) & (val_trial.x < val_right.x)<EOL>if active is not None:<EOL><INDENT>within_range = within_range & active<EOL><DEDENT>valid_left = (val_trial.df < <NUM_LIT:0>) & (val_trial.f <= f_lim)<EOL>needs_bisect = within_range & (val_trial.df < <NUM_LIT:0>) & (val_trial.f > f_lim)<EOL>left = val_where(within_range & valid_left, val_trial, val_left)<EOL>right = val_where(within_range & ~valid_left, val_trial, val_right)<EOL>bisect_args = _IntermediateResult(<EOL>iteration=tf.convert_to_tensor(value=<NUM_LIT:0>),<EOL>stopped=~needs_bisect,<EOL>failed=tf.zeros_like(within_range),  <EOL>num_evals=tf.convert_to_tensor(value=<NUM_LIT:0>),<EOL>left=left,<EOL>right=right)<EOL>return _bisect(value_and_gradients_function, bisect_args, f_lim)<EOL>

Squeezes a bracketing interval containing the minimum.

    Given an interval which brackets a minimum and a point in that interval,
    finds a smaller nested interval which also brackets the minimum. If the
    supplied point does not lie in the bracketing interval, the current interval
    is returned.

    The following description is given in terms of individual points evaluated on
    a line function to be minimized. Note, however, the implementation also
    accepts batches of points allowing to minimize multiple line functions at
    once. See details on the docstring of `value_and_gradients_function` below.

    The requirement of the interval bracketing a minimum is expressed through the
    opposite slope conditions. Assume the left end point is 'a', the right
    end point is 'b', the function to be minimized is 'f' and the derivative is
    'df'. The update procedure relies on the following conditions being satisfied:

    '''
      f(a) <= f(0) + epsilon   (1)
      df(a) < 0                (2)
      df(b) > 0                (3)
    '''

    In the first condition, epsilon is a small positive constant. The condition
    demands that the function at the left end point be not much bigger than the
    starting point (i.e. 0). This is an easy to satisfy condition because by
    assumption, we are in a direction where the function value is decreasing.
    The second and third conditions together demand that there is at least one
    zero of the derivative in between a and b.

    In addition to the interval, the update algorithm requires a third point to
    be supplied. Usually, this point would lie within the interval [a, b]. If the
    point is outside this interval, the current interval is returned. If the
    point lies within the interval, the behaviour of the function and derivative
    value at this point is used to squeeze the original interval in a manner that
    preserves the opposite slope conditions.

    For further details of this component, see the procedure U0-U3 on page 123 of
    the [Hager and Zhang (2006)][2] article.

    Note that this function does not explicitly verify whether the opposite slope
    conditions are satisfied for the supplied interval. It is assumed that this
    is so.

    Args:
      value_and_gradients_function: A Python callable that accepts a real scalar
        tensor and returns an object that can be converted to a namedtuple.
        The namedtuple should have fields 'f' and 'df' that correspond to scalar
        tensors of real dtype containing the value of the function and its
        derivative at that point. The other namedtuple fields, if present,
        should be tensors or sequences (possibly nested) of tensors.
        In usual optimization application, this function would be generated by
        projecting the multivariate objective function along some specific
        direction. The direction is determined by some other procedure but should
        be a descent direction (i.e. the derivative of the projected univariate
        function must be negative at 0.).
        Alternatively, the function may represent the batching of `n` such line
        functions (e.g. projecting a single multivariate objective function along
        `n` distinct directions at once) accepting n points as input, i.e. a
        tensor of shape [n], and the fields 'f' and 'df' in the returned
        namedtuple should each be a tensor of shape [n], with the corresponding
        function values and derivatives at the input points.
      val_left: Return value of value_and_gradients_function at the left
        end point of the bracketing interval (labelles 'a' above).
      val_right: Return value of value_and_gradients_function at the right
        end point of the bracketing interval (labelles 'b' above).
      val_trial: Return value of value_and_gradients_function at the trial point
        to be used to shrink the interval (labelled 'c' above).
      f_lim: real `Tensor` of shape [n]. The function value threshold for
        the approximate Wolfe conditions to be checked for each batch member.
      active: optional boolean `Tensor` of shape [n]. Relevant in batching mode
        only, indicates batch members on which the update procedure should be
        applied. On non-active members the current left/right interval is returned
        unmodified.

    Returns:
      A namedtuple containing the following fields:
        iteration: An int32 scalar `Tensor`. The number of iterations performed
          by the bisect algorithm.
        stopped: A boolean `Tensor` of shape [n]. True for those batch members
          where the bisection algorithm terminated.
        failed: A boolean `Tensor` of shape [n]. True for those batch members
          where an error was encountered.
        num_evals: An int32 scalar `Tensor`. The number of times the objective
          function was evaluated.
        left: Return value of value_and_gradients_function at the updated left
          end point of the interval found.
        right: Return value of value_and_gradients_function at the updated right
          end point of the interval found.


already_stopped = search_interval.failed | search_interval.converged<EOL>bracketed = search_interval.right.df >= <NUM_LIT:0><EOL>needs_bisect = (<EOL>search_interval.right.df < <NUM_LIT:0>) & (search_interval.right.f > f_lim)<EOL>initial_args = _IntermediateResult(<EOL>iteration=search_interval.iterations,<EOL>stopped=already_stopped | bracketed | needs_bisect,<EOL>failed=search_interval.failed,<EOL>num_evals=search_interval.func_evals,<EOL>left=search_interval.left,<EOL>right=search_interval.right)<EOL>def _loop_cond(curr):<EOL><INDENT>return (curr.iteration <<EOL>max_iterations) & ~tf.reduce_all(input_tensor=curr.stopped)<EOL><DEDENT>def _loop_body(curr):<EOL><INDENT>"""<STR_LIT>"""<EOL>new_right = value_and_gradients_function(expansion_param * curr.right.x)<EOL>left = val_where(curr.stopped, curr.left, curr.right)<EOL>right = val_where(curr.stopped, curr.right, new_right)<EOL>failed = curr.failed | ~is_finite(right)<EOL>bracketed = right.df >= <NUM_LIT:0><EOL>needs_bisect = (right.df < <NUM_LIT:0>) & (right.f > f_lim)<EOL>return [_IntermediateResult(<EOL>iteration=curr.iteration + <NUM_LIT:1>,<EOL>stopped=curr.stopped | failed | bracketed | needs_bisect,<EOL>failed=failed,<EOL>num_evals=curr.num_evals + <NUM_LIT:1>,<EOL>left=left,<EOL>right=right)]<EOL><DEDENT>bracket_result = tf.while_loop(<EOL>cond=_loop_cond, body=_loop_body, loop_vars=[initial_args])[<NUM_LIT:0>]<EOL>needs_bisect = (<EOL>(bracket_result.right.df < <NUM_LIT:0>) & (bracket_result.right.f > f_lim))<EOL>stopped = already_stopped | bracket_result.failed | ~needs_bisect<EOL>left = val_where(stopped, bracket_result.left, search_interval.left)<EOL>bisect_args = bracket_result._replace(stopped=stopped, left=left)<EOL>return _bisect(value_and_gradients_function, bisect_args, f_lim)<EOL>

Brackets the minimum given an initial starting point.

    Applies the Hager Zhang bracketing algorithm to find an interval containing
    a region with points satisfying Wolfe conditions. Uses the supplied initial
    step size 'c', the right end point of the provided search interval, to find
    such an interval. The only condition on 'c' is that it should be positive.
    For more details see steps B0-B3 in [Hager and Zhang (2006)][2].

    Args:
      value_and_gradients_function: A Python callable that accepts a real scalar
        tensor and returns a namedtuple containing the value filed `f` of the
        function and its derivative value field `df` at that point.
        Alternatively, the function may representthe batching of `n` such line
        functions (e.g. projecting a single multivariate objective function along
        `n` distinct directions at once) accepting n points as input, i.e. a
        tensor of shape [n], and return a tuple of two tensors of shape [n], the
        function values and the corresponding derivatives at the input points.
      search_interval: A namedtuple describing the current search interval,
        must include the fields:
        - converged: Boolean `Tensor` of shape [n], indicating batch members
            where search has already converged. Interval for these batch members
            wont be modified.
        - failed: Boolean `Tensor` of shape [n], indicating batch members
            where search has already failed. Interval for these batch members
            wont be modified.
        - iterations: Scalar int32 `Tensor`. Number of line search iterations
            so far.
        - func_evals: Scalar int32 `Tensor`. Number of function evaluations
            so far.
        - left: A namedtuple, as returned by value_and_gradients_function
            evaluated at 0, the left end point of the current interval.
        - right: A namedtuple, as returned by value_and_gradients_function,
            of the right end point of the current interval (labelled 'c' above).
      f_lim: real `Tensor` of shape [n]. The function value threshold for
        the approximate Wolfe conditions to be checked for each batch member.
      max_iterations: Int32 scalar `Tensor`. The maximum number of iterations
        permitted. The limit applies equally to all batch members.
      expansion_param: Scalar positive `Tensor` of real dtype. Must be greater
        than `1.`. Used to expand the initial interval in case it does not bracket
        a minimum.

    Returns:
      A namedtuple with the following fields.
        iteration: An int32 scalar `Tensor`. The number of iterations performed.
          Bounded above by `max_iterations` parameter.
        stopped: A boolean `Tensor` of shape [n]. True for those batch members
          where the algorithm terminated before reaching `max_iterations`.
        failed: A boolean `Tensor` of shape [n]. True for those batch members
          where an error was encountered during bracketing.
        num_evals: An int32 scalar `Tensor`. The number of times the objective
          function was evaluated.
        left: Return value of value_and_gradients_function at the updated left
          end point of the interval found.
        right: Return value of value_and_gradients_function at the updated right
          end point of the interval found.


failed = ~is_finite(initial_left, initial_right)<EOL>needs_bisect = (initial_right.df < <NUM_LIT:0>) & (initial_right.f > f_lim)<EOL>bisect_args = _IntermediateResult(<EOL>iteration=tf.convert_to_tensor(value=<NUM_LIT:0>),<EOL>stopped=failed | ~needs_bisect,<EOL>failed=failed,<EOL>num_evals=tf.convert_to_tensor(value=<NUM_LIT:0>),<EOL>left=initial_left,<EOL>right=initial_right)<EOL>return _bisect(value_and_gradients_function, bisect_args, f_lim)<EOL>

Bisects an interval and updates to satisfy opposite slope conditions.

    Corresponds to the step U3 in [Hager and Zhang (2006)][2].

    Args:
      value_and_gradients_function: A Python callable that accepts a real scalar
        tensor and returns a namedtuple containing the value filed `f` of the
        function and its derivative value field `df` at that point.
        Alternatively, the function may representthe batching of `n` such line
        functions (e.g. projecting a single multivariate objective function along
        `n` distinct directions at once) accepting n points as input, i.e. a
        tensor of shape [n], and return a tuple of two tensors of shape [n], the
        function values and the corresponding derivatives at the input points.
      initial_left: Return value of value_and_gradients_function at the left end
        point of the current bracketing interval.
      initial_right: Return value of value_and_gradients_function at the right end
        point of the current bracketing interval.
      f_lim: real `Tensor` of shape [n]. The function value threshold for
        the approximate Wolfe conditions to be checked for each batch member.

    Returns:
      A namedtuple containing the following fields:
        iteration: An int32 scalar `Tensor`. The number of iterations performed.
          Bounded above by `max_iterations` parameter.
        stopped: A boolean scalar `Tensor`. True if the bisect algorithm
          terminated.
        failed: A scalar boolean tensor. Indicates whether the objective function
          failed to produce a finite value.
        num_evals: A scalar int32 tensor. The number of value and gradients
          function evaluations.
        left: Return value of value_and_gradients_function at the left end
          point of the bracketing interval found.
        right: Return value of value_and_gradients_function at the right end
          point of the bracketing interval found.


def _bisect(value_and_gradients_function, initial_args, f_lim):

def _loop_cond(curr):<EOL><INDENT>return ~tf.reduce_all(input_tensor=curr.stopped)<EOL><DEDENT>def _loop_body(curr):<EOL><INDENT>"""<STR_LIT>"""<EOL>mid = value_and_gradients_function((curr.left.x + curr.right.x) / <NUM_LIT:2>)<EOL>failed = (curr.failed | ~is_finite(mid) |<EOL>tf.equal(mid.x, curr.left.x) | tf.equal(mid.x, curr.right.x))<EOL>to_update = ~(curr.stopped | failed)<EOL>update_left = (mid.df < <NUM_LIT:0>) & (mid.f <= f_lim)<EOL>left = val_where(to_update & update_left, mid, curr.left)<EOL>right = val_where(to_update & ~update_left, mid, curr.right)<EOL>stopped = curr.stopped | failed | (right.df >= <NUM_LIT:0>)<EOL>return [_IntermediateResult(<EOL>iteration=curr.iteration,<EOL>stopped=stopped,<EOL>failed=failed,<EOL>num_evals=curr.num_evals + <NUM_LIT:1>,<EOL>left=left,<EOL>right=right)]<EOL><DEDENT>return tf.while_loop(<EOL>cond=_loop_cond, body=_loop_body, loop_vars=[initial_args])[<NUM_LIT:0>]<EOL>

Actual implementation of bisect given initial_args in a _BracketResult.


val_1_finite = tf.math.is_finite(val_1.f) & tf.math.is_finite(val_1.df)<EOL>if val_2 is not None:<EOL><INDENT>return val_1_finite & tf.math.is_finite(val_2.f) & tf.math.is_finite(<EOL>val_2.df)<EOL><DEDENT>return val_1_finite<EOL>

Checks if the supplied values are finite.

    Args:
      val_1: A namedtuple instance with the function value and derivative,
        as returned e.g. by value_and_gradients_function evaluations.
      val_2: (Optional) A namedtuple instance with the function value and
        derivative, as returned e.g. by value_and_gradients_function evaluations.

    Returns:
      is_finite: Scalar boolean `Tensor` indicating whether the function value
        and the derivative in `val_1` (and optionally in `val_2`) are all finite.


exact_wolfe_suff_dec = (sufficient_decrease_param * val_0.df >=<EOL>(val_c.f - val_0.f) / val_c.x)<EOL>wolfe_curvature = val_c.df >= curvature_param * val_0.df<EOL>exact_wolfe = exact_wolfe_suff_dec & wolfe_curvature<EOL>approx_wolfe_applies = val_c.f <= f_lim<EOL>approx_wolfe_suff_dec = ((<NUM_LIT:2> * sufficient_decrease_param - <NUM_LIT:1>) * val_0.df<EOL>>= val_c.df)<EOL>approx_wolfe = approx_wolfe_applies & approx_wolfe_suff_dec & wolfe_curvature<EOL>is_satisfied = exact_wolfe | approx_wolfe<EOL>return is_satisfied<EOL>

Checks whether the Wolfe or approx Wolfe conditions are satisfied.

    The Wolfe conditions are a set of stopping criteria for an inexact line search
    algorithm. Let f(a) be the function value along the search direction and
    df(a) the derivative along the search direction evaluated a distance 'a'.
    Here 'a' is the distance along the search direction. The Wolfe conditions are:

      ```None
        f(a) <= f(0) + delta * a * df(0)   (Armijo/Sufficient decrease condition)
        df(a) >= sigma * df(0)             (Weak curvature condition)
      ```
    `delta` and `sigma` are two user supplied parameters satisfying:
     `0 < delta < sigma <= 1.`. In the following, delta is called
     `sufficient_decrease_param` and sigma is called `curvature_param`.

    On a finite precision machine, the Wolfe conditions are difficult to satisfy
    when one is close to the minimum. Hence, Hager-Zhang propose replacing
    the sufficient decrease condition with the following condition on the
    derivative in the vicinity of a minimum.

      ```None
        df(a) <= (2 * delta - 1) * df(0)  (Approx Wolfe sufficient decrease)
      ```
    This condition is only used if one is near the minimum. This is tested using

      ```None
        f(a) <= f(0) + epsilon * |f(0)|
      ```
    The following function checks both the Wolfe and approx Wolfe conditions.
    Here, `epsilon` is a small positive constant. In the following, the argument
    `f_lim` corresponds to the product: epsilon * |f(0)|.

    Args:
      val_0: A namedtuple, as returned by value_and_gradients_function
        evaluated at 0.
      val_c: A namedtuple, as returned by value_and_gradients_function
        evaluated at the point to be tested.
      f_lim: Scalar `Tensor` of real dtype. The function value threshold for
        the approximate Wolfe conditions to be checked.
      sufficient_decrease_param: Positive scalar `Tensor` of real dtype.
        Bounded above by the curvature param. Corresponds to 'delta' in the
        terminology of [Hager and Zhang (2006)][2].
      curvature_param: Positive scalar `Tensor` of real dtype. Bounded above
        by `1.`. Corresponds to 'sigma' in the terminology of
        [Hager Zhang (2005)][1].

    Returns:
      is_satisfied: A scalar boolean `Tensor` which is True if either the
        Wolfe or approximate Wolfe conditions are satisfied.


return (val_a.x * val_b.df - val_b.x * val_a.df) / (val_b.df - val_a.df)<EOL>

Returns the secant interpolation for the minimum.

    The secant method is a technique for finding roots of nonlinear functions.
    When finding the minimum, one applies the secant method to the derivative
    of the function.
    For an arbitrary function and a bounding interval, the secant approximation
    can produce the next point which is outside the bounding interval. However,
    with the assumption of opposite slope condtion on the interval [a,b] the new
    point c is always bracketed by [a,b]. Note that by assumption,
    f'(a) < 0 and f'(b) > 0.
    Hence c is a weighted average of a and b and thus always in [a, b].

    Args:
      val_a: A namedtuple with the left end point, function value and derivative,
        of the current interval (i.e. a).
      val_b: A namedtuple with the right end point, function value and derivative,
        of the current interval (i.e. b).

    Returns:
      approx_minimum: A scalar real `Tensor`. An approximation to the point
        at which the derivative vanishes.


if isinstance(dtype, tf.DType):<EOL><INDENT>dtype = dtype.as_numpy_dtype()<EOL><DEDENT>return np.finfo(dtype).eps<EOL>

Returns the machine epsilon for the supplied dtype.


with tf.compat.v1.name_scope(name, '<STR_LIT>', [<EOL>initial_step_size, value_at_initial_step, value_at_zero, converged,<EOL>threshold_use_approximate_wolfe_condition, shrinkage_param,<EOL>expansion_param, sufficient_decrease_param, curvature_param]):<EOL><INDENT>val_0, val_initial, f_lim, prepare_evals = _prepare_args(<EOL>value_and_gradients_function,<EOL>initial_step_size,<EOL>value_at_initial_step,<EOL>value_at_zero,<EOL>threshold_use_approximate_wolfe_condition)<EOL>valid_inputs = (hzl.is_finite(val_0) & (val_0.df < <NUM_LIT:0>) &<EOL>tf.math.is_finite(val_initial.x) & (val_initial.x > <NUM_LIT:0>))<EOL>if converged is None:<EOL><INDENT>init_converged = tf.zeros_like(valid_inputs)  <EOL><DEDENT>else:<EOL><INDENT>init_converged = tf.convert_to_tensor(value=converged)<EOL><DEDENT>failed = ~init_converged & ~valid_inputs<EOL>active = ~init_converged & valid_inputs<EOL>fix_step_evals, val_c, fix_failed = _fix_step_size(<EOL>value_and_gradients_function, val_initial, active,<EOL>step_size_shrink_param)<EOL>init_interval = HagerZhangLineSearchResult(<EOL>converged=init_converged,<EOL>failed=failed | fix_failed,<EOL>func_evals=prepare_evals + fix_step_evals,<EOL>iterations=tf.convert_to_tensor(value=<NUM_LIT:0>),<EOL>left=val_0,<EOL>right=hzl.val_where(init_converged, val_0, val_c))<EOL>def _apply_bracket_and_search():<EOL><INDENT>"""<STR_LIT>"""<EOL>return _bracket_and_search(<EOL>value_and_gradients_function, init_interval, f_lim, max_iterations,<EOL>shrinkage_param, expansion_param, sufficient_decrease_param,<EOL>curvature_param)<EOL><DEDENT>init_active = ~init_interval.failed & ~init_interval.converged<EOL>return prefer_static.cond(<EOL>tf.reduce_any(input_tensor=init_active),<EOL>_apply_bracket_and_search,<EOL>lambda: init_interval)<EOL><DEDENT>

The Hager Zhang line search algorithm.

    Performs an inexact line search based on the algorithm of
    [Hager and Zhang (2006)][2].
    The univariate objective function `value_and_gradients_function` is typically
    generated by projecting a multivariate objective function along a search
    direction. Suppose the multivariate function to be minimized is
    `g(x1,x2, .. xn)`. Let (d1, d2, ..., dn) be the direction along which we wish
    to perform a line search. Then the projected univariate function to be used
    for line search is

    ```None
      f(a) = g(x1 + d1 * a, x2 + d2 * a, ..., xn + dn * a)
    ```

    The directional derivative along (d1, d2, ..., dn) is needed for this
    procedure. This also corresponds to the derivative of the projected function
    `f(a)` with respect to `a`. Note that this derivative must be negative for
    `a = 0` if the direction is a descent direction.

    The usual stopping criteria for the line search is the satisfaction of the
    (weak) Wolfe conditions. For details of the Wolfe conditions, see
    ref. [3]. On a finite precision machine, the exact Wolfe conditions can
    be difficult to satisfy when one is very close to the minimum and as argued
    by [Hager and Zhang (2005)][1], one can only expect the minimum to be
    determined within square root of machine precision. To improve the situation,
    they propose to replace the Wolfe conditions with an approximate version
    depending on the derivative of the function which is applied only when one
    is very close to the minimum. The following algorithm implements this
    enhanced scheme.

    ### Usage:

    Primary use of line search methods is as an internal component of a class of
    optimization algorithms (called line search based methods as opposed to
    trust region methods). Hence, the end user will typically not want to access
    line search directly. In particular, inexact line search should not be
    confused with a univariate minimization method. The stopping criteria of line
    search is the satisfaction of Wolfe conditions and not the discovery of the
    minimum of the function.

    With this caveat in mind, the following example illustrates the standalone
    usage of the line search.

    ```python
      # Define value and gradient namedtuple
      ValueAndGradient = namedtuple('ValueAndGradient', ['x', 'f', 'df'])
      # Define a quadratic target with minimum at 1.3.
      def value_and_gradients_function(x):
        return ValueAndGradient(x=x, f=(x - 1.3) ** 2, df=2 * (x-1.3))
      # Set initial step size.
      step_size = tf.constant(0.1)
      ls_result = tfp.optimizer.linesearch.hager_zhang(
          value_and_gradients_function, initial_step_size=step_size)
      # Evaluate the results.
      with tf.Session() as session:
        results = session.run(ls_result)
        # Ensure convergence.
        assert results.converged
        # If the line search converged, the left and the right ends of the
        # bracketing interval are identical.
        assert results.left.x == result.right.x
        # Print the number of evaluations and the final step size.
        print ("Final Step Size: %f, Evaluations: %d" % (results.left.x,
                                                         results.func_evals))
    ```

    ### References:
    [1]: William Hager, Hongchao Zhang. A new conjugate gradient method with
      guaranteed descent and an efficient line search. SIAM J. Optim., Vol 16. 1,
      pp. 170-172. 2005.
      https://www.math.lsu.edu/~hozhang/papers/cg_descent.pdf

    [2]: William Hager, Hongchao Zhang. Algorithm 851: CG_DESCENT, a conjugate
      gradient method with guaranteed descent. ACM Transactions on Mathematical
      Software, Vol 32., 1, pp. 113-137. 2006.
      http://users.clas.ufl.edu/hager/papers/CG/cg_compare.pdf

    [3]: Jorge Nocedal, Stephen Wright. Numerical Optimization. Springer Series in
      Operations Research. pp 33-36. 2006

    Args:
      value_and_gradients_function: A Python callable that accepts a real scalar
        tensor and returns a namedtuple with the fields 'x', 'f', and 'df' that
        correspond to scalar tensors of real dtype containing the point at which
        the function was evaluated, the value of the function, and its
        derivative at that point. The other namedtuple fields, if present,
        should be tensors or sequences (possibly nested) of tensors.
        In usual optimization application, this function would be generated by
        projecting the multivariate objective function along some specific
        direction. The direction is determined by some other procedure but should
        be a descent direction (i.e. the derivative of the projected univariate
        function must be negative at 0.).
        Alternatively, the function may represent the batching of `n` such line
        functions (e.g. projecting a single multivariate objective function along
        `n` distinct directions at once) accepting n points as input, i.e. a
        tensor of shape [n], and the fields 'x', 'f' and 'df' in the returned
        namedtuple should each be a tensor of shape [n], with the corresponding
        input points, function values, and derivatives at those input points.
      initial_step_size: (Optional) Scalar positive `Tensor` of real dtype, or
        a tensor of shape [n] in batching mode. The initial value (or values) to
        try to bracket the minimum. Default is `1.` as a float32.
        Note that this point need not necessarily bracket the minimum for the line
        search to work correctly but the supplied value must be greater than 0.
        A good initial value will make the search converge faster.
      value_at_initial_step: (Optional) The full return value of evaluating
        value_and_gradients_function at initial_step_size, i.e. a namedtuple with
        'x', 'f', 'df', if already known by the caller. If supplied the value of
        `initial_step_size` will be ignored, otherwise the tuple will be computed
        by evaluating value_and_gradients_function.
      value_at_zero: (Optional) The full return value of
        value_and_gradients_function at `0.`, i.e. a namedtuple with
        'x', 'f', 'df', if already known by the caller. If not supplied the tuple
        will be computed by evaluating value_and_gradients_function.
      converged: (Optional) In batching mode a tensor of shape [n], indicating
        batch members which have already converged and no further search should
        be performed. These batch members are also reported as converged in the
        output, and both their `left` and `right` are set to the
        `value_at_initial_step`.
      threshold_use_approximate_wolfe_condition: Scalar positive `Tensor`
        of real dtype. Corresponds to the parameter 'epsilon' in
        [Hager and Zhang (2006)][2]. Used to estimate the
        threshold at which the line search switches to approximate Wolfe
        conditions.
      shrinkage_param: Scalar positive Tensor of real dtype. Must be less than
        `1.`. Corresponds to the parameter `gamma` in
        [Hager and Zhang (2006)][2].
        If the secant**2 step does not shrink the bracketing interval by this
        proportion, a bisection step is performed to reduce the interval width.
      expansion_param: Scalar positive `Tensor` of real dtype. Must be greater
        than `1.`. Used to expand the initial interval in case it does not bracket
        a minimum. Corresponds to `rho` in [Hager and Zhang (2006)][2].
      sufficient_decrease_param: Positive scalar `Tensor` of real dtype.
        Bounded above by the curvature param. Corresponds to `delta` in the
        terminology of [Hager and Zhang (2006)][2].
      curvature_param: Positive scalar `Tensor` of real dtype. Bounded above
        by `1.`. Corresponds to 'sigma' in the terminology of
        [Hager and Zhang (2006)][2].
      step_size_shrink_param: Positive scalar `Tensor` of real dtype. Bounded
        above by `1`. If the supplied step size is too big (i.e. either the
        objective value or the gradient at that point is infinite), this factor
        is used to shrink the step size until it is finite.
      max_iterations: Positive scalar `Tensor` of integral dtype or None. The
        maximum number of iterations to perform in the line search. The number of
        iterations used to bracket the minimum are also counted against this
        parameter.
      name: (Optional) Python str. The name prefixed to the ops created by this
        function. If not supplied, the default name 'hager_zhang' is used.

    Returns:
      results: A namedtuple containing the following attributes.
        converged: Boolean `Tensor` of shape [n]. Whether a point satisfying
          Wolfe/Approx wolfe was found.
        failed: Boolean `Tensor` of shape [n]. Whether line search failed e.g.
          if either the objective function or the gradient are not finite at
          an evaluation point.
        iterations: Scalar int32 `Tensor`. Number of line search iterations made.
        func_evals: Scalar int32 `Tensor`. Number of function evaluations made.
        left: A namedtuple, as returned by value_and_gradients_function,
          of the left end point of the final bracketing interval. Values are
          equal to those of `right` on batch members where converged is True.
          Otherwise, it corresponds to the last interval computed.
        right: A namedtuple, as returned by value_and_gradients_function,
          of the right end point of the final bracketing interval. Values are
          equal to those of `left` on batch members where converged is True.
          Otherwise, it corresponds to the last interval computed.


<EOL>iter_max = np.ceil(-np.log2(_machine_eps(val_c_input.x.dtype)))<EOL>def _cond(i, val_c, to_fix):<EOL><INDENT>del val_c  <EOL>return (i < iter_max) & tf.reduce_any(input_tensor=to_fix)<EOL><DEDENT>def _body(i, val_c, to_fix):<EOL><INDENT>next_c = tf.where(to_fix, val_c.x * step_size_shrink_param, val_c.x)<EOL>next_val_c = value_and_gradients_function(next_c)<EOL>still_to_fix = to_fix & ~hzl.is_finite(next_val_c)<EOL>return (i + <NUM_LIT:1>, next_val_c, still_to_fix)<EOL><DEDENT>to_fix = active & ~hzl.is_finite(val_c_input)<EOL>return tf.while_loop(<EOL>cond=_cond, body=_body, loop_vars=(<NUM_LIT:0>, val_c_input, to_fix))<EOL>

Shrinks the input step size until the value and grad become finite.


bracket_result = hzl.bracket(value_and_gradients_function, init_interval,<EOL>f_lim, max_iterations, expansion_param)<EOL>converged = init_interval.converged | _very_close(<EOL>bracket_result.left.x, bracket_result.right.x)<EOL>exhausted_iterations = ~converged & tf.greater_equal(<EOL>bracket_result.iteration, max_iterations)<EOL>line_search_args = HagerZhangLineSearchResult(<EOL>converged=converged,<EOL>failed=bracket_result.failed | exhausted_iterations,<EOL>iterations=bracket_result.iteration,<EOL>func_evals=bracket_result.num_evals,<EOL>left=bracket_result.left,<EOL>right=bracket_result.right)<EOL>return _line_search_after_bracketing(<EOL>value_and_gradients_function, line_search_args, init_interval.left,<EOL>f_lim, max_iterations, sufficient_decrease_param, curvature_param,<EOL>shrinkage_param)<EOL>

Brackets the minimum and performs a line search.

    Args:
      value_and_gradients_function: A Python callable that accepts a real scalar
        tensor and returns a namedtuple with the fields 'x', 'f', and 'df' that
        correspond to scalar tensors of real dtype containing the point at which
        the function was evaluated, the value of the function, and its
        derivative at that point. The other namedtuple fields, if present,
        should be tensors or sequences (possibly nested) of tensors.
        In usual optimization application, this function would be generated by
        projecting the multivariate objective function along some specific
        direction. The direction is determined by some other procedure but should
        be a descent direction (i.e. the derivative of the projected univariate
        function must be negative at 0.).
        Alternatively, the function may represent the batching of `n` such line
        functions (e.g. projecting a single multivariate objective function along
        `n` distinct directions at once) accepting n points as input, i.e. a
        tensor of shape [n], and the fields 'x', 'f' and 'df' in the returned
        namedtuple should each be a tensor of shape [n], with the corresponding
        input points, function values, and derivatives at those input points.
      init_interval: Instance of `HagerZhangLineSearchResults` containing
        the initial line search interval. The gradient of init_interval.left must
        be negative (i.e. must be a descent direction), while init_interval.right
        must be positive and finite.
      f_lim: Scalar `Tensor` of float dtype.
      max_iterations: Positive scalar `Tensor` of integral dtype. The maximum
        number of iterations to perform in the line search. The number of
        iterations used to bracket the minimum are also counted against this
        parameter.
      shrinkage_param: Scalar positive Tensor of real dtype. Must be less than
        `1.`. Corresponds to the parameter `gamma` in [Hager and Zhang (2006)][2].
      expansion_param: Scalar positive `Tensor` of real dtype. Must be greater
        than `1.`. Used to expand the initial interval in case it does not bracket
        a minimum. Corresponds to `rho` in [Hager and Zhang (2006)][2].
      sufficient_decrease_param: Positive scalar `Tensor` of real dtype.
        Bounded above by the curvature param. Corresponds to `delta` in the
        terminology of [Hager and Zhang (2006)][2].
      curvature_param: Positive scalar `Tensor` of real dtype. Bounded above
        by `1.`. Corresponds to 'sigma' in the terminology of
        [Hager and Zhang (2006)][2].

    Returns:
      A namedtuple containing the following fields.
        converged: Boolean `Tensor` of shape [n]. Whether a point satisfying
          Wolfe/Approx wolfe was found.
        failed: Boolean `Tensor` of shape [n]. Whether line search failed e.g.
          if either the objective function or the gradient are not finite at
          an evaluation point.
        iterations: Scalar int32 `Tensor`. Number of line search iterations made.
        func_evals: Scalar int32 `Tensor`. Number of function evaluations made.
        left: A namedtuple, as returned by value_and_gradients_function,
          of the left end point of the updated bracketing interval.
        right: A namedtuple, as returned by value_and_gradients_function,
          of the right end point of the updated bracketing interval.


def _loop_cond(curr_interval):<EOL><INDENT>"""<STR_LIT>"""<EOL>active = ~(curr_interval.converged | curr_interval.failed)<EOL>return (curr_interval.iterations <<EOL>max_iterations) & tf.reduce_any(input_tensor=active)<EOL><DEDENT>def _loop_body(curr_interval):<EOL><INDENT>"""<STR_LIT>"""<EOL>secant2_raw_result = hzl.secant2(<EOL>value_and_gradients_function, val_0, curr_interval, f_lim,<EOL>sufficient_decrease_param, curvature_param)<EOL>secant2_result = HagerZhangLineSearchResult(<EOL>converged=secant2_raw_result.converged,<EOL>failed=secant2_raw_result.failed,<EOL>iterations=curr_interval.iterations + <NUM_LIT:1>,<EOL>func_evals=secant2_raw_result.num_evals,<EOL>left=secant2_raw_result.left,<EOL>right=secant2_raw_result.right)<EOL>should_check_shrinkage = ~(secant2_result.converged | secant2_result.failed)<EOL>def _do_check_shrinkage():<EOL><INDENT>"""<STR_LIT>"""<EOL>old_width = curr_interval.right.x - curr_interval.left.x<EOL>new_width = secant2_result.right.x - secant2_result.left.x<EOL>sufficient_shrinkage = new_width < old_width * shrinkage_param<EOL>func_is_flat = (<EOL>_very_close(curr_interval.left.f, curr_interval.right.f) &<EOL>_very_close(secant2_result.left.f, secant2_result.right.f))<EOL>new_converged = (<EOL>should_check_shrinkage & sufficient_shrinkage & func_is_flat)<EOL>needs_inner_bisect = should_check_shrinkage & ~sufficient_shrinkage<EOL>inner_bisect_args = secant2_result._replace(<EOL>converged=secant2_result.converged | new_converged)<EOL>def _apply_inner_bisect():<EOL><INDENT>return _line_search_inner_bisection(<EOL>value_and_gradients_function, inner_bisect_args,<EOL>needs_inner_bisect, f_lim)<EOL><DEDENT>return prefer_static.cond(<EOL>tf.reduce_any(input_tensor=needs_inner_bisect),<EOL>_apply_inner_bisect,<EOL>lambda: inner_bisect_args)<EOL><DEDENT>next_args = prefer_static.cond(<EOL>tf.reduce_any(input_tensor=should_check_shrinkage),<EOL>_do_check_shrinkage,<EOL>lambda: secant2_result)<EOL>interval_shrunk = (<EOL>~next_args.failed & _very_close(next_args.left.x, next_args.right.x))<EOL>return [next_args._replace(converged=next_args.converged | interval_shrunk)]<EOL><DEDENT>return tf.while_loop(<EOL>cond=_loop_cond,<EOL>body=_loop_body,<EOL>loop_vars=[search_interval],<EOL>parallel_iterations=<NUM_LIT:1>)[<NUM_LIT:0>]<EOL>

The main loop of line search after the minimum has been bracketed.

    Args:
      value_and_gradients_function: A Python callable that accepts a real scalar
        tensor and returns a namedtuple with the fields 'x', 'f', and 'df' that
        correspond to scalar tensors of real dtype containing the point at which
        the function was evaluated, the value of the function, and its
        derivative at that point. The other namedtuple fields, if present,
        should be tensors or sequences (possibly nested) of tensors.
        In usual optimization application, this function would be generated by
        projecting the multivariate objective function along some specific
        direction. The direction is determined by some other procedure but should
        be a descent direction (i.e. the derivative of the projected univariate
        function must be negative at 0.).
        Alternatively, the function may represent the batching of `n` such line
        functions (e.g. projecting a single multivariate objective function along
        `n` distinct directions at once) accepting n points as input, i.e. a
        tensor of shape [n], and the fields 'x', 'f' and 'df' in the returned
        namedtuple should each be a tensor of shape [n], with the corresponding
        input points, function values, and derivatives at those input points.
      search_interval: Instance of `HagerZhangLineSearchResults` containing
        the current line search interval.
      val_0: A namedtuple as returned by value_and_gradients_function evaluated
        at `0.`. The gradient must be negative (i.e. must be a descent direction).
      f_lim: Scalar `Tensor` of float dtype.
      max_iterations: Positive scalar `Tensor` of integral dtype. The maximum
        number of iterations to perform in the line search. The number of
        iterations used to bracket the minimum are also counted against this
        parameter.
      sufficient_decrease_param: Positive scalar `Tensor` of real dtype.
        Bounded above by the curvature param. Corresponds to `delta` in the
        terminology of [Hager and Zhang (2006)][2].
      curvature_param: Positive scalar `Tensor` of real dtype. Bounded above
        by `1.`. Corresponds to 'sigma' in the terminology of
        [Hager and Zhang (2006)][2].
      shrinkage_param: Scalar positive Tensor of real dtype. Must be less than
        `1.`. Corresponds to the parameter `gamma` in [Hager and Zhang (2006)][2].

    Returns:
      A namedtuple containing the following fields.
        converged: Boolean `Tensor` of shape [n]. Whether a point satisfying
          Wolfe/Approx wolfe was found.
        failed: Boolean `Tensor` of shape [n]. Whether line search failed e.g.
          if either the objective function or the gradient are not finite at
          an evaluation point.
        iterations: Scalar int32 `Tensor`. Number of line search iterations made.
        func_evals: Scalar int32 `Tensor`. Number of function evaluations made.
        left: A namedtuple, as returned by value_and_gradients_function,
          of the left end point of the updated bracketing interval.
        right: A namedtuple, as returned by value_and_gradients_function,
          of the right end point of the updated bracketing interval.


midpoint = (search_interval.left.x + search_interval.right.x) / <NUM_LIT:2><EOL>val_mid = value_and_gradients_function(midpoint)<EOL>is_valid_mid = hzl.is_finite(val_mid)<EOL>still_active = active & is_valid_mid<EOL>new_failed = active & ~is_valid_mid<EOL>next_inteval = search_interval._replace(<EOL>failed=search_interval.failed | new_failed,<EOL>func_evals=search_interval.func_evals + <NUM_LIT:1>)<EOL>def _apply_update():<EOL><INDENT>update_result = hzl.update(<EOL>value_and_gradients_function, next_inteval.left, next_inteval.right,<EOL>val_mid, f_lim, active=still_active)<EOL>return HagerZhangLineSearchResult(<EOL>converged=next_inteval.converged,<EOL>failed=next_inteval.failed | update_result.failed,<EOL>iterations=next_inteval.iterations + update_result.iteration,<EOL>func_evals=next_inteval.func_evals + update_result.num_evals,<EOL>left=update_result.left,<EOL>right=update_result.right)<EOL><DEDENT>return prefer_static.cond(<EOL>tf.reduce_any(input_tensor=still_active),<EOL>_apply_update,<EOL>lambda: next_inteval)<EOL>

Performs bisection and updates the interval.


eval_count = <NUM_LIT:0><EOL>if val_initial is None:<EOL><INDENT>if initial_step_size is not None:<EOL><INDENT>initial_step_size = tf.convert_to_tensor(value=initial_step_size)<EOL><DEDENT>else:<EOL><INDENT>initial_step_size = tf.convert_to_tensor(value=<NUM_LIT:1.0>, dtype=tf.float32)<EOL><DEDENT>val_initial = value_and_gradients_function(initial_step_size)<EOL>eval_count += <NUM_LIT:1><EOL><DEDENT>if val_0 is None:<EOL><INDENT>x_0 = tf.zeros_like(val_initial.x)<EOL>val_0 = value_and_gradients_function(x_0)<EOL>eval_count += <NUM_LIT:1><EOL><DEDENT>f_lim = val_0.f + (approximate_wolfe_threshold * tf.abs(val_0.f))<EOL>return val_0, val_initial, f_lim, tf.convert_to_tensor(value=eval_count)<EOL>

Prepares the arguments for the line search initialization.

    Args:
      value_and_gradients_function: A Python callable that accepts a real scalar
        tensor and returns a namedtuple with the fields 'x', 'f', and 'df' that
        correspond to scalar tensors of real dtype containing the point at which
        the function was evaluated, the value of the function, and its
        derivative at that point. The other namedtuple fields, if present,
        should be tensors or sequences (possibly nested) of tensors.
        In usual optimization application, this function would be generated by
        projecting the multivariate objective function along some specific
        direction. The direction is determined by some other procedure but should
        be a descent direction (i.e. the derivative of the projected univariate
        function must be negative at 0.).
        Alternatively, the function may represent the batching of `n` such line
        functions (e.g. projecting a single multivariate objective function along
        `n` distinct directions at once) accepting n points as input, i.e. a
        tensor of shape [n], and the fields 'x', 'f' and 'df' in the returned
        namedtuple should each be a tensor of shape [n], with the corresponding
        input points, function values, and derivatives at those input points.
      initial_step_size: Scalar positive `Tensor` of real dtype, or a tensor of
        shape [n] in batching mode. The initial value (or values) to try to
        bracket the minimum. Default is `1.` as a float32.
        Note that this point need not necessarily bracket the minimum for the line
        search to work correctly but the supplied value must be greater than 0.
        A good initial value will make the search converge faster.
      val_initial: The full return value of evaluating
        value_and_gradients_function at initial_step_size, i.e. a namedtuple with
        'x', 'f', 'df', if already known by the caller. If not None the value of
        `initial_step_size` will be ignored, otherwise the tuple will be computed
        by evaluating value_and_gradients_function.
      val_0: The full return value of value_and_gradients_function at `0.`, i.e.
        a namedtuple with 'x', 'f', 'df', if already known by the caller. If None
        the tuple will be computed by evaluating value_and_gradients_function.
      approximate_wolfe_threshold: Scalar positive `Tensor` of
        real dtype. Corresponds to the parameter 'epsilon' in
        [Hager and Zhang (2006)][2]. Used to estimate the
        threshold at which the line search switches to approximate Wolfe
        conditions.

    Returns:
      left: A namedtuple, as returned by value_and_gradients_function,
        containing the value and derivative of the function at `0.`.
      val_initial: A namedtuple, as returned by value_and_gradients_function,
        containing the value and derivative of the function at
        `initial_step_size`.
      f_lim: Real `Tensor` of shape [n]. The function value threshold for
        the approximate Wolfe conditions to be checked.
      eval_count: Scalar int32 `Tensor`. The number of target function
        evaluations made by this function.


x = tf.convert_to_tensor(value=x)<EOL>if x.dtype == tf.bool:<EOL><INDENT>return tf.where(x, tf.fill(x.shape, '<STR_LIT:True>'), tf.fill(x.shape, '<STR_LIT:False>'))<EOL><DEDENT>return x<EOL>

Converts a bool tensor to a string with True/False values.


flat_values = []<EOL>for value in values:<EOL><INDENT>if hasattr(value, '<STR_LIT>'):<EOL><INDENT>for field in value._fields:<EOL><INDENT>flat_values.extend([field, _to_str(getattr(value, field))])<EOL><DEDENT>continue<EOL><DEDENT>if isinstance(value, (list, tuple)):<EOL><INDENT>for v in value:<EOL><INDENT>flat_values.append(_to_str(v))<EOL><DEDENT>continue<EOL><DEDENT>flat_values.append(_to_str(value))<EOL><DEDENT>return tf.compat.v1.Print(pass_through_tensor, flat_values)<EOL>

Wrapper for tf.Print which supports lists and namedtuples for printing.


if initial_inverse_hessian_estimate is not None:<EOL><INDENT>raise NotImplementedError(<EOL>'<STR_LIT>')<EOL><DEDENT>if stopping_condition is None:<EOL><INDENT>stopping_condition = bfgs_utils.converged_all<EOL><DEDENT>with tf.compat.v1.name_scope(name, '<STR_LIT>', [initial_position, tolerance]):<EOL><INDENT>initial_position = tf.convert_to_tensor(<EOL>value=initial_position, name='<STR_LIT>')<EOL>dtype = initial_position.dtype.base_dtype<EOL>tolerance = tf.convert_to_tensor(<EOL>value=tolerance, dtype=dtype, name='<STR_LIT>')<EOL>f_relative_tolerance = tf.convert_to_tensor(<EOL>value=f_relative_tolerance, dtype=dtype, name='<STR_LIT>')<EOL>x_tolerance = tf.convert_to_tensor(<EOL>value=x_tolerance, dtype=dtype, name='<STR_LIT>')<EOL>max_iterations = tf.convert_to_tensor(<EOL>value=max_iterations, name='<STR_LIT>')<EOL>def _cond(state):<EOL><INDENT>"""<STR_LIT>"""<EOL>return ((state.num_iterations < max_iterations) &<EOL>tf.logical_not(stopping_condition(state.converged, state.failed)))<EOL><DEDENT>def _body(current_state):<EOL><INDENT>"""<STR_LIT>"""<EOL>search_direction = _get_search_direction(current_state)<EOL>next_state = bfgs_utils.line_search_step(<EOL>current_state,<EOL>value_and_gradients_function, search_direction,<EOL>tolerance, f_relative_tolerance, x_tolerance, stopping_condition)<EOL>should_update = ~(next_state.converged | next_state.failed)<EOL>state_after_inv_hessian_update = bfgs_utils.update_fields(<EOL>next_state,<EOL>position_deltas=_queue_push(<EOL>current_state.position_deltas, should_update,<EOL>next_state.position - current_state.position),<EOL>gradient_deltas=_queue_push(<EOL>current_state.gradient_deltas, should_update,<EOL>next_state.objective_gradient - current_state.objective_gradient))<EOL>return [state_after_inv_hessian_update]<EOL><DEDENT>initial_state = _get_initial_state(value_and_gradients_function,<EOL>initial_position,<EOL>num_correction_pairs,<EOL>tolerance)<EOL>return tf.while_loop(<EOL>cond=_cond,<EOL>body=_body,<EOL>loop_vars=[initial_state],<EOL>parallel_iterations=parallel_iterations)[<NUM_LIT:0>]<EOL><DEDENT>

Applies the L-BFGS algorithm to minimize a differentiable function.

    Performs unconstrained minimization of a differentiable function using the
    L-BFGS scheme. See [Nocedal and Wright(2006)][1] for details of the algorithm.

    ### Usage:

    The following example demonstrates the L-BFGS optimizer attempting to find the
    minimum for a simple high-dimensional quadratic objective function.

    ```python
      # A high-dimensional quadratic bowl.
      ndims = 60
      minimum = np.ones([ndims], dtype='float64')
      scales = np.arange(ndims, dtype='float64') + 1.0

      # The objective function and the gradient.
      def quadratic(x):
        value = tf.reduce_sum(scales * (x - minimum) ** 2)
        return value, tf.gradients(value, x)[0]

      start = np.arange(ndims, 0, -1, dtype='float64')
      optim_results = tfp.optimizer.lbfgs_minimize(
          quadratic, initial_position=start, num_correction_pairs=10,
          tolerance=1e-8)

      with tf.Session() as session:
        results = session.run(optim_results)
        # Check that the search converged
        assert(results.converged)
        # Check that the argmin is close to the actual value.
        np.testing.assert_allclose(results.position, minimum)
    ```

    ### References:

    [1] Jorge Nocedal, Stephen Wright. Numerical Optimization. Springer Series
        in Operations Research. pp 176-180. 2006

    http://pages.mtu.edu/~struther/Courses/OLD/Sp2013/5630/Jorge_Nocedal_Numerical_optimization_267490.pdf

    Args:
      value_and_gradients_function:  A Python callable that accepts a point as a
        real `Tensor` and returns a tuple of `Tensor`s of real dtype containing
        the value of the function and its gradient at that point. The function
        to be minimized. The input is of shape `[..., n]`, where `n` is the size
        of the domain of input points, and all others are batching dimensions.
        The first component of the return value is a real `Tensor` of matching
        shape `[...]`. The second component (the gradient) is also of shape
        `[..., n]` like the input value to the function.
      initial_position: Real `Tensor` of shape `[..., n]`. The starting point, or
        points when using batching dimensions, of the search procedure. At these
        points the function value and the gradient norm should be finite.
      num_correction_pairs: Positive integer. Specifies the maximum number of
        (position_delta, gradient_delta) correction pairs to keep as implicit
        approximation of the Hessian matrix.
      tolerance: Scalar `Tensor` of real dtype. Specifies the gradient tolerance
        for the procedure. If the supremum norm of the gradient vector is below
        this number, the algorithm is stopped.
      x_tolerance: Scalar `Tensor` of real dtype. If the absolute change in the
        position between one iteration and the next is smaller than this number,
        the algorithm is stopped.
      f_relative_tolerance: Scalar `Tensor` of real dtype. If the relative change
        in the objective value between one iteration and the next is smaller
        than this value, the algorithm is stopped.
      initial_inverse_hessian_estimate: None. Option currently not supported.
      max_iterations: Scalar positive int32 `Tensor`. The maximum number of
        iterations for L-BFGS updates.
      parallel_iterations: Positive integer. The number of iterations allowed to
        run in parallel.
      stopping_condition: (Optional) A Python function that takes as input two
        Boolean tensors of shape `[...]`, and returns a Boolean scalar tensor.
        The input tensors are `converged` and `failed`, indicating the current
        status of each respective batch member; the return value states whether
        the algorithm should stop. The default is tfp.optimizer.converged_all
        which only stops when all batch members have either converged or failed.
        An alternative is tfp.optimizer.converged_any which stops as soon as one
        batch member has converged, or when all have failed.
      name: (Optional) Python str. The name prefixed to the ops created by this
        function. If not supplied, the default name 'minimize' is used.

    Returns:
      optimizer_results: A namedtuple containing the following items:
        converged: Scalar boolean tensor indicating whether the minimum was
          found within tolerance.
        failed:  Scalar boolean tensor indicating whether a line search
          step failed to find a suitable step size satisfying Wolfe
          conditions. In the absence of any constraints on the
          number of objective evaluations permitted, this value will
          be the complement of `converged`. However, if there is
          a constraint and the search stopped due to available
          evaluations being exhausted, both `failed` and `converged`
          will be simultaneously False.
        num_objective_evaluations: The total number of objective
          evaluations performed.
        position: A tensor containing the last argument value found
          during the search. If the search converged, then
          this value is the argmin of the objective function.
        objective_value: A tensor containing the value of the objective
          function at the `position`. If the search converged, then this is
          the (local) minimum of the objective function.
        objective_gradient: A tensor containing the gradient of the objective
          function at the `position`. If the search converged the
          max-norm of this tensor should be below the tolerance.
        position_deltas: A tensor encoding information about the latest
          changes in `position` during the algorithm execution.
        gradient_deltas: A tensor encoding information about the latest
          changes in `objective_gradient` during the algorithm execution.


init_args = bfgs_utils.get_initial_state_args(<EOL>value_and_gradients_function,<EOL>initial_position,<EOL>tolerance)<EOL>empty_queue = _make_empty_queue_for(num_correction_pairs, initial_position)<EOL>init_args.update(position_deltas=empty_queue, gradient_deltas=empty_queue)<EOL>return LBfgsOptimizerResults(**init_args)<EOL>

Create LBfgsOptimizerResults with initial state of search procedure.


<EOL>num_elements = tf.minimum(<EOL>state.num_iterations,<EOL>distribution_util.prefer_static_shape(state.position_deltas)[<NUM_LIT:0>])<EOL>def _two_loop_algorithm():<EOL><INDENT>"""<STR_LIT>"""<EOL>position_deltas = state.position_deltas[-num_elements:]<EOL>gradient_deltas = state.gradient_deltas[-num_elements:]<EOL>inv_rhos = tf.reduce_sum(<EOL>input_tensor=gradient_deltas * position_deltas, axis=-<NUM_LIT:1>)<EOL>def first_loop(acc, args):<EOL><INDENT>_, q_direction = acc<EOL>position_delta, gradient_delta, inv_rho = args<EOL>alpha = tf.reduce_sum(<EOL>input_tensor=position_delta * q_direction, axis=-<NUM_LIT:1>) / inv_rho<EOL>direction_delta = tf.expand_dims(alpha, axis=-<NUM_LIT:1>) * gradient_delta<EOL>return (alpha, q_direction - direction_delta)<EOL><DEDENT>zero = tf.zeros_like(inv_rhos[<NUM_LIT:0>])<EOL>alphas, q_directions = tf.scan(<EOL>first_loop, [position_deltas, gradient_deltas, inv_rhos],<EOL>initializer=(zero, state.objective_gradient), reverse=True)<EOL>gamma_k = inv_rhos[-<NUM_LIT:1>] / tf.reduce_sum(<EOL>input_tensor=gradient_deltas[-<NUM_LIT:1>] * gradient_deltas[-<NUM_LIT:1>], axis=-<NUM_LIT:1>)<EOL>r_direction = tf.expand_dims(gamma_k, axis=-<NUM_LIT:1>) * q_directions[<NUM_LIT:0>]<EOL>def second_loop(r_direction, args):<EOL><INDENT>alpha, position_delta, gradient_delta, inv_rho = args<EOL>beta = tf.reduce_sum(<EOL>input_tensor=gradient_delta * r_direction, axis=-<NUM_LIT:1>) / inv_rho<EOL>direction_delta = tf.expand_dims(alpha - beta, axis=-<NUM_LIT:1>) * position_delta<EOL>return r_direction + direction_delta<EOL><DEDENT>r_directions = tf.scan(<EOL>second_loop, [alphas, position_deltas, gradient_deltas, inv_rhos],<EOL>initializer=r_direction)<EOL>return -r_directions[-<NUM_LIT:1>]<EOL><DEDENT>return prefer_static.cond(tf.equal(num_elements, <NUM_LIT:0>),<EOL>(lambda: -state.objective_gradient),<EOL>_two_loop_algorithm)<EOL>

Computes the search direction to follow at the current state.

    On the `k`-th iteration of the main L-BFGS algorithm, the state has collected
    the most recent `m` correction pairs in position_deltas and gradient_deltas,
    where `k = state.num_iterations` and `m = min(k, num_correction_pairs)`.

    Assuming these, the code below is an implementation of the L-BFGS two-loop
    recursion algorithm given by [Nocedal and Wright(2006)][1]:

    ```None
      q_direction = objective_gradient
      for i in reversed(range(m)):  # First loop.
        inv_rho[i] = gradient_deltas[i]^T * position_deltas[i]
        alpha[i] = position_deltas[i]^T * q_direction / inv_rho[i]
        q_direction = q_direction - alpha[i] * gradient_deltas[i]

      kth_inv_hessian_factor = (gradient_deltas[-1]^T * position_deltas[-1] /
                                gradient_deltas[-1]^T * gradient_deltas[-1])
      r_direction = kth_inv_hessian_factor * I * q_direction

      for i in range(m):  # Second loop.
        beta = gradient_deltas[i]^T * r_direction / inv_rho[i]
        r_direction = r_direction + position_deltas[i] * (alpha[i] - beta)

      return -r_direction  # Approximates - H_k * objective_gradient.
    ```

    Args:
      state: A `LBfgsOptimizerResults` tuple with the current state of the
        search procedure.

    Returns:
      A real `Tensor` of the same shape as the `state.position`. The direction
      along which to perform line search.


queue_shape = tf.concat(<EOL>[[k], distribution_util.prefer_static_shape(element)], axis=<NUM_LIT:0>)<EOL>return tf.zeros(queue_shape, dtype=element.dtype.base_dtype)<EOL>

Creates a `tf.Tensor` suitable to hold `k` element-shaped tensors.

    For example:

    ```python
      element = tf.constant([[0., 1., 2., 3., 4.],
                             [5., 6., 7., 8., 9.]])

      # A queue capable of holding 3 elements.
      _make_empty_queue_for(3, element)
      # => [[[ 0.,  0.,  0.,  0.,  0.],
      #      [ 0.,  0.,  0.,  0.,  0.]],
      #
      #     [[ 0.,  0.,  0.,  0.,  0.],
      #      [ 0.,  0.,  0.,  0.,  0.]],
      #
      #     [[ 0.,  0.,  0.,  0.,  0.],
      #      [ 0.,  0.,  0.,  0.,  0.]]]
    ```

    Args:
      k: A positive scalar integer, number of elements that each queue will hold.
      element: A `tf.Tensor`, only its shape and dtype information are relevant.

    Returns:
      A zero-filed `tf.Tensor` of shape `(k,) + tf.shape(element)` and same dtype
      as `element`.


def _queue_push(queue, should_update, new_vecs):

new_queue = tf.concat([queue[<NUM_LIT:1>:], [new_vecs]], axis=<NUM_LIT:0>)<EOL>update_pattern = tf.broadcast_to(<EOL>should_update[tf.newaxis, ..., tf.newaxis],<EOL>distribution_util.prefer_static_shape(queue))<EOL>return tf.where(update_pattern, new_queue, queue)<EOL>

Conditionally push new vectors into a batch of first-in-first-out queues.

    The `queue` of shape `[k, ..., n]` can be thought of as a batch of queues,
    each holding `k` n-D vectors; while `new_vecs` of shape `[..., n]` is a
    fresh new batch of n-D vectors. The `should_update` batch of Boolean scalars,
    i.e. shape `[...]`, indicates batch members whose corresponding n-D vector in
    `new_vecs` should be added at the back of its queue, pushing out the
    corresponding n-D vector from the front. Batch members in `new_vecs` for
    which `should_update` is False are ignored.

    Note: the choice of placing `k` at the dimension 0 of the queue is
    constrained by the L-BFGS two-loop algorithm above. The algorithm uses
    tf.scan to iterate over the `k` correction pairs simulatneously across all
    batches, and tf.scan itself can only iterate over dimension 0.

    For example:

    ```python
      k, b, n = (3, 2, 5)
      queue = tf.reshape(tf.range(30), (k, b, n))
      # => [[[ 0,  1,  2,  3,  4],
      #      [ 5,  6,  7,  8,  9]],
      #
      #     [[10, 11, 12, 13, 14],
      #      [15, 16, 17, 18, 19]],
      #
      #     [[20, 21, 22, 23, 24],
      #      [25, 26, 27, 28, 29]]]

      element = tf.reshape(tf.range(30, 40), (b, n))
      # => [[30, 31, 32, 33, 34],
            [35, 36, 37, 38, 39]]

      should_update = tf.constant([True, False])  # Shape: (b,)

      _queue_add(should_update, queue, element)
      # => [[[10, 11, 12, 13, 14],
      #      [ 5,  6,  7,  8,  9]],
      #
      #     [[20, 21, 22, 23, 24],
      #      [15, 16, 17, 18, 19]],
      #
      #     [[30, 31, 32, 33, 34],
      #      [25, 26, 27, 28, 29]]]
    ```

    Args:
      queue: A `tf.Tensor` of shape `[k, ..., n]`; a batch of queues each with
        `k` n-D vectors.
      should_update: A Boolean `tf.Tensor` of shape `[...]` indicating batch
        members where new vectors should be added to their queues.
      new_vecs: A `tf.Tensor` of shape `[..., n]`; a batch of n-D vectors to add
        at the end of their respective queues, pushing out the first element from
        each.

    Returns:
      A new `tf.Tensor` of shape `[k, ..., n]`.


return self._variable_scope<EOL>

Variable scope of all calls to `tf.get_variable`.


with tf.compat.v1.name_scope(name, '<STR_LIT>', [<EOL>initial_simplex, initial_vertex, step_sizes, objective_at_initial_simplex,<EOL>objective_at_initial_vertex, func_tolerance, position_tolerance<EOL>]):<EOL><INDENT>(<EOL>dim,<EOL>_,<EOL>simplex,<EOL>objective_at_simplex,<EOL>num_evaluations<EOL>) = _prepare_args(objective_function,<EOL>initial_simplex,<EOL>initial_vertex,<EOL>step_sizes,<EOL>objective_at_initial_simplex,<EOL>objective_at_initial_vertex,<EOL>batch_evaluate_objective)<EOL>domain_dtype = simplex.dtype<EOL>(<EOL>reflection,<EOL>expansion,<EOL>contraction,<EOL>shrinkage<EOL>) = _resolve_parameters(dim,<EOL>reflection,<EOL>expansion,<EOL>contraction,<EOL>shrinkage,<EOL>domain_dtype)<EOL>closure_kwargs = dict(<EOL>objective_function=objective_function,<EOL>dim=dim,<EOL>func_tolerance=func_tolerance,<EOL>position_tolerance=position_tolerance,<EOL>batch_evaluate_objective=batch_evaluate_objective,<EOL>reflection=reflection,<EOL>expansion=expansion,<EOL>contraction=contraction,<EOL>shrinkage=shrinkage)<EOL>def _loop_body(_, iterations, simplex, objective_at_simplex,<EOL>num_evaluations):<EOL><INDENT>(<EOL>converged,<EOL>next_simplex,<EOL>next_objective,<EOL>evaluations<EOL>) = nelder_mead_one_step(simplex, objective_at_simplex, **closure_kwargs)<EOL>return (converged, iterations + <NUM_LIT:1>, next_simplex, next_objective,<EOL>num_evaluations + evaluations)<EOL><DEDENT>initial_args = (False, <NUM_LIT:0>, simplex, objective_at_simplex,<EOL>num_evaluations)<EOL>def _is_converged(converged, num_iterations, *ignored_args):  <EOL><INDENT>not_converged = tf.logical_not(converged)<EOL>return (not_converged if max_iterations is None<EOL>else (not_converged & (num_iterations < max_iterations)))<EOL><DEDENT>(converged, num_iterations, final_simplex, final_objective_values,<EOL>final_evaluations) = tf.while_loop(<EOL>cond=_is_converged,<EOL>body=_loop_body,<EOL>loop_vars=initial_args,<EOL>parallel_iterations=parallel_iterations)<EOL>order = tf.argsort(<EOL>final_objective_values, direction='<STR_LIT>', stable=True)<EOL>best_index = order[<NUM_LIT:0>]<EOL>return NelderMeadOptimizerResults(<EOL>converged=tf.convert_to_tensor(value=converged),<EOL>num_objective_evaluations=final_evaluations,<EOL>position=final_simplex[best_index],<EOL>objective_value=final_objective_values[best_index],<EOL>final_simplex=final_simplex,<EOL>final_objective_values=final_objective_values,<EOL>num_iterations=tf.convert_to_tensor(value=num_iterations),<EOL>initial_simplex=simplex,<EOL>initial_objective_values=objective_at_simplex)<EOL><DEDENT>

Minimum of the objective function using the Nelder Mead simplex algorithm.

    Performs an unconstrained minimization of a (possibly non-smooth) function
    using the Nelder Mead simplex method. Nelder Mead method does not support
    univariate functions. Hence the dimensions of the domain must be 2 or greater.
    For details of the algorithm, see
    [Press, Teukolsky, Vetterling and Flannery(2007)][1].

    Points in the domain of the objective function may be represented as a
    `Tensor` of general shape but with rank at least 1. The algorithm proceeds
    by modifying a full rank simplex in the domain. The initial simplex may
    either be specified by the user or can be constructed using a single vertex
    supplied by the user. In the latter case, if `v0` is the supplied vertex,
    the simplex is the convex hull of the set:

    ```None
    S = {v0} + {v0 + step_i * e_i}
    ```

    Here `e_i` is a vector which is `1` along the `i`-th axis and zero elsewhere
    and `step_i` is a characteristic length scale along the `i`-th axis. If the
    step size is not supplied by the user, a unit step size is used in every axis.
    Alternately, a single step size may be specified which is used for every
    axis. The most flexible option is to supply a bespoke step size for every
    axis.

    ### Usage:

    The following example demonstrates the usage of the Nelder Mead minimzation
    on a two dimensional problem with the minimum located at a non-differentiable
    point.

    ```python
      # The objective function
      def sqrt_quadratic(x):
        return tf.sqrt(tf.reduce_sum(x ** 2, axis=-1))

      start = tf.constant([6.0, -21.0])  # Starting point for the search.
      optim_results = tfp.optimizer.nelder_mead_minimize(
          sqrt_quadratic, initial_vertex=start, func_tolerance=1e-8,
          batch_evaluate_objective=True)

      with tf.Session() as session:
        results = session.run(optim_results)
        # Check that the search converged
        assert(results.converged)
        # Check that the argmin is close to the actual value.
        np.testing.assert_allclose(results.position, np.array([0.0, 0.0]),
                                   atol=1e-7)
        # Print out the total number of function evaluations it took.
        print ("Function evaluations: %d" % results.num_objective_evaluations)
    ```

    ### References:
    [1]: William Press, Saul Teukolsky, William Vetterling and Brian Flannery.
      Numerical Recipes in C++, third edition. pp. 502-507. (2007).
      http://numerical.recipes/cpppages/chap0sel.pdf

    [2]: Jeffrey Lagarias, James Reeds, Margaret Wright and Paul Wright.
      Convergence properties of the Nelder-Mead simplex method in low dimensions,
      Siam J. Optim., Vol 9, No. 1, pp. 112-147. (1998).
      http://www.math.kent.edu/~reichel/courses/Opt/reading.material.2/nelder.mead.pdf

    [3]: Fuchang Gao and Lixing Han. Implementing the Nelder-Mead simplex
      algorithm with adaptive parameters. Computational Optimization and
      Applications, Vol 51, Issue 1, pp 259-277. (2012).
      https://pdfs.semanticscholar.org/15b4/c4aa7437df4d032c6ee6ce98d6030dd627be.pdf

    Args:
      objective_function:  A Python callable that accepts a point as a
        real `Tensor` and returns a `Tensor` of real dtype containing
        the value of the function at that point. The function
        to be minimized. If `batch_evaluate_objective` is `True`, the callable
        may be evaluated on a `Tensor` of shape `[n+1] + s ` where `n` is
        the dimension of the problem and `s` is the shape of a single point
        in the domain (so `n` is the size of a `Tensor` representing a
        single point).
        In this case, the expected return value is a `Tensor` of shape `[n+1]`.
        Note that this method does not support univariate functions so the problem
        dimension `n` must be strictly greater than 1.
      initial_simplex: (Optional) `Tensor` of real dtype. The initial simplex to
        start the search. If supplied, should be a `Tensor` of shape `[n+1] + s`
        where `n` is the dimension of the problem and `s` is the shape of a
        single point in the domain. Each row (i.e. the `Tensor` with a given
        value of the first index) is interpreted as a vertex of a simplex and
        hence the rows must be affinely independent. If not supplied, an axes
        aligned simplex is constructed using the `initial_vertex` and
        `step_sizes`. Only one and at least one of `initial_simplex` and
        `initial_vertex` must be supplied.
      initial_vertex: (Optional) `Tensor` of real dtype and any shape that can
        be consumed by the `objective_function`. A single point in the domain that
        will be used to construct an axes aligned initial simplex.
      step_sizes: (Optional) `Tensor` of real dtype and shape broadcasting
        compatible with `initial_vertex`. Supplies the simplex scale along each
        axes. Only used if `initial_simplex` is not supplied. See description
        above for details on how step sizes and initial vertex are used to
        construct the initial simplex.
      objective_at_initial_simplex: (Optional) Rank `1` `Tensor` of real dtype
        of a rank `1` `Tensor`. The value of the objective function at the
        initial simplex. May be supplied only if `initial_simplex` is
        supplied. If not supplied, it will be computed.
      objective_at_initial_vertex: (Optional) Scalar `Tensor` of real dtype. The
        value of the objective function at the initial vertex. May be supplied
        only if the `initial_vertex` is also supplied.
      batch_evaluate_objective: (Optional) Python `bool`. If True, the objective
        function will be evaluated on all the vertices of the simplex packed
        into a single tensor. If False, the objective will be mapped across each
        vertex separately. Evaluating the objective function in a batch allows
        use of vectorization and should be preferred if the objective function
        allows it.
      func_tolerance: (Optional) Scalar `Tensor` of real dtype. The algorithm
        stops if the absolute difference between the largest and the smallest
        function value on the vertices of the simplex is below this number.
      position_tolerance: (Optional) Scalar `Tensor` of real dtype. The
        algorithm stops if the largest absolute difference between the
        coordinates of the vertices is below this threshold.
      parallel_iterations: (Optional) Positive integer. The number of iterations
        allowed to run in parallel.
      max_iterations: (Optional) Scalar positive `Tensor` of dtype `int32`.
        The maximum number of iterations allowed. If `None` then no limit is
        applied.
      reflection: (Optional) Positive Scalar `Tensor` of same dtype as
        `initial_vertex`. This parameter controls the scaling of the reflected
        vertex. See, [Press et al(2007)][1] for details. If not specified,
        uses the dimension dependent prescription of [Gao and Han(2012)][3].
      expansion: (Optional) Positive Scalar `Tensor` of same dtype as
        `initial_vertex`. Should be greater than `1` and `reflection`. This
        parameter controls the expanded scaling of a reflected vertex.
        See, [Press et al(2007)][1] for details. If not specified, uses the
        dimension dependent prescription of [Gao and Han(2012)][3].
      contraction: (Optional) Positive scalar `Tensor` of same dtype as
        `initial_vertex`. Must be between `0` and `1`. This parameter controls
        the contraction of the reflected vertex when the objective function at
        the reflected point fails to show sufficient decrease.
        See, [Press et al(2007)][1] for more details. If not specified, uses
        the dimension dependent prescription of [Gao and Han(2012][3].
      shrinkage: (Optional) Positive scalar `Tensor` of same dtype as
        `initial_vertex`. Must be between `0` and `1`. This parameter is the scale
        by which the simplex is shrunk around the best point when the other
        steps fail to produce improvements.
        See, [Press et al(2007)][1] for more details. If not specified, uses
        the dimension dependent prescription of [Gao and Han(2012][3].
      name: (Optional) Python str. The name prefixed to the ops created by this
        function. If not supplied, the default name 'minimize' is used.

    Returns:
      optimizer_results: A namedtuple containing the following items:
        converged: Scalar boolean tensor indicating whether the minimum was
          found within tolerance.
        num_objective_evaluations: The total number of objective
          evaluations performed.
        position: A `Tensor` containing the last argument value found
          during the search. If the search converged, then
          this value is the argmin of the objective function.
        objective_value: A tensor containing the value of the objective
          function at the `position`. If the search
          converged, then this is the (local) minimum of
          the objective function.
        final_simplex: The last simplex constructed before stopping.
        final_objective_values: The objective function evaluated at the
          vertices of the final simplex.
        initial_simplex: The starting simplex.
        initial_objective_values: The objective function evaluated at the
          vertices of the initial simplex.
        num_iterations: The number of iterations of the main algorithm body.

    Raises:
      ValueError: If any of the following conditions hold
        1. If none or more than one of `initial_simplex` and `initial_vertex` are
          supplied.
        2. If `initial_simplex` and `step_sizes` are both specified.


with tf.compat.v1.name_scope(name, '<STR_LIT>'):<EOL><INDENT>domain_dtype = current_simplex.dtype.base_dtype<EOL>order = tf.argsort(<EOL>current_objective_values, direction='<STR_LIT>', stable=True)<EOL>(<EOL>best_index,<EOL>worst_index,<EOL>second_worst_index<EOL>) = order[<NUM_LIT:0>], order[-<NUM_LIT:1>], order[-<NUM_LIT:2>]<EOL>worst_vertex = current_simplex[worst_index]<EOL>(<EOL>best_objective_value,<EOL>worst_objective_value,<EOL>second_worst_objective_value<EOL>) = (<EOL>current_objective_values[best_index],<EOL>current_objective_values[worst_index],<EOL>current_objective_values[second_worst_index]<EOL>)<EOL>face_centroid = tf.reduce_sum(<EOL>input_tensor=current_simplex, axis=<NUM_LIT:0>) - worst_vertex<EOL>face_centroid /= tf.cast(dim, domain_dtype)<EOL>reflected = face_centroid + reflection * (face_centroid - worst_vertex)<EOL>objective_at_reflected = objective_function(reflected)<EOL>num_evaluations = <NUM_LIT:1><EOL>has_converged = _check_convergence(current_simplex,<EOL>current_simplex[best_index],<EOL>best_objective_value,<EOL>worst_objective_value,<EOL>func_tolerance,<EOL>position_tolerance)<EOL>def _converged_fn():<EOL><INDENT>return (True, current_simplex, current_objective_values, <NUM_LIT:0>)<EOL><DEDENT>case0 = has_converged, _converged_fn<EOL>accept_reflected = (<EOL>(objective_at_reflected < second_worst_objective_value) &<EOL>(objective_at_reflected >= best_objective_value))<EOL>accept_reflected_fn = _accept_reflected_fn(current_simplex,<EOL>current_objective_values,<EOL>worst_index,<EOL>reflected,<EOL>objective_at_reflected)<EOL>case1 = accept_reflected, accept_reflected_fn<EOL>do_expansion = objective_at_reflected < best_objective_value<EOL>expansion_fn = _expansion_fn(objective_function,<EOL>current_simplex,<EOL>current_objective_values,<EOL>worst_index,<EOL>reflected,<EOL>objective_at_reflected,<EOL>face_centroid,<EOL>expansion)<EOL>case2 = do_expansion, expansion_fn<EOL>do_outside_contraction = (<EOL>(objective_at_reflected < worst_objective_value) &<EOL>(objective_at_reflected >= second_worst_objective_value)<EOL>)<EOL>outside_contraction_fn = _outside_contraction_fn(<EOL>objective_function,<EOL>current_simplex,<EOL>current_objective_values,<EOL>face_centroid,<EOL>best_index,<EOL>worst_index,<EOL>reflected,<EOL>objective_at_reflected,<EOL>contraction,<EOL>shrinkage,<EOL>batch_evaluate_objective)<EOL>case3 = do_outside_contraction, outside_contraction_fn<EOL>default_fn = _inside_contraction_fn(objective_function,<EOL>current_simplex,<EOL>current_objective_values,<EOL>face_centroid,<EOL>best_index,<EOL>worst_index,<EOL>worst_objective_value,<EOL>contraction,<EOL>shrinkage,<EOL>batch_evaluate_objective)<EOL>(<EOL>converged,<EOL>next_simplex,<EOL>next_objective_at_simplex,<EOL>case_evals) = prefer_static.case([case0, case1, case2, case3],<EOL>default=default_fn, exclusive=False)<EOL>next_simplex.set_shape(current_simplex.shape)<EOL>next_objective_at_simplex.set_shape(current_objective_values.shape)<EOL>return (<EOL>converged,<EOL>next_simplex,<EOL>next_objective_at_simplex,<EOL>num_evaluations + case_evals<EOL>)<EOL><DEDENT>

A single iteration of the Nelder Mead algorithm.


def _replace_worst_with_reflected():<EOL><INDENT>next_simplex = _replace_at_index(simplex, worst_index, reflected)<EOL>next_objective_values = _replace_at_index(objective_values, worst_index,<EOL>objective_at_reflected)<EOL>return False, next_simplex, next_objective_values, <NUM_LIT:0><EOL><DEDENT>return _replace_worst_with_reflected<EOL>

Creates the condition function pair for a reflection to be accepted.


def _expand_and_maybe_replace():<EOL><INDENT>"""<STR_LIT>"""<EOL>expanded = face_centroid + expansion * (reflected - face_centroid)<EOL>expanded_objective_value = objective_function(expanded)<EOL>expanded_is_better = (expanded_objective_value <<EOL>objective_at_reflected)<EOL>accept_expanded_fn = lambda: (expanded, expanded_objective_value)<EOL>accept_reflected_fn = lambda: (reflected, objective_at_reflected)<EOL>next_pt, next_objective_value = prefer_static.cond(<EOL>expanded_is_better, accept_expanded_fn, accept_reflected_fn)<EOL>next_simplex = _replace_at_index(simplex, worst_index, next_pt)<EOL>next_objective_at_simplex = _replace_at_index(objective_values,<EOL>worst_index,<EOL>next_objective_value)<EOL>return False, next_simplex, next_objective_at_simplex, <NUM_LIT:1><EOL><DEDENT>return _expand_and_maybe_replace<EOL>

Creates the condition function pair for an expansion.


def _contraction():<EOL><INDENT>"""<STR_LIT>"""<EOL>contracted = face_centroid + contraction * (reflected - face_centroid)<EOL>objective_at_contracted = objective_function(contracted)<EOL>is_contracted_acceptable = objective_at_contracted <= objective_at_reflected<EOL>def _accept_contraction():<EOL><INDENT>next_simplex = _replace_at_index(simplex, worst_index, contracted)<EOL>objective_at_next_simplex = _replace_at_index(<EOL>objective_values,<EOL>worst_index,<EOL>objective_at_contracted)<EOL>return (False,<EOL>next_simplex,<EOL>objective_at_next_simplex,<EOL><NUM_LIT:1>)<EOL><DEDENT>def _reject_contraction():<EOL><INDENT>return _shrink_towards_best(objective_function,<EOL>simplex,<EOL>best_index,<EOL>shrinkage,<EOL>batch_evaluate_objective)<EOL><DEDENT>return prefer_static.cond(is_contracted_acceptable,<EOL>_accept_contraction,<EOL>_reject_contraction)<EOL><DEDENT>return _contraction<EOL>

Creates the condition function pair for an outside contraction.


def _contraction():<EOL><INDENT>"""<STR_LIT>"""<EOL>contracted = face_centroid - contraction * (face_centroid -<EOL>simplex[worst_index])<EOL>objective_at_contracted = objective_function(contracted)<EOL>is_contracted_acceptable = objective_at_contracted <= worst_objective_value<EOL>def _accept_contraction():<EOL><INDENT>next_simplex = _replace_at_index(simplex, worst_index, contracted)<EOL>objective_at_next_simplex = _replace_at_index(<EOL>objective_values,<EOL>worst_index,<EOL>objective_at_contracted)<EOL>return (<EOL>False,<EOL>next_simplex,<EOL>objective_at_next_simplex,<EOL><NUM_LIT:1><EOL>)<EOL><DEDENT>def _reject_contraction():<EOL><INDENT>return _shrink_towards_best(objective_function, simplex, best_index,<EOL>shrinkage, batch_evaluate_objective)<EOL><DEDENT>return prefer_static.cond(is_contracted_acceptable,<EOL>_accept_contraction,<EOL>_reject_contraction)<EOL><DEDENT>return _contraction<EOL>

Creates the condition function pair for an inside contraction.


<EOL>best_vertex = simplex[best_index]<EOL>shrunk_simplex = best_vertex + shrinkage * (simplex - best_vertex)<EOL>objective_at_shrunk_simplex, evals = _evaluate_objective_multiple(<EOL>objective_function,<EOL>shrunk_simplex,<EOL>batch_evaluate_objective)<EOL>return (False,<EOL>shrunk_simplex,<EOL>objective_at_shrunk_simplex,<EOL>evals)<EOL>

Shrinks the simplex around the best vertex.


def _replace_at_index(x, index, replacement):

x_new = tf.concat([x[:index], tf.expand_dims(replacement, axis=<NUM_LIT:0>),<EOL>x[(index + <NUM_LIT:1>):]], axis=<NUM_LIT:0>)<EOL>return x_new<EOL>


objective_convergence = tf.abs(worst_objective -<EOL>best_objective) < func_tolerance<EOL>simplex_degeneracy = tf.reduce_max(<EOL>input_tensor=tf.abs(simplex - best_vertex)) < position_tolerance<EOL>return objective_convergence | simplex_degeneracy<EOL>

Returns True if the simplex has converged.

    If the simplex size is smaller than the `position_tolerance` or the variation
    of the function value over the vertices of the simplex is smaller than the
    `func_tolerance` return True else False.

    Args:
      simplex: `Tensor` of real dtype. The simplex to test for convergence. For
        more details, see the docstring for `initial_simplex` argument
        of `minimize`.
      best_vertex: `Tensor` of real dtype and rank one less than `simplex`. The
        vertex with the best (i.e. smallest) objective value.
      best_objective: Scalar `Tensor` of real dtype. The best (i.e. smallest)
        value of the objective function at a vertex.
      worst_objective: Scalar `Tensor` of same dtype as `best_objective`. The
        worst (i.e. largest) value of the objective function at a vertex.
      func_tolerance: Scalar positive `Tensor`. The tolerance for the variation
        of the objective function value over the simplex. If the variation over
        the simplex vertices is below this threshold, convergence is True.
      position_tolerance: Scalar positive `Tensor`. The algorithm stops if the
        lengths (under the supremum norm) of edges connecting to the best vertex
        are below this threshold.

    Returns:
      has_converged: A scalar boolean `Tensor` indicating whether the algorithm
        is deemed to have converged.