if split_fn is None: split_fn = lambda _ : '<STR_LIT>'<EOL>if group_fn is None: group_fn = lambda _ : '<STR_LIT>'<EOL>sk2r = defaultdict(list) <EOL>for result in allresults:<EOL><INDENT>splitkey = split_fn(result)<EOL>sk2r[splitkey].append(result)<EOL><DEDENT>assert len(sk2r) > <NUM_LIT:0><EOL>assert isinstance(resample, int), "<STR_LIT>"<EOL>nrows = len(sk2r)<EOL>ncols = <NUM_LIT:1><EOL>figsize = figsize or (<NUM_LIT:6>, <NUM_LIT:6> * nrows)<EOL>f, axarr = plt.subplots(nrows, ncols, sharex=False, squeeze=False, figsize=figsize)<EOL>groups = list(set(group_fn(result) for result in allresults))<EOL>default_samples = <NUM_LIT><EOL>if average_group:<EOL><INDENT>resample = resample or default_samples<EOL><DEDENT>for (isplit, sk) in enumerate(sorted(sk2r.keys())):<EOL><INDENT>g2l = {}<EOL>g2c = defaultdict(int)<EOL>sresults = sk2r[sk]<EOL>gresults = defaultdict(list)<EOL>ax = axarr[isplit][<NUM_LIT:0>]<EOL>for result in sresults:<EOL><INDENT>group = group_fn(result)<EOL>g2c[group] += <NUM_LIT:1><EOL>x, y = xy_fn(result)<EOL>if x is None: x = np.arange(len(y))<EOL>x, y = map(np.asarray, (x, y))<EOL>if average_group:<EOL><INDENT>gresults[group].append((x,y))<EOL><DEDENT>else:<EOL><INDENT>if resample:<EOL><INDENT>x, y, counts = symmetric_ema(x, y, x[<NUM_LIT:0>], x[-<NUM_LIT:1>], resample, decay_steps=smooth_step)<EOL><DEDENT>l, = ax.plot(x, y, color=COLORS[groups.index(group) % len(COLORS)])<EOL>g2l[group] = l<EOL><DEDENT><DEDENT>if average_group:<EOL><INDENT>for group in sorted(groups):<EOL><INDENT>xys = gresults[group]<EOL>if not any(xys):<EOL><INDENT>continue<EOL><DEDENT>color = COLORS[groups.index(group) % len(COLORS)]<EOL>origxs = [xy[<NUM_LIT:0>] for xy in xys]<EOL>minxlen = min(map(len, origxs))<EOL>def allequal(qs):<EOL><INDENT>return all((q==qs[<NUM_LIT:0>]).all() for q in qs[<NUM_LIT:1>:])<EOL><DEDENT>if resample:<EOL><INDENT>low  = max(x[<NUM_LIT:0>] for x in origxs)<EOL>high = min(x[-<NUM_LIT:1>] for x in origxs)<EOL>usex = np.linspace(low, high, resample)<EOL>ys = []<EOL>for (x, y) in xys:<EOL><INDENT>ys.append(symmetric_ema(x, y, low, high, resample, decay_steps=smooth_step)[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert allequal([x[:minxlen] for x in origxs]),'<STR_LIT>'<EOL>usex = origxs[<NUM_LIT:0>]<EOL>ys = [xy[<NUM_LIT:1>][:minxlen] for xy in xys]<EOL><DEDENT>ymean = np.mean(ys, axis=<NUM_LIT:0>)<EOL>ystd = np.std(ys, axis=<NUM_LIT:0>)<EOL>ystderr = ystd / np.sqrt(len(ys))<EOL>l, = axarr[isplit][<NUM_LIT:0>].plot(usex, ymean, color=color)<EOL>g2l[group] = l<EOL>if shaded_err:<EOL><INDENT>ax.fill_between(usex, ymean - ystderr, ymean + ystderr, color=color, alpha=<NUM_LIT>)<EOL><DEDENT>if shaded_std:<EOL><INDENT>ax.fill_between(usex, ymean - ystd,    ymean + ystd,    color=color, alpha=<NUM_LIT>)<EOL><DEDENT><DEDENT><DEDENT>plt.tight_layout()<EOL>if any(g2l.keys()):<EOL><INDENT>ax.legend(<EOL>g2l.values(),<EOL>['<STR_LIT>'%(g, g2c[g]) for g in g2l] if average_group else g2l.keys(),<EOL>loc=<NUM_LIT:2> if legend_outside else None,<EOL>bbox_to_anchor=(<NUM_LIT:1>,<NUM_LIT:1>) if legend_outside else None)<EOL><DEDENT>ax.set_title(sk)<EOL><DEDENT>return f, axarr<EOL>

Plot multiple Results objects

xy_fn: function Result -> x,y           - function that converts results objects into tuple of x and y values.
                                          By default, x is cumsum of episode lengths, and y is episode rewards

split_fn: function Result -> hashable   - function that converts results objects into keys to split curves into sub-panels by.
                                          That is, the results r for which split_fn(r) is different will be put on different sub-panels.
                                          By default, the portion of r.dirname between last / and -<digits> is returned. The sub-panels are
                                          stacked vertically in the figure.

group_fn: function Result -> hashable   - function that converts results objects into keys to group curves by.
                                          That is, the results r for which group_fn(r) is the same will be put into the same group.
                                          Curves in the same group have the same color (if average_group is False), or averaged over
                                          (if average_group is True). The default value is the same as default value for split_fn

average_group: bool                     - if True, will average the curves in the same group and plot the mean. Enables resampling
                                          (if resample = 0, will use 512 steps)

shaded_std: bool                        - if True (default), the shaded region corresponding to standard deviation of the group of curves will be
                                          shown (only applicable if average_group = True)

shaded_err: bool                        - if True (default), the shaded region corresponding to error in mean estimate of the group of curves
                                          (that is, standard deviation divided by square root of number of curves) will be
                                          shown (only applicable if average_group = True)

figsize: tuple or None                  - size of the resulting figure (including sub-panels). By default, width is 6 and height is 6 times number of
                                          sub-panels.

legend_outside: bool                    - if True, will place the legend outside of the sub-panels.

resample: int                           - if not zero, size of the uniform grid in x direction to resample onto. Resampling is performed via symmetric
                                          EMA smoothing (see the docstring for symmetric_ema).
                                          Default is zero (no resampling). Note that if average_group is True, resampling is necessary; in that case, default
                                          value is 512.

smooth_step: float                      - when resampling (i.e. when resample > 0 or average_group is True), use this EMA decay parameter (in units of the new grid step).
                                          See docstrings for decay_steps in symmetric_ema or one_sided_ema functions.


def cg(f_Ax, b, cg_iters=<NUM_LIT:10>, callback=None, verbose=False, residual_tol=<NUM_LIT>):

p = b.copy()<EOL>r = b.copy()<EOL>x = np.zeros_like(b)<EOL>rdotr = r.dot(r)<EOL>fmtstr =  "<STR_LIT>"<EOL>titlestr =  "<STR_LIT>"<EOL>if verbose: print(titlestr % ("<STR_LIT>", "<STR_LIT>", "<STR_LIT>"))<EOL>for i in range(cg_iters):<EOL><INDENT>if callback is not None:<EOL><INDENT>callback(x)<EOL><DEDENT>if verbose: print(fmtstr % (i, rdotr, np.linalg.norm(x)))<EOL>z = f_Ax(p)<EOL>v = rdotr / p.dot(z)<EOL>x += v*p<EOL>r -= v*z<EOL>newrdotr = r.dot(r)<EOL>mu = newrdotr/rdotr<EOL>p = r + mu*p<EOL>rdotr = newrdotr<EOL>if rdotr < residual_tol:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if callback is not None:<EOL><INDENT>callback(x)<EOL><DEDENT>if verbose: print(fmtstr % (i+<NUM_LIT:1>, rdotr, np.linalg.norm(x)))  <EOL>return x<EOL>


def sync_from_root(sess, variables, comm=None):

if comm is None: comm = MPI.COMM_WORLD<EOL>import tensorflow as tf<EOL>values = comm.bcast(sess.run(variables))<EOL>sess.run([tf.assign(var, val)<EOL>for (var, val) in zip(variables, values)])<EOL>

Send the root node's parameters to every worker.
Arguments:
  sess: the TensorFlow session.
  variables: all parameter variables including optimizer's


if shutil.which('<STR_LIT>') is None:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>output = subprocess.check_output(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>return max(<NUM_LIT:0>, len(output.split(b'<STR_LIT:\n>')) - <NUM_LIT:2>)<EOL>


if '<STR_LIT>' not in os.environ:<EOL><INDENT>if sys.platform == '<STR_LIT>': <EOL><INDENT>ids = []                 <EOL><DEDENT>else:<EOL><INDENT>lrank, _lsize = get_local_rank_size(MPI.COMM_WORLD)<EOL>ids = [lrank]<EOL><DEDENT>os.environ["<STR_LIT>"] = "<STR_LIT:U+002C>".join(map(str, ids))<EOL><DEDENT>

Set CUDA_VISIBLE_DEVICES to MPI rank if not already set


this_node = platform.node()<EOL>ranks_nodes = comm.allgather((comm.Get_rank(), this_node))<EOL>node2rankssofar = defaultdict(int)<EOL>local_rank = None<EOL>for (rank, node) in ranks_nodes:<EOL><INDENT>if rank == comm.Get_rank():<EOL><INDENT>local_rank = node2rankssofar[node]<EOL><DEDENT>node2rankssofar[node] += <NUM_LIT:1><EOL><DEDENT>assert local_rank is not None<EOL>return local_rank, node2rankssofar[this_node]<EOL>

Returns the rank of each process on its machine
The processes on a given machine will be assigned ranks
    0, 1, 2, ..., N-1,
where N is the number of processes on this machine.

Useful if you want to assign one gpu per machine


localrank, _ = get_local_rank_size(comm)<EOL>if comm.Get_rank() == <NUM_LIT:0>:<EOL><INDENT>with open(path, '<STR_LIT:rb>') as fh:<EOL><INDENT>data = fh.read()<EOL><DEDENT>comm.bcast(data)<EOL><DEDENT>else:<EOL><INDENT>data = comm.bcast(None)<EOL>if localrank == <NUM_LIT:0>:<EOL><INDENT>os.makedirs(os.path.dirname(path), exist_ok=True)<EOL>with open(path, '<STR_LIT:wb>') as fh:<EOL><INDENT>fh.write(data)<EOL><DEDENT><DEDENT><DEDENT>comm.Barrier()<EOL>

Copies the file from rank 0 to all other ranks
Puts it in the same place on all machines


def dict_gather(comm, d, op='<STR_LIT>', assert_all_have_data=True):

if comm is None: return d<EOL>alldicts = comm.allgather(d)<EOL>size = comm.size<EOL>k2li = defaultdict(list)<EOL>for d in alldicts:<EOL><INDENT>for (k,v) in d.items():<EOL><INDENT>k2li[k].append(v)<EOL><DEDENT><DEDENT>result = {}<EOL>for (k,li) in k2li.items():<EOL><INDENT>if assert_all_have_data:<EOL><INDENT>assert len(li)==size, "<STR_LIT>" % (len(li), size, k)<EOL><DEDENT>if op=='<STR_LIT>':<EOL><INDENT>result[k] = np.mean(li, axis=<NUM_LIT:0>)<EOL><DEDENT>elif op=='<STR_LIT>':<EOL><INDENT>result[k] = np.sum(li, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>assert <NUM_LIT:0>, op<EOL><DEDENT><DEDENT>return result<EOL>

Perform a reduction operation over dicts


all_name2valcount = comm.gather(local_name2valcount)<EOL>if comm.rank == <NUM_LIT:0>:<EOL><INDENT>name2sum = defaultdict(float)<EOL>name2count = defaultdict(float)<EOL>for n2vc in all_name2valcount:<EOL><INDENT>for (name, (val, count)) in n2vc.items():<EOL><INDENT>try:<EOL><INDENT>val = float(val)<EOL><DEDENT>except ValueError:<EOL><INDENT>if comm.rank == <NUM_LIT:0>:<EOL><INDENT>warnings.warn('<STR_LIT>'.format(name, val))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>name2sum[name] += val * count<EOL>name2count[name] += count<EOL><DEDENT><DEDENT><DEDENT>return {name : name2sum[name] / name2count[name] for name in name2sum}<EOL><DEDENT>else:<EOL><INDENT>return {}<EOL><DEDENT>

Perform a weighted average over dicts that are each on a different node
Input: local_name2valcount: dict mapping key -> (value, count)
Returns: key -> mean


def __init__(self, env, observations, latent, estimate_q=False, vf_latent=None, sess=None, **tensors):

self.X = observations<EOL>self.state = tf.constant([])<EOL>self.initial_state = None<EOL>self.__dict__.update(tensors)<EOL>vf_latent = vf_latent if vf_latent is not None else latent<EOL>vf_latent = tf.layers.flatten(vf_latent)<EOL>latent = tf.layers.flatten(latent)<EOL>self.pdtype = make_pdtype(env.action_space)<EOL>self.pd, self.pi = self.pdtype.pdfromlatent(latent, init_scale=<NUM_LIT>)<EOL>self.action = self.pd.sample()<EOL>self.neglogp = self.pd.neglogp(self.action)<EOL>self.sess = sess or tf.get_default_session()<EOL>if estimate_q:<EOL><INDENT>assert isinstance(env.action_space, gym.spaces.Discrete)<EOL>self.q = fc(vf_latent, '<STR_LIT:q>', env.action_space.n)<EOL>self.vf = self.q<EOL><DEDENT>else:<EOL><INDENT>self.vf = fc(vf_latent, '<STR_LIT>', <NUM_LIT:1>)<EOL>self.vf = self.vf[:,<NUM_LIT:0>]<EOL><DEDENT>

Parameters:
----------
env             RL environment

observations    tensorflow placeholder in which the observations will be fed

latent          latent state from which policy distribution parameters should be inferred

vf_latent       latent state from which value function should be inferred (if None, then latent is used)

sess            tensorflow session to run calculations in (if None, default session is used)

**tensors       tensorflow tensors for additional attributes such as state or mask


def step(self, observation, **extra_feed):

a, v, state, neglogp = self._evaluate([self.action, self.vf, self.state, self.neglogp], observation, **extra_feed)<EOL>if state.size == <NUM_LIT:0>:<EOL><INDENT>state = None<EOL><DEDENT>return a, v, state, neglogp<EOL>

Compute next action(s) given the observation(s)

Parameters:
----------

observation     observation data (either single or a batch)

**extra_feed    additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__)

Returns:
-------
(action, value estimate, next state, negative log likelihood of the action under current policy parameters) tuple


return self._evaluate(self.vf, ob, *args, **kwargs)<EOL>

Compute value estimate(s) given the observation(s)

Parameters:
----------

observation     observation data (either single or a batch)

**extra_feed    additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__)

Returns:
-------
value estimate


raise NotImplementedError()<EOL>


self._v = value<EOL>

Value remains constant over time.

        Parameters
        ----------
        value: float
            Constant value of the schedule


return self._v<EOL>


def __init__(self, endpoints, interpolation=linear_interpolation, outside_value=None):

idxes = [e[<NUM_LIT:0>] for e in endpoints]<EOL>assert idxes == sorted(idxes)<EOL>self._interpolation = interpolation<EOL>self._outside_value = outside_value<EOL>self._endpoints = endpoints<EOL>

Piecewise schedule.

        endpoints: [(int, int)]
            list of pairs `(time, value)` meanining that schedule should output
            `value` when `t==time`. All the values for time must be sorted in
            an increasing order. When t is between two times, e.g. `(time_a, value_a)`
            and `(time_b, value_b)`, such that `time_a <= t < time_b` then value outputs
            `interpolation(value_a, value_b, alpha)` where alpha is a fraction of
            time passed between `time_a` and `time_b` for time `t`.
        interpolation: lambda float, float, float: float
            a function that takes value to the left and to the right of t according
            to the `endpoints`. Alpha is the fraction of distance from left endpoint to
            right endpoint that t has covered. See linear_interpolation for example.
        outside_value: float
            if the value is requested outside of all the intervals sepecified in
            `endpoints` this value is returned. If None then AssertionError is
            raised when outside value is requested.


for (l_t, l), (r_t, r) in zip(self._endpoints[:-<NUM_LIT:1>], self._endpoints[<NUM_LIT:1>:]):<EOL><INDENT>if l_t <= t and t < r_t:<EOL><INDENT>alpha = float(t - l_t) / (r_t - l_t)<EOL>return self._interpolation(l, r, alpha)<EOL><DEDENT><DEDENT>assert self._outside_value is not None<EOL>return self._outside_value<EOL>


def __init__(self, schedule_timesteps, final_p, initial_p=<NUM_LIT:1.0>):

self.schedule_timesteps = schedule_timesteps<EOL>self.final_p = final_p<EOL>self.initial_p = initial_p<EOL>

Linear interpolation between initial_p and final_p over
        schedule_timesteps. After this many timesteps pass final_p is
        returned.

        Parameters
        ----------
        schedule_timesteps: int
            Number of timesteps for which to linearly anneal initial_p
            to final_p
        initial_p: float
            initial output value
        final_p: float
            final output value


fraction = min(float(t) / self.schedule_timesteps, <NUM_LIT:1.0>)<EOL>return self.initial_p + fraction * (self.final_p - self.initial_p)<EOL>


img_nhwc = np.asarray(img_nhwc)<EOL>N, h, w, c = img_nhwc.shape<EOL>H = int(np.ceil(np.sqrt(N)))<EOL>W = int(np.ceil(float(N)/H))<EOL>img_nhwc = np.array(list(img_nhwc) + [img_nhwc[<NUM_LIT:0>]*<NUM_LIT:0> for _ in range(N, H*W)])<EOL>img_HWhwc = img_nhwc.reshape(H, W, h, w, c)<EOL>img_HhWwc = img_HWhwc.transpose(<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>, <NUM_LIT:3>, <NUM_LIT:4>)<EOL>img_Hh_Ww_c = img_HhWwc.reshape(H*h, W*w, c)<EOL>return img_Hh_Ww_c<EOL>

Tile N images into one big PxQ image
(P,Q) are chosen to be as close as possible, and if N
is square, then P=Q.

input: img_nhwc, list or array of images, ndim=4 once turned into array
    n = batch index, h = height, w = width, c = channel
returns:
    bigim_HWc, ndarray with ndim=3


scaled_images = tf.cast(unscaled_images, tf.float32) / <NUM_LIT><EOL>activ = tf.nn.relu<EOL>h = activ(conv(scaled_images, '<STR_LIT>', nf=<NUM_LIT:32>, rf=<NUM_LIT:8>, stride=<NUM_LIT:4>, init_scale=np.sqrt(<NUM_LIT:2>),<EOL>**conv_kwargs))<EOL>h2 = activ(conv(h, '<STR_LIT>', nf=<NUM_LIT:64>, rf=<NUM_LIT:4>, stride=<NUM_LIT:2>, init_scale=np.sqrt(<NUM_LIT:2>), **conv_kwargs))<EOL>h3 = activ(conv(h2, '<STR_LIT>', nf=<NUM_LIT:64>, rf=<NUM_LIT:3>, stride=<NUM_LIT:1>, init_scale=np.sqrt(<NUM_LIT:2>), **conv_kwargs))<EOL>h3 = conv_to_fc(h3)<EOL>return activ(fc(h3, '<STR_LIT>', nh=<NUM_LIT>, init_scale=np.sqrt(<NUM_LIT:2>)))<EOL>


@register("<STR_LIT>")<EOL>def mlp(num_layers=<NUM_LIT:2>, num_hidden=<NUM_LIT:64>, activation=tf.tanh, layer_norm=False):

def network_fn(X):<EOL><INDENT>h = tf.layers.flatten(X)<EOL>for i in range(num_layers):<EOL><INDENT>h = fc(h, '<STR_LIT>'.format(i), nh=num_hidden, init_scale=np.sqrt(<NUM_LIT:2>))<EOL>if layer_norm:<EOL><INDENT>h = tf.contrib.layers.layer_norm(h, center=True, scale=True)<EOL><DEDENT>h = activation(h)<EOL><DEDENT>return h<EOL><DEDENT>return network_fn<EOL>

Stack of fully-connected layers to be used in a policy / q-function approximator

Parameters:
----------

num_layers: int                 number of fully-connected layers (default: 2)

num_hidden: int                 size of fully-connected layers (default: 64)

activation:                     activation function (default: tf.tanh)

Returns:
-------

function that builds fully connected network with a given input tensor / placeholder


def network_fn(X, nenv=<NUM_LIT:1>):<EOL><INDENT>nbatch = X.shape[<NUM_LIT:0>]<EOL>nsteps = nbatch // nenv<EOL>h = tf.layers.flatten(X)<EOL>M = tf.placeholder(tf.float32, [nbatch]) <EOL>S = tf.placeholder(tf.float32, [nenv, <NUM_LIT:2>*nlstm]) <EOL>xs = batch_to_seq(h, nenv, nsteps)<EOL>ms = batch_to_seq(M, nenv, nsteps)<EOL>if layer_norm:<EOL><INDENT>h5, snew = utils.lnlstm(xs, ms, S, scope='<STR_LIT>', nh=nlstm)<EOL><DEDENT>else:<EOL><INDENT>h5, snew = utils.lstm(xs, ms, S, scope='<STR_LIT>', nh=nlstm)<EOL><DEDENT>h = seq_to_batch(h5)<EOL>initial_state = np.zeros(S.shape.as_list(), dtype=float)<EOL>return h, {'<STR_LIT:S>':S, '<STR_LIT:M>':M, '<STR_LIT:state>':snew, '<STR_LIT>':initial_state}<EOL><DEDENT>return network_fn<EOL>

Builds LSTM (Long-Short Term Memory) network to be used in a policy.
Note that the resulting function returns not only the output of the LSTM
(i.e. hidden state of lstm for each step in the sequence), but also a dictionary
with auxiliary tensors to be set as policy attributes.

Specifically,
    S is a placeholder to feed current state (LSTM state has to be managed outside policy)
    M is a placeholder for the mask (used to mask out observations after the end of the episode, but can be used for other purposes too)
    initial_state is a numpy array containing initial lstm state (usually zeros)
    state is the output LSTM state (to be fed into S at the next call)

An example of usage of lstm-based policy can be found here: common/tests/test_doc_examples.py/test_lstm_example

Parameters:
----------

nlstm: int          LSTM hidden state size

layer_norm: bool    if True, layer-normalized version of LSTM is used

Returns:
-------

function that builds LSTM with a given input tensor / placeholder


@register("<STR_LIT>")<EOL>def conv_only(convs=[(<NUM_LIT:32>, <NUM_LIT:8>, <NUM_LIT:4>), (<NUM_LIT:64>, <NUM_LIT:4>, <NUM_LIT:2>), (<NUM_LIT:64>, <NUM_LIT:3>, <NUM_LIT:1>)], **conv_kwargs):

def network_fn(X):<EOL><INDENT>out = tf.cast(X, tf.float32) / <NUM_LIT><EOL>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>for num_outputs, kernel_size, stride in convs:<EOL><INDENT>out = layers.convolution2d(out,<EOL>num_outputs=num_outputs,<EOL>kernel_size=kernel_size,<EOL>stride=stride,<EOL>activation_fn=tf.nn.relu,<EOL>**conv_kwargs)<EOL><DEDENT><DEDENT>return out<EOL><DEDENT>return network_fn<EOL>

convolutions-only net

Parameters:
----------

conv:       list of triples (filter_number, filter_size, stride) specifying parameters for each layer.

Returns:

function that takes tensorflow tensor as input and returns the output of the last convolutional layer


if callable(name):<EOL><INDENT>return name<EOL><DEDENT>elif name in mapping:<EOL><INDENT>return mapping[name]<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(name))<EOL><DEDENT>

If you want to register your own network outside models.py, you just need:

Usage Example:
-------------
from baselines.common.models import register
@register("your_network_name")
def your_network_define(**net_kwargs):
    ...
    return network_fn


minutes_left = seconds_left // <NUM_LIT><EOL>seconds_left %= <NUM_LIT><EOL>hours_left = minutes_left // <NUM_LIT><EOL>minutes_left %= <NUM_LIT><EOL>days_left = hours_left // <NUM_LIT><EOL>hours_left %= <NUM_LIT><EOL>def helper(cnt, name):<EOL><INDENT>return "<STR_LIT>".format(str(cnt), name, ('<STR_LIT:s>' if cnt > <NUM_LIT:1> else '<STR_LIT>'))<EOL><DEDENT>if days_left > <NUM_LIT:0>:<EOL><INDENT>msg = helper(days_left, '<STR_LIT>')<EOL>if hours_left > <NUM_LIT:0>:<EOL><INDENT>msg += '<STR_LIT>' + helper(hours_left, '<STR_LIT>')<EOL><DEDENT>return msg<EOL><DEDENT>if hours_left > <NUM_LIT:0>:<EOL><INDENT>msg = helper(hours_left, '<STR_LIT>')<EOL>if minutes_left > <NUM_LIT:0>:<EOL><INDENT>msg += '<STR_LIT>' + helper(minutes_left, '<STR_LIT>')<EOL><DEDENT>return msg<EOL><DEDENT>if minutes_left > <NUM_LIT:0>:<EOL><INDENT>return helper(minutes_left, '<STR_LIT>')<EOL><DEDENT>return '<STR_LIT>'<EOL>

Print the number of seconds in human readable format.

    Examples:
    2 days
    2 hours and 37 minutes
    less than a minute

    Paramters
    ---------
    seconds_left: int
        Number of seconds to be converted to the ETA
    Returns
    -------
    eta: str
        String representing the pretty ETA.


def boolean_flag(parser, name, default=False, help=None):

dest = name.replace('<STR_LIT:->', '<STR_LIT:_>')<EOL>parser.add_argument("<STR_LIT>" + name, action="<STR_LIT:store_true>", default=default, dest=dest, help=help)<EOL>parser.add_argument("<STR_LIT>" + name, action="<STR_LIT>", dest=dest)<EOL>

Add a boolean flag to argparse parser.

    Parameters
    ----------
    parser: argparse.Parser
        parser to add the flag to
    name: str
        --<name> will enable the flag, while --no-<name> will disable it
    default: bool or None
        default value of the flag
    help: str
        help string for the flag


currentenv = env<EOL>while True:<EOL><INDENT>if classname == currentenv.class_name():<EOL><INDENT>return currentenv<EOL><DEDENT>elif isinstance(currentenv, gym.Wrapper):<EOL><INDENT>currentenv = currentenv.env<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % classname)<EOL><DEDENT><DEDENT>

Given an a gym environment possibly wrapped multiple times, returns a wrapper
    of class named classname or raises ValueError if no such wrapper was applied

    Parameters
    ----------
    env: gym.Env of gym.Wrapper
        gym environment
    classname: str
        name of the wrapper

    Returns
    -------
    wrapper: gym.Wrapper
        wrapper named classname


def relatively_safe_pickle_dump(obj, path, compression=False):

temp_storage = path + "<STR_LIT>"<EOL>if compression:<EOL><INDENT>with tempfile.NamedTemporaryFile() as uncompressed_file:<EOL><INDENT>pickle.dump(obj, uncompressed_file)<EOL>uncompressed_file.file.flush()<EOL>with zipfile.ZipFile(temp_storage, "<STR_LIT:w>", compression=zipfile.ZIP_DEFLATED) as myzip:<EOL><INDENT>myzip.write(uncompressed_file.name, "<STR_LIT:data>")<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>with open(temp_storage, "<STR_LIT:wb>") as f:<EOL><INDENT>pickle.dump(obj, f)<EOL><DEDENT><DEDENT>os.rename(temp_storage, path)<EOL>

This is just like regular pickle dump, except from the fact that failure cases are
    different:

        - It's never possible that we end up with a pickle in corrupted state.
        - If a there was a different file at the path, that file will remain unchanged in the
          even of failure (provided that filesystem rename is atomic).
        - it is sometimes possible that we end up with useless temp file which needs to be
          deleted manually (it will be removed automatically on the next function call)

    The indended use case is periodic checkpoints of experiment state, such that we never
    corrupt previous checkpoints if the current one fails.

    Parameters
    ----------
    obj: object
        object to pickle
    path: str
        path to the output file
    compression: bool
        if true pickle will be compressed


if compression:<EOL><INDENT>with zipfile.ZipFile(path, "<STR_LIT:r>", compression=zipfile.ZIP_DEFLATED) as myzip:<EOL><INDENT>with myzip.open("<STR_LIT:data>") as f:<EOL><INDENT>return pickle.load(f)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>with open(path, "<STR_LIT:rb>") as f:<EOL><INDENT>return pickle.load(f)<EOL><DEDENT><DEDENT>

Unpickle a possible compressed pickle.

    Parameters
    ----------
    path: str
        path to the output file
    compression: bool
        if true assumes that pickle was compressed when created and attempts decompression.

    Returns
    -------
    obj: object
        the unpickled object


def __init__(self, gamma, init_value=None):

self._value = init_value<EOL>self._gamma = gamma<EOL>

Keep a running estimate of a quantity. This is a bit like mean
        but more sensitive to recent changes.

        Parameters
        ----------
        gamma: float
            Must be between 0 and 1, where 0 is the most sensitive to recent
            changes.
        init_value: float or None
            Initial value of the estimate. If None, it will be set on the first update.


if self._value is None:<EOL><INDENT>self._value = new_val<EOL><DEDENT>else:<EOL><INDENT>self._value = self._gamma * self._value + (<NUM_LIT:1.0> - self._gamma) * new_val<EOL><DEDENT>

Update the estimate.

        Parameters
        ----------
        new_val: float
            new observated value of estimated quantity.


return self._value<EOL>


def wrap_deepmind_retro(env, scale=True, frame_stack=<NUM_LIT:4>):

env = WarpFrame(env)<EOL>env = ClipRewardEnv(env)<EOL>if frame_stack > <NUM_LIT:1>:<EOL><INDENT>env = FrameStack(env, frame_stack)<EOL><DEDENT>if scale:<EOL><INDENT>env = ScaledFloatFrame(env)<EOL><DEDENT>return env<EOL>

Configure environment for retro games, using config similar to DeepMind-style Atari in wrap_deepmind


def __init__(self, env, k, channel=<NUM_LIT:1>):

gym.Wrapper.__init__(self, env)<EOL>shp = env.observation_space.shape<EOL>self.channel = channel<EOL>self.observation_space = gym.spaces.Box(low=<NUM_LIT:0>, high=<NUM_LIT:255>,<EOL>shape=(shp[<NUM_LIT:0>], shp[<NUM_LIT:1>], shp[<NUM_LIT:2>] + k - <NUM_LIT:1>),<EOL>dtype=env.observation_space.dtype)<EOL>self.k = k<EOL>self.frames = deque([], maxlen=k)<EOL>shp = env.observation_space.shape<EOL>

Stack one channel (channel keyword) from previous frames


gym.ObservationWrapper.__init__(self, env)<EOL>(oldh, oldw, oldc) = env.observation_space.shape<EOL>newshape = (oldh//ratio, oldw//ratio, oldc)<EOL>self.observation_space = gym.spaces.Box(low=<NUM_LIT:0>, high=<NUM_LIT:255>,<EOL>shape=newshape, dtype=np.uint8)<EOL>


gym.ObservationWrapper.__init__(self, env)<EOL>(oldh, oldw, _oldc) = env.observation_space.shape<EOL>self.observation_space = gym.spaces.Box(low=<NUM_LIT:0>, high=<NUM_LIT:255>,<EOL>shape=(oldh, oldw, <NUM_LIT:1>), dtype=np.uint8)<EOL>


get_current().logkv(key, val)<EOL>

Log a value of some diagnostic
Call this once for each diagnostic quantity, each iteration
If called many times, last value will be used.


get_current().logkv_mean(key, val)<EOL>

The same as logkv(), but if called many times, values averaged.


for (k, v) in d.items():<EOL><INDENT>logkv(k, v)<EOL><DEDENT>


return get_current().dumpkvs()<EOL>

Write all of the diagnostics from the current iteration


get_current().log(*args, level=level)<EOL>

Write the sequence of args, with no separators, to the console and output files (if you've configured an output file).


Set logging threshold on current logger.


return get_current().get_dir()<EOL>

Get directory that log files are being written to.
will be None if there is no output directory (i.e., if you didn't call start)


def decorator_with_name(func):<EOL><INDENT>def func_wrapper(*args, **kwargs):<EOL><INDENT>with profile_kv(n):<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT><DEDENT>return func_wrapper<EOL><DEDENT>return decorator_with_name<EOL>

Usage:
@profile("my_func")
def my_func(): code


def configure(dir=None, format_strs=None, comm=None):

if dir is None:<EOL><INDENT>dir = os.getenv('<STR_LIT>')<EOL><DEDENT>if dir is None:<EOL><INDENT>dir = osp.join(tempfile.gettempdir(),<EOL>datetime.datetime.now().strftime("<STR_LIT>"))<EOL><DEDENT>assert isinstance(dir, str)<EOL>os.makedirs(dir, exist_ok=True)<EOL>log_suffix = '<STR_LIT>'<EOL>rank = <NUM_LIT:0><EOL>for varname in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if varname in os.environ:<EOL><INDENT>rank = int(os.environ[varname])<EOL><DEDENT><DEDENT>if rank > <NUM_LIT:0>:<EOL><INDENT>log_suffix = "<STR_LIT>" % rank<EOL><DEDENT>if format_strs is None:<EOL><INDENT>if rank == <NUM_LIT:0>:<EOL><INDENT>format_strs = os.getenv('<STR_LIT>', '<STR_LIT>').split('<STR_LIT:U+002C>')<EOL><DEDENT>else:<EOL><INDENT>format_strs = os.getenv('<STR_LIT>', '<STR_LIT>').split('<STR_LIT:U+002C>')<EOL><DEDENT><DEDENT>format_strs = filter(None, format_strs)<EOL>output_formats = [make_output_format(f, dir, log_suffix) for f in format_strs]<EOL>Logger.CURRENT = Logger(dir=dir, output_formats=output_formats, comm=comm)<EOL>log('<STR_LIT>'%dir)<EOL>

If comm is provided, average all numerical stats across that comm


import pandas<EOL>import numpy as np<EOL>from glob import glob<EOL>import tensorflow as tf<EOL>if osp.isdir(path):<EOL><INDENT>fnames = glob(osp.join(path, "<STR_LIT>"))<EOL><DEDENT>elif osp.basename(path).startswith("<STR_LIT>"):<EOL><INDENT>fnames = [path]<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError("<STR_LIT>"%path)<EOL><DEDENT>tag2pairs = defaultdict(list)<EOL>maxstep = <NUM_LIT:0><EOL>for fname in fnames:<EOL><INDENT>for summary in tf.train.summary_iterator(fname):<EOL><INDENT>if summary.step > <NUM_LIT:0>:<EOL><INDENT>for v in summary.summary.value:<EOL><INDENT>pair = (summary.step, v.simple_value)<EOL>tag2pairs[v.tag].append(pair)<EOL><DEDENT>maxstep = max(summary.step, maxstep)<EOL><DEDENT><DEDENT><DEDENT>data = np.empty((maxstep, len(tag2pairs)))<EOL>data[:] = np.nan<EOL>tags = sorted(tag2pairs.keys())<EOL>for (colidx,tag) in enumerate(tags):<EOL><INDENT>pairs = tag2pairs[tag]<EOL>for (step, value) in pairs:<EOL><INDENT>data[step-<NUM_LIT:1>, colidx] = value<EOL><DEDENT><DEDENT>return pandas.DataFrame(data, columns=tags)<EOL>

path : a tensorboard file OR a directory, where we will find all TB files
       of the form events.*


@staticmethod<EOL><INDENT>def create_datapoint(name, columns, points):<DEDENT>

return {<EOL>"<STR_LIT>": "<STR_LIT:s>",<EOL>"<STR_LIT:name>": name,<EOL>"<STR_LIT>": columns,<EOL>"<STR_LIT>": points,<EOL>}<EOL>

Create datastructure in InfluxDB 0.8 data format
:param name:
:param columns:
:param points:
:return:


def create_series(self, num_series, batch_size=<NUM_LIT>):

datapoints = []<EOL>for _ in range(num_series):<EOL><INDENT>name = self.dummy_seriesname()<EOL>datapoints.append(self.create_datapoint(name, ["<STR_LIT:value>"], [[<NUM_LIT:1>]]))<EOL><DEDENT>for data in tqdm(self.batch(datapoints, batch_size)):<EOL><INDENT>self.client.write_points(data)<EOL><DEDENT>

Write one data point for each series name to initialize the series
:param num_series: Number of different series names to create
:param batch_size: Number of series to create at the same time
:return:


def write_points(self, series_name, start_date, end_date, resolution=<NUM_LIT:10>, batch_size=<NUM_LIT>):

start_ts = int(start_date.strftime("<STR_LIT:%s>"))<EOL>end_ts = int(end_date.strftime("<STR_LIT:%s>"))<EOL>range_seconds = end_ts - start_ts<EOL>num_datapoints = range_seconds / resolution<EOL>timestamps = [start_ts + i * resolution for i in range(num_datapoints)]<EOL>columns = ["<STR_LIT:time>", "<STR_LIT:value>"]<EOL>for batch in tqdm(self.batch(timestamps, batch_size)):<EOL><INDENT>points = []<EOL>for timestamp in batch:<EOL><INDENT>point = random.randint(<NUM_LIT:1>, <NUM_LIT:100>)<EOL>points.append([timestamp, point])<EOL><DEDENT>datapoint = self.create_datapoint(series_name, columns, points)<EOL>self.client.write_points([datapoint])<EOL><DEDENT>

Create sample datapoints between two dates with the given resolution (in seconds)
:param series_name:
:param start_date:
:param end_date:
:param resolution:
:param batch_size:


l = len(iterable)<EOL>for ndx in range(<NUM_LIT:0>, l, n):<EOL><INDENT>yield iterable[ndx:min(ndx + n, l)]<EOL><DEDENT>

From http://stackoverflow.com/a/8290508/270334
:param n:
:param iterable:


return '<STR_LIT:U+0020>'.join([Keyword.LIST.upper(), Keyword.SERIES.upper(), self.series_stmt])<EOL>

Standard string representation of drop query


return '<STR_LIT:U+0020>'.join([Keyword.DROP.upper(), Keyword.SERIES.upper(), self.series_stmt])<EOL>


if not self.time_overlap:<EOL><INDENT>return TimeExpression.INFLUXDB_EPOCH<EOL><DEDENT>return self.time_overlap.start<EOL>

Get the smallest date in the query
E.g. in simple queries like select * from foo where time > now() - 24h
this would be the date of yesterday
This can be useful for checking if very old data is queried for example.
:return:


<EOL>statements = [<EOL>(Keyword.SELECT, self.select_stmt),<EOL>(Keyword.FROM, self.from_stmt)<EOL>]<EOL>if self.where_stmt:<EOL><INDENT>statements.append((Keyword.WHERE, self.where_stmt))<EOL><DEDENT>if self.limit_stmt:<EOL><INDENT>statements.append((Keyword.LIMIT, self.limit_stmt))<EOL><DEDENT>if self.group_by_stmt:<EOL><INDENT>statements.append((Keyword.GROUP_BY, ['<STR_LIT>', self.group_by_stmt]))<EOL><DEDENT>return self._format_statements(statements)<EOL>

Standard string representation of select query


return None<EOL>

The resolution is the interval given in the the group_by field.
(e.g. in the expression "group by time(10s)" it would be 10s)
Only Select queries can have a resolution.
All others don't, so we set this to None by default.


return None<EOL>

The duration is the timespan to query data for.
It can be set in the "where" clause.
For instance, in the query:
"select * from myseries where time > now() - 24h"
the duration would be 24h.
Durations can only be given in Select and Delete queries
so we set this to None by default.


return None<EOL>

Returns an estimate for the number datapoints that this query
will return, where datapoints = duration * resolution
Datapoints can only occur in Select and Delete queries
so we set this to None by default.


statements = [Keyword.DELETE.upper(), Keyword.FROM.upper(), self.from_stmt]<EOL>if self.where_stmt:<EOL><INDENT>statements.extend([Keyword.WHERE.upper(), self.where_stmt])<EOL><DEDENT>return '<STR_LIT:U+0020>'.join(statements)<EOL>

Standard string representation of delete query


query = urllib.unquote(query).decode('<STR_LIT>')<EOL>return query.lower()<EOL>

Prepared raw query for parsing
:param query: The query to sanitize


def handle_error(self, request, client_address):

cls, e = sys.exc_info()[:<NUM_LIT:2>]<EOL>if cls is socket.error or cls is ssl.SSLError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>return HTTPServer.handle_error(self, request, client_address)<EOL><DEDENT>

Overwrite error handling to suppress socket/ssl related errors
:param client_address: Address of client
:param request: Request causing an error


return self.protector.check(query_string)<EOL>

Check if the query_string is allowed by the Protector rule set


parsed_params = urlparse.parse_qs(parameters)<EOL>if '<STR_LIT:q>' not in parsed_params:<EOL><INDENT>return []<EOL><DEDENT>queries = parsed_params['<STR_LIT:q>']<EOL>if not isinstance(queries, list):<EOL><INDENT>queries = [queries]<EOL><DEDENT>return queries<EOL>

Get a list of all queries (q=... parameters) from an URL parameter string
:param parameters: The url parameter list


def _handle_request(self, scheme, netloc, path, headers, body=None, method="<STR_LIT:GET>"):

backend_url = "<STR_LIT>".format(scheme, netloc, path)<EOL>try:<EOL><INDENT>response = self.http_request.request(backend_url, method=method, body=body, headers=dict(headers))<EOL>self._return_response(response)<EOL><DEDENT>except Exception as e:<EOL><INDENT>body = "<STR_LIT>".format(e.message)<EOL>logging.debug(body)<EOL>self.send_error(httplib.SERVICE_UNAVAILABLE, body)<EOL><DEDENT>


def send_error(self, code, message=None):

message = message.strip()<EOL>self.log_error("<STR_LIT>", code, message)<EOL>self.send_response(code)<EOL>self.send_header("<STR_LIT:Content-Type>", "<STR_LIT>")<EOL>self.send_header('<STR_LIT>', '<STR_LIT>')<EOL>self.end_headers()<EOL>if message:<EOL><INDENT>self.wfile.write(message)<EOL><DEDENT>

Send and log plain text error reply.
:param code:
:param message:


self.filter_headers(response.msg)<EOL>if "<STR_LIT>" in response.msg:<EOL><INDENT>del response.msg["<STR_LIT>"]<EOL><DEDENT>self.send_response(response.status, response.reason)<EOL>for header_key, header_value in response.msg.items():<EOL><INDENT>self.send_header(header_key, header_value)<EOL><DEDENT>body = response.read()<EOL>self.send_header('<STR_LIT>', str(len(body)))<EOL>self.end_headers()<EOL>self.wfile.write(body)<EOL>


strings = []<EOL>with open(filepath, '<STR_LIT:r>') as f:<EOL><INDENT>strings = f.readlines()<EOL>strings = [x.strip(u'<STR_LIT:\n>') for x in strings]<EOL>strings = [x for x in strings if x and not x.startswith(u'<STR_LIT:#>')]<EOL>strings.insert(<NUM_LIT:0>, u"<STR_LIT>")<EOL><DEDENT>return strings<EOL>

Get the list of naughty_strings.

    By default this will get the strings from the blns.txt file

    Code is a simple port of what is already in the /scripts directory

    :param filepath: Optional filepath the the blns.txt file
    :returns: The list of naughty strings


<EOL>config = flatten(default_config.DEFAULT_CONFIG)<EOL>cli_config = flatten(parse_args())<EOL>if "<STR_LIT>" in cli_config:<EOL><INDENT>logging.info("<STR_LIT>".format(cli_config['<STR_LIT>']))<EOL>configfile = parse_configfile(cli_config['<STR_LIT>'])<EOL>config = overwrite_config(config, configfile)<EOL><DEDENT>config = overwrite_config(config, cli_config)<EOL>if '<STR_LIT>' in config:<EOL><INDENT>if config['<STR_LIT>'] == <NUM_LIT:1>:<EOL><INDENT>logging.getLogger().setLevel(logging.INFO)<EOL><DEDENT>elif config['<STR_LIT>'] > <NUM_LIT:1>:<EOL><INDENT>logging.getLogger().setLevel(logging.DEBUG)<EOL><DEDENT><DEDENT>return ObjectView(config)<EOL>

Load settings from default config and optionally
overwrite with config file and commandline parameters
(in that order).


with open(configfile) as f:<EOL><INDENT>try:<EOL><INDENT>return yaml.safe_load(f)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logging.fatal("<STR_LIT>", e)<EOL>exit(-<NUM_LIT:1>)<EOL><DEDENT><DEDENT>

Read settings from file
:param configfile:


def flatten(d, parent_key='<STR_LIT>', sep='<STR_LIT:_>'):

items = []<EOL>for k, v in d.items():<EOL><INDENT>new_key = parent_key + sep + k if parent_key else k<EOL>if isinstance(v, collections.MutableMapping):<EOL><INDENT>items.extend(flatten(v, new_key, sep=sep).items())<EOL><DEDENT>else:<EOL><INDENT>items.append((new_key, v))<EOL><DEDENT><DEDENT>return dict(items)<EOL>

Flatten keys in a dictionary
Example:
flatten({'a': 1, 'c': {'a': 2, 'b': {'x': 5, 'y' : 10}}, 'd': [1, 2, 3]})
=> {'a': 1, 'c_a': 2, 'c_b_x': 5, 'd': [1, 2, 3], 'c_b_y': 10}
:param d:  Dictionary to flatten
:param sep: Separator between keys
:param parent_key: Key to merge with


config = loader.load_config()<EOL>if config.version:<EOL><INDENT>show_version()<EOL><DEDENT>if config.show_rules:<EOL><INDENT>show_rules()<EOL><DEDENT>if not config.configfile and not (hasattr(config, "<STR_LIT:status>") or hasattr(config, "<STR_LIT>")):<EOL><INDENT>show_configfile_warning()<EOL><DEDENT>check_write_permissions(config.logfile)<EOL>start_proxy(config)<EOL>


try:<EOL><INDENT>open(file, '<STR_LIT:a>')<EOL><DEDENT>except IOError:<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>".format(file))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

Check if we can write to the given file

Otherwise since we might detach the process to run in the background
we might never find out that writing failed and get an ugly
exit message on startup. For example:
ERROR: Child exited immediately with non-zero exit code 127

So we catch this error upfront and print a nicer error message
with a hint on how to fix it.


from rules.loader import import_rules<EOL>from rules.rule_list import all_rules<EOL>rules = import_rules(all_rules)<EOL>print("<STR_LIT>")<EOL>for name, rule in rules.iteritems():<EOL><INDENT>heading = "<STR_LIT>".format(rule.description(), name)<EOL>print("<STR_LIT>".format(heading))<EOL>for line in rule.reason():<EOL><INDENT>print(line)<EOL><DEDENT>print("<STR_LIT>")<EOL><DEDENT>sys.exit(<NUM_LIT:0>)<EOL>

Show the list of available rules and quit
:return:


from version import __version__<EOL>print("<STR_LIT>".format(__package__, __version__))<EOL>sys.exit(<NUM_LIT:0>)<EOL>


protector = Protector(config.rules, config.whitelist)<EOL>protector_daemon = ProtectorDaemon(config=config, protector=protector)<EOL>daemon = daemonocle.Daemon(<EOL>pidfile=config.pidfile,<EOL>detach=(not config.foreground),<EOL>shutdown_callback=shutdown,<EOL>worker=protector_daemon.run<EOL>)<EOL>daemon.do_action(config.command)<EOL>

Start the http proxy
:param config:
:return:


def __init__(self, rules, whitelist=[], safe_mode=True):

self.parser = QueryParser()<EOL>self.guard = Guard(rules)<EOL>self.sanitizer = Sanitizer()<EOL>self.whitelist = whitelist<EOL>self.safe_mode = safe_mode<EOL>

:param rules: A list of rules to evaluate
:param safe_mode: If set to True, allow the query in case it can not be parsed
:return:


if query.get_type() in {Keyword.LIST, Keyword.DROP}:<EOL><INDENT>series = query.series_stmt<EOL><DEDENT>else:<EOL><INDENT>series = query.from_stmt<EOL><DEDENT>if len(series) >= self.min_series_name_length:<EOL><INDENT>return Ok(True)<EOL><DEDENT>return Err("<STR_LIT>")<EOL>


if query.get_type() not in {Keyword.SELECT}:<EOL><INDENT>return Ok(True)<EOL><DEDENT>earliest_date = query.get_earliest_date()<EOL>if earliest_date >= self.min_start_date:<EOL><INDENT>return Ok(True)<EOL><DEDENT>if query.limit_stmt:<EOL><INDENT>return Ok(True)<EOL><DEDENT>return Err(("<STR_LIT>"<EOL>"<STR_LIT>").format(self.min_start_date.strftime("<STR_LIT>"),<EOL>earliest_date))<EOL>


if query.get_type() not in {Keyword.SELECT, Keyword.DELETE}:<EOL><INDENT>return Ok(True)<EOL><DEDENT>datapoints = query.get_datapoints()<EOL>if datapoints <= self.max_datapoints:<EOL><INDENT>return Ok(True)<EOL><DEDENT>return Err(("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>").format(datapoints))<EOL>


pass<EOL>

:return: A short description of the rule


pass<EOL>

When and why the rule is useful.

This should return a list of lines instead of a long string.
It's easier to format line breaks this way.

:return: The reason for the rule


pass<EOL>

Check if a given query is permitted
:param query:
:return: result.Ok() if permitted, result.Err() if not.


if query.get_type() != Keyword.DROP:<EOL><INDENT>return Ok(True)<EOL><DEDENT>return Err("<STR_LIT>")<EOL>


rule = importlib.import_module(path)<EOL>return rule<EOL>

Load the given rule
:param path: Import path to rule


if query.get_type() != Keyword.DELETE:<EOL><INDENT>return Ok(True)<EOL><DEDENT>return Err("<STR_LIT>")<EOL>


if query.get_type() not in {Keyword.SELECT}:<EOL><INDENT>return Ok(True)<EOL><DEDENT>if query.get_resolution() > <NUM_LIT:0>:<EOL><INDENT>return Ok(True)<EOL><DEDENT>return Err("<STR_LIT>")<EOL>


self.keywords = {<EOL>Keyword.SELECT, Keyword.FROM, Keyword.WHERE,<EOL>Keyword.LIMIT, Keyword.ORDER,<EOL>Keyword.GROUP_BY, Keyword.DROP, Keyword.DELETE,<EOL>Keyword.LIST, Keyword.SERIES<EOL>}<EOL>self.time_parser = TimeRangeParser()<EOL>self.duration_parser = DurationParser()<EOL>self.resolution_parser = ResolutionParser()<EOL>self.datapoints_parser = DatapointsParser()<EOL>self.parsed_time = None<EOL>self.parsed_resolution = None<EOL>self.parsed_datapoints = None<EOL>self.parsed_time_overlap = None<EOL>

Initialize query parsers and valid InfluxDB keywords


self._reset()<EOL>if not isinstance(raw_query_string, basestring):<EOL><INDENT>return None<EOL><DEDENT>query_string = self._cleanup(raw_query_string)<EOL>parts = self._split(query_string)<EOL>parts = self._sanitize_keywords(parts)<EOL>tokens = self._tokenize(parts)<EOL>if tokens:<EOL><INDENT>self.parsed_resolution = self._parse_resolution(tokens)<EOL>self.parsed_time = self._parse_time(tokens)<EOL>self.parsed_time_overlap = self._parse_duration(self.parsed_time)<EOL>self.parsed_datapoints = self._parse_datapoints(<EOL>self.parsed_time_overlap.timespan_seconds(),<EOL>self.parsed_resolution,<EOL>self.parse_keyword(Keyword.LIMIT, tokens)<EOL>)<EOL><DEDENT>return self.create_query_object(tokens)<EOL>

Parse a raw query string into fields
:param raw_query_string: Raw InfluxDB query string


return [word.strip() for word in query.split('<STR_LIT:U+0020>')]<EOL>

Split query strings into tokens
:param query: A sanitized query string


try:<EOL><INDENT>query_type = tokens['<STR_LIT:type>']<EOL>return getattr(self, '<STR_LIT>' % query_type)(tokens)<EOL><DEDENT>except (KeyError, TypeError):<EOL><INDENT>return self.invalid_query(tokens)<EOL><DEDENT>

Analyze query tokens and create an InfluxDBStatement from them
Return None on error
:param tokens: A list of InfluxDB query tokens


if not tokens[Keyword.SELECT]:<EOL><INDENT>return None<EOL><DEDENT>if not tokens[Keyword.FROM]:<EOL><INDENT>return None<EOL><DEDENT>return SelectQuery(<EOL>self.parse_keyword(Keyword.SELECT, tokens),<EOL>self.parse_keyword(Keyword.FROM, tokens),<EOL>where_stmt=self.parse_keyword(Keyword.WHERE, tokens),<EOL>limit_stmt=self.parse_keyword(Keyword.LIMIT, tokens),<EOL>group_by_stmt=self.parse_group(tokens),<EOL>duration=self.parsed_time_overlap.timespan_seconds(),<EOL>resolution=self.parsed_resolution,<EOL>time_ranges=self.parsed_time,<EOL>time_overlap=self.parsed_time_overlap,<EOL>datapoints=self.parsed_datapoints<EOL>)<EOL>

Parse tokens of select query
:param tokens: A list of InfluxDB query tokens


if not tokens[Keyword.SERIES]:<EOL><INDENT>tokens[Keyword.SERIES] = '<STR_LIT>'<EOL><DEDENT>return ListQuery(self.parse_keyword(Keyword.SERIES, tokens))<EOL>

Parse tokens of list query
:param tokens: A list of InfluxDB query tokens


if not tokens[Keyword.SERIES]:<EOL><INDENT>return None<EOL><DEDENT>return DropQuery(self.parse_keyword(Keyword.SERIES, tokens))<EOL>

Parse tokens of drop query
:param tokens: A list of InfluxDB query tokens


<EOL>if not tokens[Keyword.FROM]:<EOL><INDENT>return None<EOL><DEDENT>where_stmt = self.parse_keyword(Keyword.WHERE, tokens)<EOL>if where_stmt:<EOL><INDENT>if not where_stmt.startswith('<STR_LIT:time>'):<EOL><INDENT>return None<EOL><DEDENT><DEDENT>return DeleteQuery(<EOL>self.parse_keyword(Keyword.FROM, tokens),<EOL>self.parse_keyword(Keyword.WHERE, tokens)<EOL>)<EOL>

Parse tokens of delete query
:param tokens: A list of InfluxDB query tokens


logging.warning("<STR_LIT>")<EOL>try:<EOL><INDENT>logging.warning("<STR_LIT>", '<STR_LIT:U+0020>'.join(tokens))<EOL><DEDENT>except TypeError:<EOL><INDENT>pass<EOL><DEDENT>return None<EOL>

Handler for invalid queries
:param tokens: A list of InfluxDB query tokens


return self.time_parser.parse(self.parse_keyword(Keyword.WHERE, tokens))<EOL>

Parse the date range for the query

E.g. WHERE time > now() - 48h AND time < now() - 24h
would result in DateRange(datetime_start, datetime_end)
where
datetime_start would be parsed from now() - 48h
and
datetime_end would be parsed from now() - 24h

:param tokens:
:return:


return self.resolution_parser.parse(self.parse_keyword(Keyword.GROUP_BY, tokens))<EOL>

Parse resolution from the GROUP BY statement.
E.g. GROUP BY time(10s) would mean a 10 second resolution
:param tokens:
:return:


return self.duration_parser.parse(parsed_time)<EOL>

Parse duration in seconds for the given query.
The duration is parsed from the DateRange of the query.
For example, consider the following query:
select * from bla where time > now() - 2h
The DateRange would be [now() - 2h, now()]
Expressing this as seconds would be 2h => 2*60*60 seconds
This would be the duration of the query.

:param parsed_time:
:return:


def _parse_datapoints(self, parsed_duration, parsed_resolution, limit):

return self.datapoints_parser.parse(parsed_duration, parsed_resolution, limit)<EOL>

Parse the number of datapoints of a query.
This can be calculated from the given duration and resolution of the query.
E.g. if the query has a duation of 2*60*60 = 7200 seconds and a resolution of 10 seconds
then the number of datapoints would be 7200/10 => 7200 datapoints.

:param parsed_duration:
:param parsed_resolution:
:param limit:
:return:


if not group_by_stmt:<EOL><INDENT>return Resolution.MAX_RESOLUTION<EOL><DEDENT>m = self.GROUP_BY_TIME_PATTERN.match(group_by_stmt)<EOL>if not m:<EOL><INDENT>return None<EOL><DEDENT>value = int(m.group(<NUM_LIT:1>))<EOL>unit = m.group(<NUM_LIT:2>)<EOL>resolution = self.convert_to_seconds(value, unit)<EOL>return max(resolution, Resolution.MAX_RESOLUTION)<EOL>

Extract the data resolution of a query in seconds
E.g. "group by time(99s)" => 99

:param group_by_stmt: A raw InfluxDB group by statement


@staticmethod<EOL><INDENT>def parse(duration_seconds, resolution_seconds=Resolution.MAX_RESOLUTION, limit=None):<DEDENT>

if not duration_seconds or duration_seconds < <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>if not resolution_seconds or resolution_seconds <= <NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>num_datapoints = duration_seconds / resolution_seconds<EOL>if limit:<EOL><INDENT>num_datapoints = min(int(limit), num_datapoints)<EOL><DEDENT>return int(math.ceil(num_datapoints))<EOL>

num_datapoints = min(duration/resolution, limit)

:param duration_seconds: Time duration (in seconds) for which datapoints should be returned
:param resolution_seconds: Time interval (in seconds) between data points
:param limit: Maximum number of datapoints to return


@classmethod<EOL><INDENT>def get_object(cls, api_token, cert_id):<DEDENT>

certificate = cls(token=api_token, id=cert_id)<EOL>certificate.load()<EOL>return certificate<EOL>

Class method that will return a Certificate object by its ID.


data = self.get_data("<STR_LIT>" % self.id)<EOL>certificate = data["<STR_LIT>"]<EOL>for attr in certificate.keys():<EOL><INDENT>setattr(self, attr, certificate[attr])<EOL><DEDENT>return self<EOL>

Load the Certificate object from DigitalOcean.

Requires self.id to be set.


params = {<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT:type>": self.type,<EOL>"<STR_LIT>": self.dns_names,<EOL>"<STR_LIT>": self.private_key,<EOL>"<STR_LIT>": self.leaf_certificate,<EOL>"<STR_LIT>": self.certificate_chain<EOL>}<EOL>data = self.get_data("<STR_LIT>", type=POST, params=params)<EOL>if data:<EOL><INDENT>self.id = data['<STR_LIT>']['<STR_LIT:id>']<EOL>self.not_after = data['<STR_LIT>']['<STR_LIT>']<EOL>self.sha1_fingerprint = data['<STR_LIT>']['<STR_LIT>']<EOL>self.created_at = data['<STR_LIT>']['<STR_LIT>']<EOL>self.type = data['<STR_LIT>']['<STR_LIT:type>']<EOL>self.dns_names = data['<STR_LIT>']['<STR_LIT>']<EOL>self.state = data['<STR_LIT>']['<STR_LIT:state>']<EOL><DEDENT>return self<EOL>


return self.get_data("<STR_LIT>" % self.id, type=DELETE)<EOL>


def assert_url_query_equal(self, url1, url2):

base1, qlist1 = self.split_url(url1)<EOL>base2, qlist2 = self.split_url(url2)<EOL>self.assertEqual(base1, base2)<EOL>self.assertEqual(qlist1, qlist2)<EOL>

Test if two URL queries are equal

        The key=value pairs after the ? in a URL can occur in any order
        (especially since dicts in python 3 are not deterministic across runs).
        The method sorts the key=value pairs and then compares the URLs.


@classmethod<EOL><INDENT>def get_object(cls, api_token, domain_name):<DEDENT>

domain = cls(token=api_token, name=domain_name)<EOL>domain.load()<EOL>return domain<EOL>

Class method that will return a Domain object by ID.


<EOL>return self.get_data("<STR_LIT>" % self.name, type=DELETE)<EOL>


def create_new_domain_record(self, *args, **kwargs):

data = {<EOL>"<STR_LIT:type>": kwargs.get("<STR_LIT:type>", None),<EOL>"<STR_LIT:name>": kwargs.get("<STR_LIT:name>", None),<EOL>"<STR_LIT:data>": kwargs.get("<STR_LIT:data>", None)<EOL>}<EOL>if kwargs.get("<STR_LIT>", None):<EOL><INDENT>data['<STR_LIT>'] = kwargs.get("<STR_LIT>", None)<EOL><DEDENT>if kwargs.get("<STR_LIT:port>", None):<EOL><INDENT>data['<STR_LIT:port>'] = kwargs.get("<STR_LIT:port>", None)<EOL><DEDENT>if kwargs.get("<STR_LIT>", None):<EOL><INDENT>data['<STR_LIT>'] = kwargs.get("<STR_LIT>", None)<EOL><DEDENT>return self.get_data(<EOL>"<STR_LIT>" % self.name,<EOL>type=POST,<EOL>params=data<EOL>)<EOL>

Create new domain record.
https://developers.digitalocean.com/#create-a-new-domain-record

Args:
    type: The record type (A, MX, CNAME, etc).
    name: The host name, alias, or service being defined by the record
    data: Variable data depending on record type.

Optional Args:
    priority: The priority of the host
    port: The port that the service is accessible on
    weight: The weight of records with the same priority


<EOL>data = {<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT>": self.ip_address,<EOL>}<EOL>domain = self.get_data("<STR_LIT>", type=POST, params=data)<EOL>return domain<EOL>


if params is None:<EOL><INDENT>params = {}<EOL><DEDENT>records = []<EOL>data = self.get_data("<STR_LIT>" % self.name, type=GET, params=params)<EOL>for record_data in data['<STR_LIT>']:<EOL><INDENT>record = Record(domain_name=self.name, **record_data)<EOL>record.token = self.token<EOL>records.append(record)<EOL><DEDENT>return records<EOL>


@classmethod<EOL><INDENT>def get_object(cls, api_token, ip):<DEDENT>

floating_ip = cls(token=api_token, ip=ip)<EOL>floating_ip.load()<EOL>return floating_ip<EOL>

Class method that will return a FloatingIP object by its IP.

Args:
    api_token: str - token
    ip: str - floating ip address


data = self.get_data('<STR_LIT>' % self.ip, type=GET)<EOL>floating_ip = data['<STR_LIT>']<EOL>for attr in floating_ip.keys():<EOL><INDENT>setattr(self, attr, floating_ip[attr])<EOL><DEDENT>return self<EOL>

Load the FloatingIP object from DigitalOcean.

Requires self.ip to be set.


data = self.get_data('<STR_LIT>',<EOL>type=POST,<EOL>params={'<STR_LIT>': self.droplet_id})<EOL>if data:<EOL><INDENT>self.ip = data['<STR_LIT>']['<STR_LIT>']<EOL>self.region = data['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>return self<EOL>

Creates a FloatingIP and assigns it to a Droplet.

Note: Every argument and parameter given to this method will be
assigned to the object.

Args:
    droplet_id: int - droplet id


data = self.get_data('<STR_LIT>',<EOL>type=POST,<EOL>params={'<STR_LIT>': self.region_slug})<EOL>if data:<EOL><INDENT>self.ip = data['<STR_LIT>']['<STR_LIT>']<EOL>self.region = data['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>return self<EOL>

Creates a FloatingIP in a region without assigning
it to a specific Droplet.

Note: Every argument and parameter given to this method will be
assigned to the object.

Args:
    region_slug: str - region's slug (e.g. 'nyc3')


return self.get_data('<STR_LIT>' % self.ip, type=DELETE)<EOL>


return self.get_data(<EOL>"<STR_LIT>" % self.ip,<EOL>type=POST,<EOL>params={"<STR_LIT:type>": "<STR_LIT>", "<STR_LIT>": droplet_id}<EOL>)<EOL>

Assign a FloatingIP to a Droplet.

Args:
    droplet_id: int - droplet id


return self.get_data(<EOL>"<STR_LIT>" % self.ip,<EOL>type=POST,<EOL>params={"<STR_LIT:type>": "<STR_LIT>"}<EOL>)<EOL>


@classmethod<EOL><INDENT>def get_object(cls, api_token, droplet_id):<DEDENT>

droplet = cls(token=api_token, id=droplet_id)<EOL>droplet.load()<EOL>return droplet<EOL>

Class method that will return a Droplet object by ID.

        Args:
            api_token (str): token
            droplet_id (int): droplet id


data = super(Droplet, self).get_data(*args, **kwargs)<EOL>if "<STR_LIT:type>" in kwargs:<EOL><INDENT>if kwargs["<STR_LIT:type>"] == POST:<EOL><INDENT>self.__check_actions_in_data(data)<EOL><DEDENT><DEDENT>return data<EOL>

Customized version of get_data to perform __check_actions_in_data


droplets = self.get_data("<STR_LIT>" % self.id)<EOL>droplet = droplets['<STR_LIT>']<EOL>for attr in droplet.keys():<EOL><INDENT>setattr(self, attr, droplet[attr])<EOL><DEDENT>for net in self.networks['<STR_LIT>']:<EOL><INDENT>if net['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>self.private_ip_address = net['<STR_LIT>']<EOL><DEDENT>if net['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>self.ip_address = net['<STR_LIT>']<EOL><DEDENT><DEDENT>if self.networks['<STR_LIT>']:<EOL><INDENT>self.ip_v6_address = self.networks['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT>']<EOL>if "<STR_LIT>" in self.features:<EOL><INDENT>self.backups = True<EOL><DEDENT>else:<EOL><INDENT>self.backups = False<EOL><DEDENT>if "<STR_LIT>" in self.features:<EOL><INDENT>self.ipv6 = True<EOL><DEDENT>else:<EOL><INDENT>self.ipv6 = False<EOL><DEDENT>if "<STR_LIT>" in self.features:<EOL><INDENT>self.private_networking = True<EOL><DEDENT>else:<EOL><INDENT>self.private_networking = False<EOL><DEDENT><DEDENT>if "<STR_LIT>" in droplets:<EOL><INDENT>self.tags = droplets["<STR_LIT>"]<EOL><DEDENT>return self<EOL>

Fetch data about droplet - use this instead of get_data()


def _perform_action(self, params, return_dict=True):

action = self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params=params<EOL>)<EOL>if return_dict:<EOL><INDENT>return action<EOL><DEDENT>else:<EOL><INDENT>action = action[u'<STR_LIT:action>']<EOL>return_action = Action(token=self.token)<EOL>for attr in action.keys():<EOL><INDENT>setattr(return_action, attr, action[attr])<EOL><DEDENT>return return_action<EOL><DEDENT>

Perform a droplet action.

Args:
    params (dict): parameters of the action

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


return self._perform_action({'<STR_LIT:type>': '<STR_LIT>'}, return_dict)<EOL>

Boot up the droplet

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


shutdown the droplet

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


restart the droplet

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


reset the root password

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


def resize(self, new_size_slug, return_dict=True, disk=True):

options = {"<STR_LIT:type>": "<STR_LIT>", "<STR_LIT:size>": new_size_slug}<EOL>if disk: options["<STR_LIT>"] = "<STR_LIT:true>"<EOL>return self._perform_action(options, return_dict)<EOL>

Resize the droplet to a new size slug.
        https://developers.digitalocean.com/documentation/v2/#resize-a-droplet

        Args:
            new_size_slug (str): name of new size

        Optional Args:
            return_dict (bool): Return a dict when True (default),
                                otherwise return an Action.
            disk (bool): If a permanent resize, with disk changes included.

        Returns dict or Action


def take_snapshot(self, snapshot_name, return_dict=True, power_off=False):

if power_off is True and self.status != "<STR_LIT>":<EOL><INDENT>action = self.power_off(return_dict=False)<EOL>action.wait()<EOL>self.load()<EOL><DEDENT>return self._perform_action(<EOL>{"<STR_LIT:type>": "<STR_LIT>", "<STR_LIT:name>": snapshot_name},<EOL>return_dict<EOL>)<EOL>

Take a snapshot!

        Args:
            snapshot_name (str): name of snapshot

        Optional Args:
            return_dict (bool): Return a dict when True (default),
                otherwise return an Action.
            power_off (bool): Before taking the snapshot the droplet will be
                turned off with another API call. It will wait until the
                droplet will be powered off.

        Returns dict or Action


def restore(self, image_id, return_dict=True):

return self._perform_action(<EOL>{"<STR_LIT:type>": "<STR_LIT>", "<STR_LIT:image>": image_id},<EOL>return_dict<EOL>)<EOL>

Restore the droplet to an image ( snapshot or backup )

        Args:
            image_id (int): id of image

        Optional Args:
            return_dict (bool): Return a dict when True (default),
                otherwise return an Action.

        Returns dict or Action


def rebuild(self, image_id=None, return_dict=True):

if not image_id:<EOL><INDENT>image_id = self.image['<STR_LIT:id>']<EOL><DEDENT>return self._perform_action(<EOL>{"<STR_LIT:type>": "<STR_LIT>", "<STR_LIT:image>": image_id},<EOL>return_dict<EOL>)<EOL>


Enable automatic backups

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


Disable automatic backups

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


return self.get_data("<STR_LIT>" % self.id, type=DELETE)<EOL>


def rename(self, name, return_dict=True):

return self._perform_action(<EOL>{'<STR_LIT:type>': '<STR_LIT>', '<STR_LIT:name>': name},<EOL>return_dict<EOL>)<EOL>

Rename the droplet

        Args:
            name (str): new name

        Optional Args:
            return_dict (bool): Return a dict when True (default),
                otherwise return an Action.

        Returns dict or Action


return self._perform_action(<EOL>{'<STR_LIT:type>': '<STR_LIT>'},<EOL>return_dict<EOL>)<EOL>

Enable private networking on an existing Droplet where available.

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


Enable IPv6 on an existing Droplet where available.

Optional Args:
    return_dict (bool): Return a dict when True (default),
        otherwise return an Action.

Returns dict or Action


def change_kernel(self, kernel, return_dict=True):

if type(kernel) != Kernel:<EOL><INDENT>raise BadKernelObject("<STR_LIT>")<EOL><DEDENT>return self._perform_action(<EOL>{'<STR_LIT:type>': '<STR_LIT>', '<STR_LIT>': kernel.id},<EOL>return_dict<EOL>)<EOL>

Change the kernel to a new one

        Args:
            kernel : instance of digitalocean.Kernel.Kernel

        Optional Args:
            return_dict (bool): Return a dict when True (default),
                otherwise return an Action.

        Returns dict or Action


@staticmethod<EOL><INDENT>def __get_ssh_keys_id_or_fingerprint(ssh_keys, token, name):<DEDENT>

ssh_keys_id = list()<EOL>for ssh_key in ssh_keys:<EOL><INDENT>if type(ssh_key) in [int, type(<NUM_LIT:2> ** <NUM_LIT:64>)]:<EOL><INDENT>ssh_keys_id.append(int(ssh_key))<EOL><DEDENT>elif type(ssh_key) == SSHKey:<EOL><INDENT>ssh_keys_id.append(ssh_key.id)<EOL><DEDENT>elif type(ssh_key) in [type(u'<STR_LIT>'), type('<STR_LIT>')]:<EOL><INDENT>regexp_of_fingerprint = '<STR_LIT>'<EOL>match = re.match(regexp_of_fingerprint, ssh_key)<EOL>if match is not None and match.end() == len(ssh_key) - <NUM_LIT:1>:<EOL><INDENT>ssh_keys_id.append(ssh_key)<EOL><DEDENT>else:<EOL><INDENT>key = SSHKey()<EOL>key.token = token<EOL>results = key.load_by_pub_key(ssh_key)<EOL>if results is None:<EOL><INDENT>key.public_key = ssh_key<EOL>key.name = "<STR_LIT>" % name<EOL>key.create()<EOL><DEDENT>else:<EOL><INDENT>key = results<EOL><DEDENT>ssh_keys_id.append(key.id)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise BadSSHKeyFormat(<EOL>"<STR_LIT>"<EOL>+ "<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>return ssh_keys_id<EOL>

Check and return a list of SSH key IDs or fingerprints according
to DigitalOcean's API. This method is used to check and create a
droplet with the correct SSH keys.


for attr in kwargs.keys():<EOL><INDENT>setattr(self, attr, kwargs[attr])<EOL><DEDENT>if not self.size_slug and self.size:<EOL><INDENT>self.size_slug = self.size<EOL><DEDENT>ssh_keys_id = Droplet.__get_ssh_keys_id_or_fingerprint(self.ssh_keys,<EOL>self.token,<EOL>self.name)<EOL>data = {<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT:size>": self.size_slug,<EOL>"<STR_LIT:image>": self.image,<EOL>"<STR_LIT>": self.region,<EOL>"<STR_LIT>": ssh_keys_id,<EOL>"<STR_LIT>": bool(self.backups),<EOL>"<STR_LIT>": bool(self.ipv6),<EOL>"<STR_LIT>": bool(self.private_networking),<EOL>"<STR_LIT>": self.volumes,<EOL>"<STR_LIT>": self.tags,<EOL>"<STR_LIT>": bool(self.monitoring),<EOL>}<EOL>if self.user_data:<EOL><INDENT>data["<STR_LIT>"] = self.user_data<EOL><DEDENT>data = self.get_data("<STR_LIT>", type=POST, params=data)<EOL>if data:<EOL><INDENT>self.id = data['<STR_LIT>']['<STR_LIT:id>']<EOL>action_id = data['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT:id>']<EOL>self.action_ids = []<EOL>self.action_ids.append(action_id)<EOL><DEDENT>

Create the droplet with object properties.

Note: Every argument and parameter given to this method will be
assigned to the object.


return self.get_actions()<EOL>

A helper function for backwards compatibility.
Calls get_actions()


answer = self.get_data("<STR_LIT>" % self.id, type=GET)<EOL>actions = []<EOL>for action_dict in answer['<STR_LIT>']:<EOL><INDENT>action = Action(**action_dict)<EOL>action.token = self.token<EOL>action.droplet_id = self.id<EOL>action.load()<EOL>actions.append(action)<EOL><DEDENT>return actions<EOL>

Returns a list of Action objects
This actions can be used to check the droplet's status


return Action.get_object(<EOL>api_token=self.token,<EOL>action_id=action_id<EOL>)<EOL>

Returns a specific Action by its ID.

        Args:
            action_id (int): id of action


snapshots = list()<EOL>for id in self.snapshot_ids:<EOL><INDENT>snapshot = Image()<EOL>snapshot.id = id<EOL>snapshot.token = self.token<EOL>snapshots.append(snapshot)<EOL><DEDENT>return snapshots<EOL>

This method will return the snapshots/images connected to that
specific droplet.


kernels = list()<EOL>data = self.get_data("<STR_LIT>" % self.id)<EOL>while True:<EOL><INDENT>for jsond in data[u'<STR_LIT>']:<EOL><INDENT>kernel = Kernel(**jsond)<EOL>kernel.token = self.token<EOL>kernels.append(kernel)<EOL><DEDENT>try:<EOL><INDENT>url = data[u'<STR_LIT>'][u'<STR_LIT>'].get(u'<STR_LIT>')<EOL>if not url:<EOL><INDENT>break<EOL><DEDENT>data = self.get_data(url)<EOL><DEDENT>except KeyError:  <EOL><INDENT>break<EOL><DEDENT><DEDENT>return kernels<EOL>


@classmethod<EOL><INDENT>def get_object(cls, api_token, firewall_id):<DEDENT>

firewall = cls(token=api_token, id=firewall_id)<EOL>firewall.load()<EOL>return firewall<EOL>

Class method that will return a Firewall object by ID.


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params={"<STR_LIT>": droplet_ids}<EOL>)<EOL>


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=DELETE,<EOL>params={"<STR_LIT>": droplet_ids}<EOL>)<EOL>


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params={"<STR_LIT>": tags}<EOL>)<EOL>


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=DELETE,<EOL>params={"<STR_LIT>": tags}<EOL>)<EOL>


@classmethod<EOL><INDENT>def get_object(cls, api_token, ssh_key_id):<DEDENT>

ssh_key = cls(token=api_token, id=ssh_key_id)<EOL>ssh_key.load()<EOL>return ssh_key<EOL>

Class method that will return a SSHKey object by ID.


identifier = None<EOL>if self.id:<EOL><INDENT>identifier = self.id<EOL><DEDENT>elif self.fingerprint is not None:<EOL><INDENT>identifier = self.fingerprint<EOL><DEDENT>data = self.get_data("<STR_LIT>" % identifier, type=GET)<EOL>ssh_key = data['<STR_LIT>']<EOL>for attr in ssh_key.keys():<EOL><INDENT>setattr(self, attr, ssh_key[attr])<EOL><DEDENT>self.id = ssh_key['<STR_LIT:id>']<EOL>

Load the SSHKey object from DigitalOcean.

Requires either self.id or self.fingerprint to be set.


data = self.get_data("<STR_LIT>")<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>if jsoned.get('<STR_LIT>', "<STR_LIT>") == public_key:<EOL><INDENT>self.id = jsoned['<STR_LIT:id>']<EOL>self.load()<EOL>return self<EOL><DEDENT><DEDENT>return None<EOL>

This method will load a SSHKey object from DigitalOcean
from a public_key. This method will avoid problems like
uploading the same public_key twice.


input_params = {<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT>": self.public_key,<EOL>}<EOL>data = self.get_data("<STR_LIT>", type=POST, params=input_params)<EOL>if data:<EOL><INDENT>self.id = data['<STR_LIT>']['<STR_LIT:id>']<EOL><DEDENT>


input_params = {<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT>": self.public_key,<EOL>}<EOL>data = self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=PUT,<EOL>params=input_params<EOL>)<EOL>if data:<EOL><INDENT>self.id = data['<STR_LIT>']['<STR_LIT:id>']<EOL><DEDENT>


return self.get_data("<STR_LIT>" % self.id, type=DELETE)<EOL>


@classmethod<EOL><INDENT>def get_object(cls, api_token, action_id):<DEDENT>

action = cls(token=api_token, id=action_id)<EOL>action.load_directly()<EOL>return action<EOL>

Class method that will return a Action object by ID.


while self.status == u'<STR_LIT>':<EOL><INDENT>sleep(update_every_seconds)<EOL>self.load()<EOL><DEDENT>return self.status == u'<STR_LIT>'<EOL>

Wait until the action is marked as completed or with an error.
It will return True in case of success, otherwise False.

Optional Args:
    update_every_seconds - int : number of seconds to wait before
        checking if the action is completed.


return Account.get_object(api_token=self.token)<EOL>


data = self.get_data("<STR_LIT>")<EOL>regions = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>region = Region(**jsoned)<EOL>region.token = self.token<EOL>regions.append(region)<EOL><DEDENT>return regions<EOL>

This function returns a list of Region object.


params = dict()<EOL>if tag_name:<EOL><INDENT>params["<STR_LIT>"] = tag_name<EOL><DEDENT>data = self.get_data("<STR_LIT>", params=params)<EOL>droplets = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>droplet = Droplet(**jsoned)<EOL>droplet.token = self.token<EOL>for net in droplet.networks['<STR_LIT>']:<EOL><INDENT>if net['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>droplet.private_ip_address = net['<STR_LIT>']<EOL><DEDENT>if net['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>droplet.ip_address = net['<STR_LIT>']<EOL><DEDENT><DEDENT>if droplet.networks['<STR_LIT>']:<EOL><INDENT>droplet.ip_v6_address = droplet.networks['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT>']<EOL><DEDENT>if "<STR_LIT>" in droplet.features:<EOL><INDENT>droplet.backups = True<EOL><DEDENT>else:<EOL><INDENT>droplet.backups = False<EOL><DEDENT>if "<STR_LIT>" in droplet.features:<EOL><INDENT>droplet.ipv6 = True<EOL><DEDENT>else:<EOL><INDENT>droplet.ipv6 = False<EOL><DEDENT>if "<STR_LIT>" in droplet.features:<EOL><INDENT>droplet.private_networking = True<EOL><DEDENT>else:<EOL><INDENT>droplet.private_networking = False<EOL><DEDENT>droplets.append(droplet)<EOL><DEDENT>return droplets<EOL>

This function returns a list of Droplet object.


return Droplet.get_object(api_token=self.token, droplet_id=droplet_id)<EOL>


data = self.get_data("<STR_LIT>")<EOL>sizes = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>size = Size(**jsoned)<EOL>size.token = self.token<EOL>sizes.append(size)<EOL><DEDENT>return sizes<EOL>

This function returns a list of Size object.


def get_images(self, private=False, type=None):

params = {}<EOL>if private:<EOL><INDENT>params['<STR_LIT>'] = '<STR_LIT:true>'<EOL><DEDENT>if type:<EOL><INDENT>params['<STR_LIT:type>'] = type<EOL><DEDENT>data = self.get_data("<STR_LIT>", params=params)<EOL>images = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>image = Image(**jsoned)<EOL>image.token = self.token<EOL>images.append(image)<EOL><DEDENT>return images<EOL>

This function returns a list of Image object.


images = self.get_images()<EOL>return images<EOL>

This function returns a list of Image objects containing all
available DigitalOcean images, both public and private.


return Image.get_object(<EOL>api_token=self.token,<EOL>image_id_or_slug=image_id_or_slug,<EOL>)<EOL>


images = self.get_images(private=True)<EOL>return images<EOL>

This function returns a list of Image objects representing
private DigitalOcean images (e.g. snapshots and backups).


data = self.get_images()<EOL>images = list()<EOL>for i in data:<EOL><INDENT>if i.public:<EOL><INDENT>i.token = self.token<EOL>images.append(i)<EOL><DEDENT><DEDENT>return images<EOL>

This function returns a list of Image objects representing
public DigitalOcean images (e.g. base distribution images
and 'One-Click' applications).


images = self.get_images(type='<STR_LIT>')<EOL>return images<EOL>

This function returns a list of Image objects representing
public base distribution images.


This function returns a list of Image objectobjects representing
public DigitalOcean 'One-Click' application images.


data = self.get_data("<STR_LIT>")<EOL>domains = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>domain = Domain(**jsoned)<EOL>domain.token = self.token<EOL>domains.append(domain)<EOL><DEDENT>return domains<EOL>

This function returns a list of Domain object.


return Domain.get_object(api_token=self.token, domain_name=domain_name)<EOL>


data = self.get_data("<STR_LIT>")<EOL>ssh_keys = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>ssh_key = SSHKey(**jsoned)<EOL>ssh_key.token = self.token<EOL>ssh_keys.append(ssh_key)<EOL><DEDENT>return ssh_keys<EOL>

This function returns a list of SSHKey object.


return SSHKey.get_object(api_token=self.token, ssh_key_id=ssh_key_id)<EOL>


data = self.get_data("<STR_LIT>")<EOL>return [<EOL>Tag(token=self.token, **tag) for tag in data['<STR_LIT>']<EOL>]<EOL>


return Action.get_object(api_token=self.token, action_id=action_id)<EOL>

Return an Action object by a specific ID.


data = self.get_data("<STR_LIT>")<EOL>floating_ips = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>floating_ip = FloatingIP(**jsoned)<EOL>floating_ip.token = self.token<EOL>floating_ips.append(floating_ip)<EOL><DEDENT>return floating_ips<EOL>

This function returns a list of FloatingIP objects.


return FloatingIP.get_object(api_token=self.token, ip=ip)<EOL>

Returns a of FloatingIP object by its IP address.


data = self.get_data("<STR_LIT>")<EOL>load_balancers = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>load_balancer = LoadBalancer(**jsoned)<EOL>load_balancer.token = self.token<EOL>load_balancer.health_check = HealthCheck(**jsoned['<STR_LIT>'])<EOL>load_balancer.sticky_sessions = StickySesions(**jsoned['<STR_LIT>'])<EOL>forwarding_rules = list()<EOL>for rule in jsoned['<STR_LIT>']:<EOL><INDENT>forwarding_rules.append(ForwardingRule(**rule))<EOL><DEDENT>load_balancer.forwarding_rules = forwarding_rules<EOL>load_balancers.append(load_balancer)<EOL><DEDENT>return load_balancers<EOL>

Returns a list of Load Balancer objects.


return LoadBalancer.get_object(api_token=self.token, id=id)<EOL>

Returns a Load Balancer object by its ID.

Args:
    id (str): Load Balancer ID


return Certificate.get_object(api_token=self.token, cert_id=id)<EOL>

Returns a Certificate object by its ID.

Args:
    id (str): Certificate ID


data = self.get_data("<STR_LIT>")<EOL>certificates = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>cert = Certificate(**jsoned)<EOL>cert.token = self.token<EOL>certificates.append(cert)<EOL><DEDENT>return certificates<EOL>

This function returns a list of Certificate objects.


return Snapshot.get_object(<EOL>api_token=self.token, snapshot_id=snapshot_id<EOL>)<EOL>


data = self.get_data("<STR_LIT>")<EOL>return [<EOL>Snapshot(token=self.token, **snapshot)<EOL>for snapshot in data['<STR_LIT>']<EOL>]<EOL>

This method returns a list of all Snapshots.


This method returns a list of all Snapshots based on Droplets.


This method returns a list of all Snapshots based on volumes.


if region:<EOL><INDENT>url = "<STR_LIT>".format(region)<EOL><DEDENT>else:<EOL><INDENT>url = "<STR_LIT>"<EOL><DEDENT>data = self.get_data(url)<EOL>volumes = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>volume = Volume(**jsoned)<EOL>volume.token = self.token<EOL>volumes.append(volume)<EOL><DEDENT>return volumes<EOL>

This function returns a list of Volume objects.


return Volume.get_object(api_token=self.token, volume_id=volume_id)<EOL>


data = self.get_data("<STR_LIT>")<EOL>firewalls = list()<EOL>for jsoned in data['<STR_LIT>']:<EOL><INDENT>firewall = Firewall(**jsoned)<EOL>firewall.token = self.token<EOL>in_rules = list()<EOL>for rule in jsoned['<STR_LIT>']:<EOL><INDENT>in_rules.append(InboundRule(**rule))<EOL><DEDENT>firewall.inbound_rules = in_rules<EOL>out_rules = list()<EOL>for rule in jsoned['<STR_LIT>']:<EOL><INDENT>out_rules.append(OutboundRule(**rule))<EOL><DEDENT>firewall.outbound_rules = out_rules<EOL>firewalls.append(firewall)<EOL><DEDENT>return firewalls<EOL>

This function returns a list of Firewall objects.


return Firewall.get_object(<EOL>api_token=self.token,<EOL>firewall_id=firewall_id,<EOL>)<EOL>


@classmethod<EOL><INDENT>def get_object(cls, api_token, snapshot_id):<DEDENT>

snapshot = cls(token=api_token, id=snapshot_id)<EOL>snapshot.load()<EOL>return snapshot<EOL>

Class method that will return a Snapshot object by ID.


return self.get_data("<STR_LIT>" % self.id, type=DELETE)<EOL>


def __perform_request(self, url, type=GET, params=None):

if params is None:<EOL><INDENT>params = {}<EOL><DEDENT>if not self.token:<EOL><INDENT>raise TokenError("<STR_LIT>")<EOL><DEDENT>url = urlparse.urljoin(self.end_point, url)<EOL>identity = lambda x: x<EOL>json_dumps = lambda x: json.dumps(x)<EOL>lookup = {<EOL>GET: (self._session.get, {}, '<STR_LIT>', identity),<EOL>POST: (self._session.post, {'<STR_LIT>': '<STR_LIT:application/json>'}, '<STR_LIT:data>',<EOL>json_dumps),<EOL>PUT: (self._session.put, {'<STR_LIT>': '<STR_LIT:application/json>'}, '<STR_LIT:data>',<EOL>json_dumps),<EOL>DELETE: (self._session.delete,<EOL>{'<STR_LIT>': '<STR_LIT:application/json>'},<EOL>'<STR_LIT:data>', json_dumps),<EOL>}<EOL>requests_method, headers, payload, transform = lookup[type]<EOL>agent = "<STR_LIT>".format('<STR_LIT>',<EOL>__version__,<EOL>requests.__name__,<EOL>requests.__version__)<EOL>headers.update({'<STR_LIT>': '<STR_LIT>' + self.token,<EOL>'<STR_LIT>': agent})<EOL>kwargs = {'<STR_LIT>': headers, payload: transform(params)}<EOL>timeout = self.get_timeout()<EOL>if timeout:<EOL><INDENT>kwargs['<STR_LIT>'] = timeout<EOL><DEDENT>headers_str = str(headers).replace(self.token.strip(), '<STR_LIT>')<EOL>self._log.debug('<STR_LIT>' %<EOL>(type, url, payload, params, headers_str, timeout))<EOL>return requests_method(url, **kwargs)<EOL>

This method will perform the real request,
in this way we can customize only the "output" of the API call by
using self.__call_api method.
This method will return the request object.


def __deal_with_pagination(self, url, method, params, data):

all_data = data<EOL>while data.get("<STR_LIT>", {}).get("<STR_LIT>", {}).get("<STR_LIT>"):<EOL><INDENT>url, query = data["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"].split("<STR_LIT:?>", <NUM_LIT:1>)<EOL>for key, value in urlparse.parse_qs(query).items():<EOL><INDENT>params[key] = value<EOL><DEDENT>data = self.__perform_request(url, method, params).json()<EOL>for key, value in data.items():<EOL><INDENT>if isinstance(value, list) and key in all_data:<EOL><INDENT>all_data[key] += value<EOL><DEDENT>else:<EOL><INDENT>all_data[key] = value<EOL><DEDENT><DEDENT><DEDENT>return all_data<EOL>

Perform multiple calls in order to have a full list of elements
when the API are "paginated". (content list is divided in more
than one page)


timeout_str = os.environ.get(REQUEST_TIMEOUT_ENV_VAR)<EOL>if timeout_str:<EOL><INDENT>try:<EOL><INDENT>return float(timeout_str)<EOL><DEDENT>except:<EOL><INDENT>self._log.error('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>timeout_str)<EOL><DEDENT><DEDENT>return None<EOL>

Checks if any timeout for the requests to DigitalOcean is required.
To set a timeout, use the REQUEST_TIMEOUT_ENV_VAR environment
variable.


def get_data(self, url, type=GET, params=None):

if params is None:<EOL><INDENT>params = dict()<EOL><DEDENT>if type is GET:<EOL><INDENT>params.setdefault("<STR_LIT>", <NUM_LIT:200>)<EOL><DEDENT>req = self.__perform_request(url, type, params)<EOL>if req.status_code == <NUM_LIT>:<EOL><INDENT>return True<EOL><DEDENT>if req.status_code == <NUM_LIT>:<EOL><INDENT>raise NotFoundError()<EOL><DEDENT>try:<EOL><INDENT>data = req.json()<EOL><DEDENT>except ValueError as e:<EOL><INDENT>raise JSONReadError(<EOL>'<STR_LIT>' % str(e)<EOL>)<EOL><DEDENT>if not req.ok:<EOL><INDENT>msg = [data[m] for m in ("<STR_LIT:id>", "<STR_LIT:message>") if m in data][<NUM_LIT:1>]<EOL>raise DataReadError(msg)<EOL><DEDENT>self.__init_ratelimit(req.headers)<EOL>pages = data.get("<STR_LIT>", {}).get("<STR_LIT>", {})<EOL>if pages.get("<STR_LIT>") and "<STR_LIT>" not in params:<EOL><INDENT>return self.__deal_with_pagination(url, type, params, data)<EOL><DEDENT>else:<EOL><INDENT>return data<EOL><DEDENT>

This method is a basic implementation of __call_api that checks
errors too. In case of success the method will return True or the
content of the response to the request.

Pagination is automatically detected and handled accordingly


@classmethod<EOL><INDENT>def get_object(cls, api_token, image_id_or_slug):<DEDENT>

if cls._is_string(image_id_or_slug):<EOL><INDENT>image = cls(token=api_token, slug=image_id_or_slug)<EOL>image.load(use_slug=True)<EOL><DEDENT>else:<EOL><INDENT>image = cls(token=api_token, id=image_id_or_slug)<EOL>image.load()<EOL><DEDENT>return image<EOL>

Class method that will return an Image object by ID or slug.

This method is used to validate the type of the image. If it is a
number, it will be considered as an Image ID, instead if it is a
string, it will considered as slug.


if type(value) in [type(u'<STR_LIT>'), type('<STR_LIT>')]:<EOL><INDENT>return True<EOL><DEDENT>elif type(value) in [int, type(<NUM_LIT:2> ** <NUM_LIT:64>)]:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Checks if the value provided is a string (True) or not integer
(False) or something else (None).


params = {'<STR_LIT:name>': self.name,<EOL>'<STR_LIT>': self.region,<EOL>'<STR_LIT:url>': self.url,<EOL>'<STR_LIT>': self.distribution,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT>': self.tags}<EOL>data = self.get_data('<STR_LIT>', type=POST, params=params)<EOL>if data:<EOL><INDENT>for attr in data['<STR_LIT:image>'].keys():<EOL><INDENT>setattr(self, attr, data['<STR_LIT:image>'][attr])<EOL><DEDENT><DEDENT>return self<EOL>

Creates a new custom DigitalOcean Image from the Linux virtual machine
image located at the provided `url`.


identifier = None<EOL>if use_slug or not self.id:<EOL><INDENT>identifier = self.slug<EOL><DEDENT>else:<EOL><INDENT>identifier = self.id<EOL><DEDENT>if not identifier:<EOL><INDENT>raise NotFoundError("<STR_LIT>")<EOL><DEDENT>data = self.get_data("<STR_LIT>" % identifier)<EOL>image_dict = data['<STR_LIT:image>']<EOL>for attr in image_dict.keys():<EOL><INDENT>setattr(self, attr, image_dict[attr])<EOL><DEDENT>return self<EOL>

Load slug.

Loads by id, or by slug if id is not present or use slug is True.


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params={"<STR_LIT:type>": "<STR_LIT>", "<STR_LIT>": new_region_slug}<EOL>)<EOL>


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=PUT,<EOL>params={"<STR_LIT:name>": new_name}<EOL>)<EOL>


def get_data(self, url, headers=dict(), params=dict(), render_json=True):

url = urljoin(self.end_point, url)<EOL>response = requests.get(url, headers=headers, params=params,<EOL>timeout=self.get_timeout())<EOL>if render_json:<EOL><INDENT>return response.json()<EOL><DEDENT>return response.content<EOL>

Customized version of get_data to directly get the data without
using the authentication method.


@classmethod<EOL><INDENT>def get_object(cls, api_token, volume_id):<DEDENT>

volume = cls(token=api_token, id=volume_id)<EOL>volume.load()<EOL>return volume<EOL>

Class method that will return an Volume object by ID.


data = self.get_data('<STR_LIT>',<EOL>type=POST,<EOL>params={'<STR_LIT:name>': self.name,<EOL>'<STR_LIT>': self.region,<EOL>'<STR_LIT>': self.size_gigabytes,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT>': self.filesystem_type,<EOL>'<STR_LIT>': self.filesystem_label<EOL>})<EOL>if data:<EOL><INDENT>self.id = data['<STR_LIT>']['<STR_LIT:id>']<EOL>self.created_at = data['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>return self<EOL>

Creates a Block Storage volume

Note: Every argument and parameter given to this method will be
assigned to the object.

Args:
    name: string - a name for the volume
    region: string - slug identifier for the region
    size_gigabytes: int - size of the Block Storage volume in GiB
    filesystem_type: string, optional - name of the filesystem type the
        volume will be formated with ('ext4' or 'xfs')
    filesystem_label: string, optional - the label to be applied to the
        filesystem, only used in conjunction with filesystem_type

Optional Args:
    description: string - text field to describe a volume


def create_from_snapshot(self, *args, **kwargs):

data = self.get_data('<STR_LIT>',<EOL>type=POST,<EOL>params={'<STR_LIT:name>': self.name,<EOL>'<STR_LIT>': self.snapshot_id,<EOL>'<STR_LIT>': self.region,<EOL>'<STR_LIT>': self.size_gigabytes,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT>': self.filesystem_type,<EOL>'<STR_LIT>': self.filesystem_label<EOL>})<EOL>if data:<EOL><INDENT>self.id = data['<STR_LIT>']['<STR_LIT:id>']<EOL>self.created_at = data['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>return self<EOL>

Creates a Block Storage volume

Note: Every argument and parameter given to this method will be
assigned to the object.

Args:
    name: string - a name for the volume
    snapshot_id: string - unique identifier for the volume snapshot
    size_gigabytes: int - size of the Block Storage volume in GiB
    filesystem_type: string, optional - name of the filesystem type the
        volume will be formated with ('ext4' or 'xfs')
    filesystem_label: string, optional - the label to be applied to the
        filesystem, only used in conjunction with filesystem_type

Optional Args:
    description: string - text field to describe a volume


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params={"<STR_LIT:type>": "<STR_LIT>",<EOL>"<STR_LIT>": droplet_id,<EOL>"<STR_LIT>": region}<EOL>)<EOL>

Attach a Volume to a Droplet.

Args:
    droplet_id: int - droplet id
    region: string - slug identifier for the region


Detach a Volume to a Droplet.

Args:
    droplet_id: int - droplet id
    region: string - slug identifier for the region


def resize(self, size_gigabytes, region):

return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params={"<STR_LIT:type>": "<STR_LIT>",<EOL>"<STR_LIT>": size_gigabytes,<EOL>"<STR_LIT>": region}<EOL>)<EOL>

Detach a Volume to a Droplet.

Args:
    size_gigabytes: int - size of the Block Storage volume in GiB
    region: string - slug identifier for the region


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params={"<STR_LIT:name>": name}<EOL>)<EOL>

Create a snapshot of the volume.

Args:
    name: string - a human-readable name for the snapshot


data = self.get_data("<STR_LIT>" % self.id)<EOL>snapshots = list()<EOL>for jsond in data[u'<STR_LIT>']:<EOL><INDENT>snapshot = Snapshot(**jsond)<EOL>snapshot.token = self.token<EOL>snapshots.append(snapshot)<EOL><DEDENT>return snapshots<EOL>

Retrieve the list of snapshots that have been created from a volume.

Args:


@classmethod<EOL><INDENT>def get_object(cls, api_token, domain, record_id):<DEDENT>

record = cls(token=api_token, domain=domain, id=record_id)<EOL>record.load()<EOL>return record<EOL>

Class method that will return a Record object by ID and the domain.


input_params = {<EOL>"<STR_LIT:type>": self.type,<EOL>"<STR_LIT:data>": self.data,<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT>": self.priority,<EOL>"<STR_LIT:port>": self.port,<EOL>"<STR_LIT>": self.ttl,<EOL>"<STR_LIT>": self.weight,<EOL>"<STR_LIT>": self.flags,<EOL>"<STR_LIT>": self.tags<EOL>}<EOL>data = self.get_data(<EOL>"<STR_LIT>" % (self.domain),<EOL>type=POST,<EOL>params=input_params,<EOL>)<EOL>if data:<EOL><INDENT>self.id = data['<STR_LIT>']['<STR_LIT:id>']<EOL><DEDENT>

Creates a new record for a domain.

Args:
    type (str): The type of the DNS record (e.g. A, CNAME, TXT).
    name (str): The host name, alias, or service being defined by the
        record.
    data (int): Variable data depending on record type.
    priority (int): The priority for SRV and MX records.
    port (int): The port for SRV records.
    ttl (int): The time to live for the record, in seconds.
    weight (int): The weight for SRV records.
    flags (int): An unsigned integer between 0-255 used for CAA records.
    tags (string): The parameter tag for CAA records. Valid values are
        "issue", "wildissue", or "iodef"


return self.get_data(<EOL>"<STR_LIT>" % (self.domain, self.id),<EOL>type=DELETE,<EOL>)<EOL>


data = {<EOL>"<STR_LIT:type>": self.type,<EOL>"<STR_LIT:data>": self.data,<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT>": self.priority,<EOL>"<STR_LIT:port>": self.port,<EOL>"<STR_LIT>": self.ttl,<EOL>"<STR_LIT>": self.weight,<EOL>"<STR_LIT>": self.flags,<EOL>"<STR_LIT>": self.tags<EOL>}<EOL>return self.get_data(<EOL>"<STR_LIT>" % (self.domain, self.id),<EOL>type=PUT,<EOL>params=data<EOL>)<EOL>


acct = cls(token=api_token)<EOL>acct.load()<EOL>return acct<EOL>

Class method that will return an Account object.


tags = self.get_data("<STR_LIT>" % self.name)<EOL>tag = tags['<STR_LIT>']<EOL>for attr in tag.keys():<EOL><INDENT>setattr(self, attr, tag[attr])<EOL><DEDENT>return self<EOL>


for attr in kwargs.keys():<EOL><INDENT>setattr(self, attr, kwargs[attr])<EOL><DEDENT>params = {"<STR_LIT:name>": self.name}<EOL>output = self.get_data("<STR_LIT>", type="<STR_LIT:POST>", params=params)<EOL>if output:<EOL><INDENT>self.name = output['<STR_LIT>']['<STR_LIT:name>']<EOL>self.resources = output['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>


def __get_resources(self, resources, method):

tagged = self.get_data(<EOL>'<STR_LIT>' % self.name, params={<EOL>"<STR_LIT>": resources<EOL>},<EOL>type=method,<EOL>)<EOL>return tagged<EOL>

Method used to talk directly to the API (TAGs' Resources)


return self.__get_resources(resources, method='<STR_LIT:POST>')<EOL>

Add to the resources to this tag.

Attributes accepted at creation time:
    resources: array - See API.


return self.__get_resources(resources, method='<STR_LIT>')<EOL>

Remove resources from this tag.

Attributes accepted at creation time:
    resources: array - See API.


resources = []<EOL>if not isinstance(data, list): return data<EOL>for a_droplet in data:<EOL><INDENT>res = {}<EOL>try:<EOL><INDENT>if isinstance(a_droplet, unicode):<EOL><INDENT>res = {"<STR_LIT>": a_droplet, "<STR_LIT>": "<STR_LIT>"}<EOL><DEDENT><DEDENT>except NameError:<EOL><INDENT>pass<EOL><DEDENT>if isinstance(a_droplet, str) or isinstance(a_droplet, int):<EOL><INDENT>res = {"<STR_LIT>": str(a_droplet), "<STR_LIT>": "<STR_LIT>"}<EOL><DEDENT>elif isinstance(a_droplet, Droplet):<EOL><INDENT>res = {"<STR_LIT>": str(a_droplet.id), "<STR_LIT>": "<STR_LIT>"}<EOL><DEDENT>if len(res) > <NUM_LIT:0>:<EOL><INDENT>resources.append(res)<EOL><DEDENT><DEDENT>return resources<EOL>

Private method to extract from a value, the resources.
It will check the type of object in the array provided and build
the right structure for the API.


droplets = droplet<EOL>if not isinstance(droplets, list):<EOL><INDENT>droplets = [droplet]<EOL><DEDENT>resources = self.__extract_resources_from_droplets(droplets)<EOL>if len(resources) > <NUM_LIT:0>:<EOL><INDENT>return self.__add_resources(resources)<EOL><DEDENT>return False<EOL>

Add the Tag to a Droplet.

Attributes accepted at creation time:
    droplet: array of string or array of int, or array of Droplets.


droplets = droplet<EOL>if not isinstance(droplets, list):<EOL><INDENT>droplets = [droplet]<EOL><DEDENT>resources = self.__extract_resources_from_droplets(droplets)<EOL>if len(resources) > <NUM_LIT:0>:<EOL><INDENT>return self.__remove_resources(resources)<EOL><DEDENT>return False<EOL>

Remove the Tag from the Droplet.

Attributes accepted at creation time:
    droplet: array of string or array of int, or array of Droplets.


@classmethod<EOL><INDENT>def get_object(cls, api_token, id):<DEDENT>

load_balancer = cls(token=api_token, id=id)<EOL>load_balancer.load()<EOL>return load_balancer<EOL>

Class method that will return a LoadBalancer object by its ID.

Args:
    api_token (str): DigitalOcean API token
    id (str): Load Balancer ID


data = self.get_data('<STR_LIT>' % self.id, type=GET)<EOL>load_balancer = data['<STR_LIT>']<EOL>for attr in load_balancer.keys():<EOL><INDENT>if attr == '<STR_LIT>':<EOL><INDENT>health_check = HealthCheck(**load_balancer['<STR_LIT>'])<EOL>setattr(self, attr, health_check)<EOL><DEDENT>elif attr == '<STR_LIT>':<EOL><INDENT>sticky_ses = StickySesions(**load_balancer['<STR_LIT>'])<EOL>setattr(self, attr, sticky_ses)<EOL><DEDENT>elif attr == '<STR_LIT>':<EOL><INDENT>rules = list()<EOL>for rule in load_balancer['<STR_LIT>']:<EOL><INDENT>rules.append(ForwardingRule(**rule))<EOL><DEDENT>setattr(self, attr, rules)<EOL><DEDENT>else:<EOL><INDENT>setattr(self, attr, load_balancer[attr])<EOL><DEDENT><DEDENT>return self<EOL>

Loads updated attributues for a LoadBalancer object.

Requires self.id to be set.


rules_dict = [rule.__dict__ for rule in self.forwarding_rules]<EOL>params = {'<STR_LIT:name>': self.name, '<STR_LIT>': self.region,<EOL>'<STR_LIT>': rules_dict,<EOL>'<STR_LIT>': self.redirect_http_to_https}<EOL>if self.droplet_ids and self.tag:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>elif self.tag:<EOL><INDENT>params['<STR_LIT>'] = self.tag<EOL><DEDENT>else:<EOL><INDENT>params['<STR_LIT>'] = self.droplet_ids<EOL><DEDENT>if self.algorithm:<EOL><INDENT>params['<STR_LIT>'] = self.algorithm<EOL><DEDENT>if self.health_check:<EOL><INDENT>params['<STR_LIT>'] = self.health_check.__dict__<EOL><DEDENT>if self.sticky_sessions:<EOL><INDENT>params['<STR_LIT>'] = self.sticky_sessions.__dict__<EOL><DEDENT>data = self.get_data('<STR_LIT>', type=POST, params=params)<EOL>if data:<EOL><INDENT>self.id = data['<STR_LIT>']['<STR_LIT:id>']<EOL>self.ip = data['<STR_LIT>']['<STR_LIT>']<EOL>self.algorithm = data['<STR_LIT>']['<STR_LIT>']<EOL>self.health_check = HealthCheck(<EOL>**data['<STR_LIT>']['<STR_LIT>'])<EOL>self.sticky_sessions = StickySesions(<EOL>**data['<STR_LIT>']['<STR_LIT>'])<EOL>self.droplet_ids = data['<STR_LIT>']['<STR_LIT>']<EOL>self.status = data['<STR_LIT>']['<STR_LIT:status>']<EOL>self.created_at = data['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>return self<EOL>

Creates a new LoadBalancer.

Note: Every argument and parameter given to this method will be
assigned to the object.

Args:
    name (str): The Load Balancer's name
    region (str): The slug identifier for a DigitalOcean region
    algorithm (str, optional): The load balancing algorithm to be
        used. Currently, it must be either "round_robin" or
        "least_connections"
    forwarding_rules (obj:`list`): A list of `ForwrdingRules` objects
    health_check (obj, optional): A `HealthCheck` object
    sticky_sessions (obj, optional): A `StickySessions` object
    redirect_http_to_https (bool, optional): A boolean indicating
        whether HTTP requests to the Load Balancer should be
        redirected to HTTPS
    droplet_ids (obj:`list` of `int`): A list of IDs representing
        Droplets to be added to the Load Balancer (mutually
        exclusive with 'tag')
    tag (str): A string representing a DigitalOcean Droplet tag
        (mutually exclusive with 'droplet_ids')


forwarding_rules = [rule.__dict__ for rule in self.forwarding_rules]<EOL>data = {<EOL>'<STR_LIT:name>': self.name,<EOL>'<STR_LIT>': self.region['<STR_LIT>'],<EOL>'<STR_LIT>': forwarding_rules,<EOL>'<STR_LIT>': self.redirect_http_to_https<EOL>}<EOL>if self.tag:<EOL><INDENT>data['<STR_LIT>'] = self.tag<EOL><DEDENT>else:<EOL><INDENT>data['<STR_LIT>'] = self.droplet_ids<EOL><DEDENT>if self.algorithm:<EOL><INDENT>data["<STR_LIT>"] = self.algorithm<EOL><DEDENT>if self.health_check:<EOL><INDENT>data['<STR_LIT>'] = self.health_check.__dict__<EOL><DEDENT>if self.sticky_sessions:<EOL><INDENT>data['<STR_LIT>'] = self.sticky_sessions.__dict__<EOL><DEDENT>return self.get_data("<STR_LIT>" % self.id,<EOL>type=PUT,<EOL>params=data)<EOL>


return self.get_data('<STR_LIT>' % self.id, type=DELETE)<EOL>


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params={"<STR_LIT>": droplet_ids}<EOL>)<EOL>

Assign a LoadBalancer to a Droplet.

Args:
    droplet_ids (obj:`list` of `int`): A list of Droplet IDs


return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=DELETE,<EOL>params={"<STR_LIT>": droplet_ids}<EOL>)<EOL>

Unassign a LoadBalancer.

Args:
    droplet_ids (obj:`list` of `int`): A list of Droplet IDs


rules_dict = [rule.__dict__ for rule in forwarding_rules]<EOL>return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=POST,<EOL>params={"<STR_LIT>": rules_dict}<EOL>)<EOL>

Adds new forwarding rules to a LoadBalancer.

Args:
    forwarding_rules (obj:`list`): A list of `ForwrdingRules` objects


rules_dict = [rule.__dict__ for rule in forwarding_rules]<EOL>return self.get_data(<EOL>"<STR_LIT>" % self.id,<EOL>type=DELETE,<EOL>params={"<STR_LIT>": rules_dict}<EOL>)<EOL>

Removes existing forwarding rules from a LoadBalancer.

Args:
    forwarding_rules (obj:`list`): A list of `ForwrdingRules` objects


Config.incremental = incremental<EOL>Config.replace_any = replace_any<EOL>returncode = <NUM_LIT:0><EOL>for src_entry in src:<EOL><INDENT>for file, error, exc_type, tb in retype_path(<EOL>Path(src_entry),<EOL>pyi_dir=Path(pyi_dir),<EOL>targets=Path(target_dir),<EOL>src_explicitly_given=True,<EOL>quiet=quiet,<EOL>hg=hg,<EOL>):<EOL><INDENT>print(f'<STR_LIT>', file=sys.stderr)<EOL>if traceback:<EOL><INDENT>print('<STR_LIT>', file=sys.stderr)<EOL>for line in tb:<EOL><INDENT>print(line, file=sys.stderr, end='<STR_LIT>')<EOL><DEDENT>print(f'<STR_LIT>', file=sys.stderr)<EOL><DEDENT>returncode += <NUM_LIT:1><EOL><DEDENT><DEDENT>if not src and not quiet:<EOL><INDENT>print('<STR_LIT>', file=sys.stderr)<EOL><DEDENT>sys.exit(min(returncode, <NUM_LIT>))<EOL>

Re-apply type annotations from .pyi stubs to your codebase.