@call_decorator<EOL><INDENT>def __call__(self, input_or_adv, label=None, unpack=True,<EOL>max_iter=<NUM_LIT:100>,<EOL>eta=<NUM_LIT>):<DEDENT>

a = input_or_adv<EOL>del input_or_adv<EOL>del label<EOL>del unpack<EOL>if not a.has_gradient():<EOL><INDENT>return<EOL><DEDENT>if a.target_class() is not None:<EOL><INDENT>logging.fatal('<STR_LIT>')<EOL>return<EOL><DEDENT>l2_norm = np.linalg.norm(a.original_image)<EOL>min_, max_ = a.bounds()<EOL>perturbed_image = a.original_image.copy()<EOL>for i in range(max_iter):<EOL><INDENT>logits, gradients, is_adversarial =a.predictions_and_gradient(perturbed_image)<EOL>if is_adversarial:<EOL><INDENT>return<EOL><DEDENT>score = np.max(softmax(logits))<EOL>gradients = -gradients / score<EOL>gradient_l2_norm = np.linalg.norm(gradients)<EOL>delta = self._delta(eta, l2_norm, score,<EOL>gradient_l2_norm, a.num_classes())<EOL>current_pertubation = self._perturbation(delta,<EOL>gradients,<EOL>gradient_l2_norm)<EOL>perturbed_image += current_pertubation<EOL>perturbed_image = np.clip(perturbed_image, min_, max_)<EOL><DEDENT>

Parameters
----------
input_or_adv : `numpy.ndarray` or :class:`Adversarial`
    The original, unperturbed input as a `numpy.ndarray` or
    an :class:`Adversarial` instance.
label : int
    The reference label of the original input. Must be passed
    if `a` is a `numpy.ndarray`, must not be passed if `a` is
    an :class:`Adversarial` instance.
unpack : bool
    If true, returns the adversarial input, otherwise returns
    the Adversarial object.
max_iter : int
    The maximum number of iterations.
eta : float
    the eta coefficient


@call_decorator<EOL><INDENT>def __call__(self, input_or_adv, label=None, unpack=True):<DEDENT>

a = input_or_adv<EOL>del input_or_adv<EOL>del label<EOL>del unpack<EOL>image = a.original_image<EOL>dtype = a.original_image.dtype<EOL>min_, max_ = a.bounds()<EOL>shape = image.shape<EOL>n = image.size<EOL>image = image.flatten()<EOL>x0 = nprng.uniform(min_, max_, size=image.shape)<EOL>bounds = [(min_, max_)] * n<EOL>options = {'<STR_LIT>': <NUM_LIT>}<EOL>def fun(x, *args):<EOL><INDENT>"""<STR_LIT>"""<EOL>distance = a.normalized_distance(x.reshape(shape))<EOL>return distance.value, distance.gradient.reshape(-<NUM_LIT:1>)<EOL><DEDENT>def eq_constraint(x, *args):<EOL><INDENT>"""<STR_LIT>"""<EOL>_, is_adv = a.predictions(x.reshape(shape).astype(dtype))<EOL>if is_adv:<EOL><INDENT>return <NUM_LIT:0.><EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:1.><EOL><DEDENT><DEDENT>constraints = [<EOL>{<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>'<STR_LIT>': eq_constraint,<EOL>}<EOL>]<EOL>result = so.minimize(<EOL>fun,<EOL>x0,<EOL>method='<STR_LIT>',<EOL>jac=True,<EOL>bounds=bounds,<EOL>constraints=constraints,<EOL>options=options)<EOL>a.predictions(result.x.reshape(shape).astype(dtype))<EOL>

Uses SLSQP to minimize the distance between the image and the
        adversarial under the constraint that the image is adversarial.

        Parameters
        ----------
        input_or_adv : `numpy.ndarray` or :class:`Adversarial`
            The original, correctly classified image. If image is a
            numpy array, label must be passed as well. If image is
            an :class:`Adversarial` instance, label must not be passed.
        label : int
            The reference label of the original image. Must be passed
            if image is a numpy array, must not be passed if image is
            an :class:`Adversarial` instance.
        unpack : bool
            If true, returns the adversarial image, otherwise returns
            the Adversarial object.


if color_axis == <NUM_LIT:2>:<EOL><INDENT>image = _transpose_image(image)<EOL><DEDENT>dfdx = np.zeros_like(image)<EOL>dfdx[:, :, <NUM_LIT:0>] = image[:, :, <NUM_LIT:1>] - image[:, :, <NUM_LIT:0>]<EOL>dfdx[:, :, -<NUM_LIT:1>] = image[:, :, -<NUM_LIT:1>] - image[:, :, -<NUM_LIT:2>]<EOL>dfdx[:, :, <NUM_LIT:1>:-<NUM_LIT:1>] = <NUM_LIT:0.5> * (image[:, :, <NUM_LIT:2>:] - image[:, :, :-<NUM_LIT:2>])<EOL>dfdy = np.zeros_like(image)<EOL>dfdy[:, <NUM_LIT:0>, :] = image[:, <NUM_LIT:1>, :] - image[:, <NUM_LIT:0>, :]<EOL>dfdy[:, -<NUM_LIT:1>, :] = image[:, -<NUM_LIT:1>, :] - image[:, -<NUM_LIT:2>, :]<EOL>dfdy[:, <NUM_LIT:1>:-<NUM_LIT:1>, :] = <NUM_LIT:0.5> * (image[:, <NUM_LIT:2>:, :] - image[:, :-<NUM_LIT:2>, :])<EOL>return dfdx, dfdy<EOL>

Difference map of the image.
    Approximate derivatives of the function image[c, :, :]
    (e.g. PyTorch) or image[:, :, c] (e.g. Keras).
    dfdx, dfdy = difference_map(image)
    In:
    image: numpy.ndarray
        of shape C x h x w or h x w x C, with C = 1 or C = 3
        (color channels), h, w >= 3, and [type] is 'Float' or
        'Double'. Contains the values of functions f_b:
        R ^ 2 -> R ^ C, b = 1, ..., B, on the grid
        {0, ..., h - 1} x {0, ..., w - 1}.
    Out:
    dfdx: numpy.ndarray
    dfdy: numpy.ndarray
        of shape C x h x w or h x w x C contain the x and
        y derivatives of f at the points on the grid,
        approximated by central differences (except on
        boundaries):
        For c = 0, ... , C, i = 1, ..., h - 2,
        j = 1, ..., w - 2.
        e.g. for shape = c x h x w:
        dfdx[c, i, j] = (image[c, i, j + 1] -
            image[c, i, j - 1]) / 2
        dfdx[c, i, j] = (image[c, i + 1, j] -
            image[c, i - 1, j]) / 2
    positive x-direction is along rows from left to right.
    positive y-direction is along columns from above to below.


def _compose(image, vec_field, color_axis):

if color_axis == <NUM_LIT:2>:<EOL><INDENT>image = _transpose_image(image)<EOL><DEDENT>c, h, w = image.shape  <EOL>hrange = np.arange(h)<EOL>wrange = np.arange(w)<EOL>MGx, MGy = np.meshgrid(wrange, hrange)<EOL>defMGx = (MGx + vec_field[:, :, <NUM_LIT:0>]).clip(<NUM_LIT:0>, w - <NUM_LIT:1>)<EOL>defMGy = (MGy + vec_field[:, :, <NUM_LIT:1>]).clip(<NUM_LIT:0>, h - <NUM_LIT:1>)<EOL>new_image = np.empty_like(image)<EOL>for channel in range(c):<EOL><INDENT>interpolation = RectBivariateSpline(hrange, wrange, image[channel],<EOL>kx=<NUM_LIT:1>, ky=<NUM_LIT:1>)<EOL>new_image[channel] = interpolation(defMGy, defMGx, grid=False)<EOL><DEDENT>if color_axis == <NUM_LIT:2>:<EOL><INDENT>return _re_transpose_image(new_image)<EOL><DEDENT>else:<EOL><INDENT>return new_image<EOL><DEDENT>

Calculate the composition of the function image with the vector
    field vec_field by interpolation.
    new_func = compose(image, vec_field)
    In:
    image: numpy.ndarray
        of shape C x h x w with C = 3 or C = 1 (color channels),
        h, w >= 2, and [type] = 'Float' or 'Double'.
        Contains the values of a function f: R ^ 2 -> R ^ C
        on the grid {0, ..., h - 1} x {0, ..., w - 1}.
    vec_field: numpy.array
        of shape (h, w, 2)
    vec_field[y, x, 0] is the x-coordinate of the vector vec_field[y, x]
    vec_field[y, x, 1] is the y-coordinate of the vector vec_field[y, x]
    positive x-direction is along rows from left to right
    positive y-direction is along columns from above to below


def _create_vec_field(fval, gradf, d1x, d2x, color_axis, smooth=<NUM_LIT:0>):

if color_axis == <NUM_LIT:2>:<EOL><INDENT>gradf = _transpose_image(gradf)<EOL><DEDENT>c, h, w = gradf.shape  <EOL>alpha1 = np.sum(gradf * d1x, axis=<NUM_LIT:0>)<EOL>alpha2 = np.sum(gradf * d2x, axis=<NUM_LIT:0>)<EOL>norm_squared_alpha = (alpha1 ** <NUM_LIT:2>).sum() + (alpha2 ** <NUM_LIT:2>).sum()<EOL>if smooth > <NUM_LIT:0>:<EOL><INDENT>alpha1 = gaussian_filter(alpha1, smooth)<EOL>alpha2 = gaussian_filter(alpha2, smooth)<EOL>norm_squared_alpha = (alpha1 ** <NUM_LIT:2>).sum() + (alpha2 ** <NUM_LIT:2>).sum()<EOL>alpha1 = gaussian_filter(alpha1, smooth)<EOL>alpha2 = gaussian_filter(alpha2, smooth)<EOL><DEDENT>vec_field = np.empty((h, w, <NUM_LIT:2>))<EOL>vec_field[:, :, <NUM_LIT:0>] = -fval * alpha1 / norm_squared_alpha<EOL>vec_field[:, :, <NUM_LIT:1>] = -fval * alpha2 / norm_squared_alpha<EOL>return vec_field<EOL>

Calculate the deformation vector field
    In:
    fval: float
    gradf: numpy.ndarray
        of shape C x h x w with C = 3 or C = 1
        (color channels), h, w >= 1.
    d1x: numpy.ndarray
        of shape C x h x w and [type] = 'Float' or 'Double'.
    d2x: numpy.ndarray
        of shape C x h x w and [type] = 'Float' or 'Double'.
    smooth: float
        Width of the Gaussian kernel used for smoothing
        (default is 0 for no smoothing).
    Out:
    vec_field: numpy.ndarray
        of shape (2, h, w).


@call_decorator<EOL><INDENT>def __call__(self, input_or_adv, unpack=True, max_iter=<NUM_LIT:100>,<EOL>max_norm=np.inf, label=None, smooth=<NUM_LIT:1.0>, subsample=<NUM_LIT:10>):<DEDENT>

a = input_or_adv<EOL>del input_or_adv<EOL>del label<EOL>del unpack<EOL>if not a.has_gradient():<EOL><INDENT>return<EOL><DEDENT>perturbed = a.original_image.copy()  <EOL>image_original = a.original_image.copy()<EOL>target_class = a.target_class()<EOL>targeted = target_class is not None<EOL>original_label = a.original_class<EOL>if targeted:<EOL><INDENT>logits, _ = a.predictions(perturbed)<EOL>pred_sorted = (-logits).argsort()<EOL>index_of_target_class, = np.where(pred_sorted == target_class)<EOL>ind_of_candidates = index_of_target_class<EOL><DEDENT>else:<EOL><INDENT>logging.info('<STR_LIT>'.format(subsample))<EOL>assert isinstance(subsample, int)<EOL>assert subsample >= <NUM_LIT:2><EOL>ind_of_candidates = np.arange(<NUM_LIT:1>, subsample)<EOL><DEDENT>num_classes = ind_of_candidates.size<EOL>n = <NUM_LIT:0>  <EOL>color_axis = a.channel_axis(batch=False)  <EOL>assert color_axis in [<NUM_LIT:0>, <NUM_LIT:2>]<EOL>hw = [perturbed.shape[i] for i in range(perturbed.ndim)<EOL>if i != color_axis]<EOL>h, w = hw<EOL>logits, is_adv = a.predictions(perturbed)<EOL>candidates = np.argsort(-logits)[ind_of_candidates]<EOL>fx = logits - logits[original_label]<EOL>norm_full = <NUM_LIT:0>  <EOL>vec_field_full = np.zeros((h, w, <NUM_LIT:2>))  <EOL>current_label = original_label<EOL>logging.info('<STR_LIT>')<EOL>logging.info('<STR_LIT>'.format(current_label))<EOL>for step in range(max_iter):<EOL><INDENT>n += <NUM_LIT:1><EOL>_, is_adv = a.predictions(perturbed)<EOL>if is_adv:<EOL><INDENT>a.predictions(perturbed)<EOL>logging.info(<EOL>'<STR_LIT>'.format(<EOL>original_label, current_label))<EOL>self.vector_field = vec_field_full<EOL>return<EOL><DEDENT>d1x, d2x = _difference_map(perturbed, color_axis)<EOL>logits_for_grad = np.zeros_like(logits)<EOL>logits_for_grad[original_label] = <NUM_LIT:1><EOL>grad_original = a.backward(logits_for_grad, perturbed)<EOL>norm_min = np.inf<EOL>for target_no in range(num_classes):<EOL><INDENT>target_label = candidates[target_no]<EOL>logits_for_grad = np.zeros_like(logits)<EOL>logits_for_grad[target_label] = <NUM_LIT:1><EOL>grad_target = a.backward(logits_for_grad, perturbed)<EOL>dfx = grad_target - grad_original<EOL>f_im = fx[target_label]<EOL>vec_field_target = _create_vec_field(<EOL>f_im, dfx, d1x, d2x, color_axis, smooth)<EOL>vec_field_target += vec_field_full<EOL>norm_target = np.linalg.norm(vec_field_target.ravel())<EOL>if norm_target < norm_min:<EOL><INDENT>norm_min = norm_target<EOL>vec_field_min = vec_field_target<EOL><DEDENT><DEDENT>perturbed = _compose(image_original.copy(), vec_field_min,<EOL>color_axis)<EOL>vec_field_full = vec_field_min<EOL>norm_full = norm_min<EOL>logits, _ = a.predictions(perturbed)<EOL>current_label = np.argmax(logits)<EOL>fx = logits - logits[current_label]<EOL>logging.info('<STR_LIT>'.format(n))<EOL>logging.info('<STR_LIT>'.format(current_label))<EOL>logging.info('<STR_LIT>'.format(norm_full))<EOL><DEDENT>logits, _ = a.predictions(perturbed)<EOL>current_label = np.argmax(logits)<EOL>logging.info('<STR_LIT>'.format(original_label, current_label))<EOL>a.predictions(perturbed)<EOL>self.vector_field = vec_field_full<EOL>return<EOL>

Parameters
        ----------
        input_or_adv : `numpy.ndarray` or :class:`Adversarial`
            The original, unperturbed input as a `numpy.ndarray` or
            an :class:`Adversarial` instance.
        label : int
            The reference label of the original input. Must be passed
            if `a` is a `numpy.ndarray`, must not be passed if `a` is
            an :class:`Adversarial` instance.
        unpack : bool
            If true, returns the adversarial input, otherwise returns
            the Adversarial object.
        max_iter : int > 0
            Maximum number of iterations (default max_iter = 100).
        max_norm : float
            Maximum l2 norm of vector field (default max_norm = numpy.inf).
        smooth : float >= 0
            Width of the Gaussian kernel used for smoothing.
            (default is smooth = 0 for no smoothing).
        subsample : int >= 2
            Limit on the number of the most likely classes that should
            be considered. A small value is usually sufficient and much
            faster. (default subsample = 10)


return self.__best_adversarial<EOL>


return self.__best_adversarial_output<EOL>

The model predictions for the best adversarial found so far.

        None if no adversarial has been found.


if self.output is None:<EOL><INDENT>return None<EOL><DEDENT>return np.argmax(self.output)<EOL>

The argmax of the model predictions for the best adversarial found so far.

        None if no adversarial has been found.


return self.__best_distance<EOL>

The distance of the adversarial input to the original input.


return self.__original_image<EOL>


<EOL>return self.__original_class<EOL>

The class of the original input (ground-truth, not model prediction).


@property<EOL><INDENT>def _model(self):  <DEDENT>

return self.__model<EOL>


@property<EOL><INDENT>def _criterion(self):  <DEDENT>

return self.__criterion<EOL>


@property<EOL><INDENT>def _distance(self):  <DEDENT>

return self.__distance<EOL>


return self.__distance(<EOL>self.__original_image_for_distance,<EOL>image,<EOL>bounds=self.bounds())<EOL>

Calculates the distance of a given image to the
        original image.

        Parameters
        ----------
        image : `numpy.ndarray`
            The image that should be compared to the original image.

        Returns
        -------
        :class:`Distance`
            The distance between the given image and the original image.


return self.__threshold is not Noneand self.__best_distance <= self.__threshold<EOL>

Returns True if a threshold is given and the currently
        best adversarial distance is smaller than the threshold.


def __is_adversarial(self, image, predictions, in_bounds):

is_adversarial = self.__criterion.is_adversarial(<EOL>predictions, self.__original_class)<EOL>assert isinstance(is_adversarial, bool) orisinstance(is_adversarial, np.bool_)<EOL>if is_adversarial:<EOL><INDENT>is_best, distance = self.__new_adversarial(<EOL>image, predictions, in_bounds)<EOL><DEDENT>else:<EOL><INDENT>is_best = False<EOL>distance = None<EOL><DEDENT>return is_adversarial, is_best, distance<EOL>

Interface to criterion.is_adverarial that calls
        __new_adversarial if necessary.

        Parameters
        ----------
        predictions : :class:`numpy.ndarray`
            A vector with the pre-softmax predictions for some image.
        label : int
            The label of the unperturbed reference image.


try:<EOL><INDENT>target_class = self.__criterion.target_class()<EOL><DEDENT>except AttributeError:<EOL><INDENT>target_class = None<EOL><DEDENT>return target_class<EOL>

Interface to criterion.target_class for attacks.


axis = self.__model.channel_axis()<EOL>if not batch:<EOL><INDENT>axis = axis - <NUM_LIT:1><EOL><DEDENT>return axis<EOL>

Interface to model.channel_axis for attacks.

        Parameters
        ----------
        batch : bool
            Controls whether the index of the axis for a batch of images
            (4 dimensions) or a single image (3 dimensions) should be returned.


try:<EOL><INDENT>self.__model.gradient<EOL>self.__model.predictions_and_gradient<EOL><DEDENT>except AttributeError:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return True<EOL><DEDENT>

Returns true if _backward and _forward_backward can be called
        by an attack, False otherwise.


def predictions(self, image, strict=True, return_details=False):

in_bounds = self.in_bounds(image)<EOL>assert not strict or in_bounds<EOL>self._total_prediction_calls += <NUM_LIT:1><EOL>predictions = self.__model.predictions(image)<EOL>is_adversarial, is_best, distance = self.__is_adversarial(<EOL>image, predictions, in_bounds)<EOL>assert predictions.ndim == <NUM_LIT:1><EOL>if return_details:<EOL><INDENT>return predictions, is_adversarial, is_best, distance<EOL><DEDENT>else:<EOL><INDENT>return predictions, is_adversarial<EOL><DEDENT>

Interface to model.predictions for attacks.

        Parameters
        ----------
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).
        strict : bool
            Controls if the bounds for the pixel values should be checked.


def batch_predictions(<EOL>self, images, greedy=False, strict=True, return_details=False):

if strict:<EOL><INDENT>in_bounds = self.in_bounds(images)<EOL>assert in_bounds<EOL><DEDENT>self._total_prediction_calls += len(images)<EOL>predictions = self.__model.batch_predictions(images)<EOL>assert predictions.ndim == <NUM_LIT:2><EOL>assert predictions.shape[<NUM_LIT:0>] == images.shape[<NUM_LIT:0>]<EOL>if return_details:<EOL><INDENT>assert greedy<EOL><DEDENT>adversarials = []<EOL>for i in range(len(predictions)):<EOL><INDENT>if strict:<EOL><INDENT>in_bounds_i = True<EOL><DEDENT>else:<EOL><INDENT>in_bounds_i = self.in_bounds(images[i])<EOL><DEDENT>is_adversarial, is_best, distance = self.__is_adversarial(<EOL>images[i], predictions[i], in_bounds_i)<EOL>if is_adversarial and greedy:<EOL><INDENT>if return_details:<EOL><INDENT>return predictions, is_adversarial, i, is_best, distance<EOL><DEDENT>else:<EOL><INDENT>return predictions, is_adversarial, i<EOL><DEDENT><DEDENT>adversarials.append(is_adversarial)<EOL><DEDENT>if greedy:  <EOL><INDENT>if return_details:<EOL><INDENT>return predictions, False, None, False, None<EOL><DEDENT>else:<EOL><INDENT>return predictions, False, None<EOL><DEDENT><DEDENT>is_adversarial = np.array(adversarials)<EOL>assert is_adversarial.ndim == <NUM_LIT:1><EOL>assert is_adversarial.shape[<NUM_LIT:0>] == images.shape[<NUM_LIT:0>]<EOL>return predictions, is_adversarial<EOL>

Interface to model.batch_predictions for attacks.

        Parameters
        ----------
        images : `numpy.ndarray`
            Batch of inputs with shape as expected by the model.
        greedy : bool
            Whether the first adversarial should be returned.
        strict : bool
            Controls if the bounds for the pixel values should be checked.


def gradient(self, image=None, label=None, strict=True):

assert self.has_gradient()<EOL>if image is None:<EOL><INDENT>image = self.__original_image<EOL><DEDENT>if label is None:<EOL><INDENT>label = self.__original_class<EOL><DEDENT>assert not strict or self.in_bounds(image)<EOL>self._total_gradient_calls += <NUM_LIT:1><EOL>gradient = self.__model.gradient(image, label)<EOL>assert gradient.shape == image.shape<EOL>return gradient<EOL>

Interface to model.gradient for attacks.

        Parameters
        ----------
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).
            Defaults to the original image.
        label : int
            Label used to calculate the loss that is differentiated.
            Defaults to the original label.
        strict : bool
            Controls if the bounds for the pixel values should be checked.


def predictions_and_gradient(<EOL>self, image=None, label=None, strict=True, return_details=False):

assert self.has_gradient()<EOL>if image is None:<EOL><INDENT>image = self.__original_image<EOL><DEDENT>if label is None:<EOL><INDENT>label = self.__original_class<EOL><DEDENT>in_bounds = self.in_bounds(image)<EOL>assert not strict or in_bounds<EOL>self._total_prediction_calls += <NUM_LIT:1><EOL>self._total_gradient_calls += <NUM_LIT:1><EOL>predictions, gradient = self.__model.predictions_and_gradient(image, label)  <EOL>is_adversarial, is_best, distance = self.__is_adversarial(<EOL>image, predictions, in_bounds)<EOL>assert predictions.ndim == <NUM_LIT:1><EOL>assert gradient.shape == image.shape<EOL>if return_details:<EOL><INDENT>return predictions, gradient, is_adversarial, is_best, distance<EOL><DEDENT>else:<EOL><INDENT>return predictions, gradient, is_adversarial<EOL><DEDENT>

Interface to model.predictions_and_gradient for attacks.

        Parameters
        ----------
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).
            Defaults to the original image.
        label : int
            Label used to calculate the loss that is differentiated.
            Defaults to the original label.
        strict : bool
            Controls if the bounds for the pixel values should be checked.


def backward(self, gradient, image=None, strict=True):

assert self.has_gradient()<EOL>assert gradient.ndim == <NUM_LIT:1><EOL>if image is None:<EOL><INDENT>image = self.__original_image<EOL><DEDENT>assert not strict or self.in_bounds(image)<EOL>self._total_gradient_calls += <NUM_LIT:1><EOL>gradient = self.__model.backward(gradient, image)<EOL>assert gradient.shape == image.shape<EOL>return gradient<EOL>

Interface to model.backward for attacks.

        Parameters
        ----------
        gradient : `numpy.ndarray`
            Gradient of some loss w.r.t. the logits.
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).

        Returns
        -------
        gradient : `numpy.ndarray`
            The gradient w.r.t the image.

        See Also
        --------
        :meth:`gradient`


if weights_uri is None:<EOL><INDENT>logging.info("<STR_LIT>")<EOL>return<EOL><DEDENT>hash_digest = sha256_hash(weights_uri)<EOL>local_path = home_directory_path(FOLDER, hash_digest)<EOL>exists_locally = path_exists(local_path)<EOL>filename = _filename_from_uri(weights_uri)<EOL>file_path = os.path.join(local_path, filename)<EOL>if exists_locally:<EOL><INDENT>logging.info("<STR_LIT>")  <EOL><DEDENT>else:<EOL><INDENT>_download(file_path, weights_uri, local_path)<EOL><DEDENT>if unzip:<EOL><INDENT>file_path = _extract(local_path, filename)<EOL><DEDENT>return file_path<EOL>

Provides utilities to download and extract packages
containing model weights when creating foolbox-zoo compatible
repositories, if the weights are not part of the repository itself.

Examples
--------

Download and unzip weights:

>>> from foolbox import zoo
>>> url = 'https://github.com/MadryLab/mnist_challenge_models/raw/master/secret.zip'  # noqa F501
>>> weights_path = zoo.fetch_weights(url, unzip=True)

:param weights_uri: the URI to fetch the weights from
:param unzip: should be `True` if the file to be downloaded is
       a zipped package
:return: local path where the weights have been downloaded
         and potentially unzipped to


hash_digest = sha256_hash(git_uri)<EOL>local_path = home_directory_path(FOLDER, hash_digest)<EOL>exists_locally = path_exists(local_path)<EOL>if not exists_locally:<EOL><INDENT>_clone_repo(git_uri, local_path)<EOL><DEDENT>else:<EOL><INDENT>logging.info(  <EOL>"<STR_LIT>")  <EOL><DEDENT>return local_path<EOL>

Clone a remote git repository to a local path.

:param git_uri: the URI to the git repository to be cloned
:return: the generated local path where the repository has been cloned to


def get_model(url, module_name='<STR_LIT>', **kwargs):

repo_path = clone(url)<EOL>loader = ModelLoader.get()<EOL>model = loader.load(repo_path, module_name=module_name, **kwargs)<EOL>return model<EOL>

Provides utilities to download foolbox-compatible robust models
to easily test attacks against them by simply providing a git-URL.

Examples
--------

Instantiate a model:

>>> from foolbox import zoo
>>> url = "https://github.com/bveliqi/foolbox-zoo-dummy.git"
>>> model = zoo.get_model(url)  # doctest: +SKIP

Only works with a foolbox-zoo compatible repository.
I.e. models need to have a `foolbox_model.py` file
with a `create()`-function, which returns a foolbox-wrapped model.

Using the kwargs parameter it is possible to input an arbitrary number
of parameters to this methods call. These parameters are forwarded to
the instantiated model.

Example repositories:

    - https://github.com/bethgelab/AnalysisBySynthesis
    - https://github.com/bethgelab/mnist_challenge
    - https://github.com/bethgelab/cifar10_challenge
    - https://github.com/bethgelab/convex_adversarial
    - https://github.com/wielandbrendel/logit-pairing-foolbox.git
    - https://github.com/bethgelab/defensive-distillation.git

:param url: URL to the git repository
:param module_name: the name of the module to import
:param kwargs: Optional set of parameters that will be used by the
    to be instantiated model.
:return: a foolbox-wrapped model instance


@abstractmethod<EOL><INDENT>def load(self, path, module_name='<STR_LIT>', **kwargs):<DEDENT>

pass<EOL>

Load a model from a local path, to which a git repository
has been previously cloned to.

:param path: the path to the local repository containing the code
:param module_name: the name of the module to import
:param kwargs: parameters for the to be loaded model
:return: a foolbox-wrapped model


config = api.get_config()<EOL>workspace = config['<STR_LIT>']['<STR_LIT>']<EOL>default_workspace = os.path.expanduser("<STR_LIT>")<EOL>if workspace != default_workspace and os.path.exists(workspace):<EOL><INDENT>print("<STR_LIT>", workspace)<EOL>shutil.rmtree(workspace)<EOL><DEDENT>


<EOL>result = runner.invoke(dgitmod.profile, ['<STR_LIT>'])<EOL>output = result.output<EOL>output = output.split("<STR_LIT:\n>")<EOL>workspaces = [o.strip() for o in output if "<STR_LIT>" in o]<EOL>if len(workspaces) > <NUM_LIT:0>:<EOL><INDENT>workspace = workspaces[<NUM_LIT:0>]<EOL>workspace = workspace.replace("<STR_LIT>","<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>workspace = os.path.join(os.getcwd(), '<STR_LIT>')<EOL><DEDENT>default_workspace = os.path.expanduser("<STR_LIT>")<EOL>if ((workspace != default_workspace) and<EOL>os.path.exists(workspace)):<EOL><INDENT>print("<STR_LIT>", workspace)<EOL>shutil.rmtree(workspace)<EOL><DEDENT>


datapackage = repo.package<EOL>url = self.url<EOL>token = self.token<EOL>headers = {<EOL>'<STR_LIT>': '<STR_LIT>'.format(token),<EOL>'<STR_LIT:Content-Type>': '<STR_LIT:application/json>'<EOL>}<EOL>try:<EOL><INDENT>r = requests.post(url,<EOL>data = json.dumps(datapackage),<EOL>headers=headers)<EOL>return r<EOL><DEDENT>except Exception as e: <EOL><INDENT>raise NetworkError()<EOL><DEDENT>return "<STR_LIT>"<EOL>

Post to the metadata server

Parameters
----------

repo


<EOL>try:<EOL><INDENT>fh = open(filename, '<STR_LIT:rb>')<EOL>table_set = any_tableset(fh) <EOL><DEDENT>except:<EOL><INDENT>table_set = None<EOL><DEDENT>return table_set<EOL>

Guess the filetype and read the file into row sets


table_set = self.read_file(filename)<EOL>if table_set is None: <EOL><INDENT>return [] <EOL><DEDENT>row_set = table_set.tables[<NUM_LIT:0>]<EOL>offset, headers = headers_guess(row_set.sample)<EOL>row_set.register_processor(headers_processor(headers))<EOL>row_set.register_processor(offset_processor(offset + <NUM_LIT:1>))<EOL>types = type_guess(row_set.sample, strict=True)<EOL>sample = next(row_set.sample)<EOL>clean = lambda v: str(v) if not isinstance(v, str) else v <EOL>schema = []<EOL>for i, h in enumerate(headers):<EOL><INDENT>schema.append([h,<EOL>str(types[i]),<EOL>clean(sample[i].value)])<EOL><DEDENT>return schema<EOL>


status = []<EOL>with cd(repo.rootdir):<EOL><INDENT>files = spec.get('<STR_LIT>', ['<STR_LIT:*>'])<EOL>resource_files = repo.find_matching_files(files)<EOL>files = glob2.glob("<STR_LIT>")<EOL>disk_files = [f for f in files if os.path.isfile(f) and f != "<STR_LIT>"]<EOL>allfiles = list(set(resource_files + disk_files))<EOL>allfiles.sort()<EOL>for f in allfiles:<EOL><INDENT>if f in resource_files and f in disk_files:<EOL><INDENT>r = repo.get_resource(f)<EOL>coded_sha256 = r['<STR_LIT>']<EOL>computed_sha256 = compute_sha256(f)<EOL>if computed_sha256 != coded_sha256:<EOL><INDENT>status.append({<EOL>'<STR_LIT:target>': f,<EOL>'<STR_LIT>': "<STR_LIT>",<EOL>'<STR_LIT>': self.name,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT:message>': "<STR_LIT>"<EOL>})<EOL><DEDENT>else:<EOL><INDENT>status.append({<EOL>'<STR_LIT:target>': f,<EOL>'<STR_LIT>': "<STR_LIT>",<EOL>'<STR_LIT>': self.name,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT:status>': '<STR_LIT:OK>',<EOL>'<STR_LIT:message>': "<STR_LIT>"<EOL>})<EOL><DEDENT><DEDENT>elif f in resource_files:<EOL><INDENT>status.append({<EOL>'<STR_LIT:target>': f,<EOL>'<STR_LIT>': "<STR_LIT>",<EOL>'<STR_LIT>': self.name,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT:message>': "<STR_LIT>"<EOL>})<EOL><DEDENT>else:<EOL><INDENT>status.append({<EOL>'<STR_LIT:target>': f,<EOL>'<STR_LIT>': "<STR_LIT>",<EOL>'<STR_LIT>': self.name,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT:message>': "<STR_LIT>"<EOL>})<EOL><DEDENT><DEDENT><DEDENT>return status<EOL>

Check the integrity of the datapackage.json


status = []<EOL>if len(spec['<STR_LIT>']) == <NUM_LIT:0>: <EOL><INDENT>return status <EOL><DEDENT>with cd(repo.rootdir):<EOL><INDENT>rules = None <EOL>if '<STR_LIT>' in spec and len(spec['<STR_LIT>']) > <NUM_LIT:0>: <EOL><INDENT>rulesfiles = spec['<STR_LIT>']<EOL>rules = {} <EOL>for f in rulesfiles: <EOL><INDENT>d = json.loads(open(f).read())<EOL>rules.update(d)<EOL><DEDENT><DEDENT>elif '<STR_LIT>' in spec: <EOL><INDENT>rules = {<EOL>'<STR_LIT>': spec['<STR_LIT>'] <EOL>}<EOL><DEDENT>if rules is None or len(rules) == <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>raise InvalidParameters("<STR_LIT>")<EOL><DEDENT>files = dict([(f, open(f).read()) for f in spec['<STR_LIT>']])<EOL>for r in rules:<EOL><INDENT>if '<STR_LIT>' not in rules[r]:<EOL><INDENT>continue<EOL><DEDENT>minr2 = float(rules[r]['<STR_LIT>'])<EOL>for f in files:<EOL><INDENT>match = re.search(r"<STR_LIT>", files[f])<EOL>if match is None:<EOL><INDENT>status.append({<EOL>'<STR_LIT:target>': f,<EOL>'<STR_LIT>': self.name,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT>': r,<EOL>'<STR_LIT:status>': "<STR_LIT>",<EOL>'<STR_LIT:message>': "<STR_LIT>"<EOL>})<EOL><DEDENT>else:<EOL><INDENT>r2 = match.group(<NUM_LIT:1>)<EOL>r2 = float(r2)<EOL>if r2 > minr2:<EOL><INDENT>status.append({<EOL>'<STR_LIT:target>': f,<EOL>'<STR_LIT>': self.name,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT>': r,<EOL>'<STR_LIT:status>': "<STR_LIT:OK>",<EOL>'<STR_LIT:message>': "<STR_LIT>"<EOL>})<EOL><DEDENT>else:<EOL><INDENT>status.append({<EOL>'<STR_LIT:target>': f,<EOL>'<STR_LIT>': self.name,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT>': r,<EOL>'<STR_LIT:status>': "<STR_LIT>",<EOL>'<STR_LIT:message>': "<STR_LIT>"<EOL>})<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return status<EOL>

Evaluate the files identified for checksum.


<EOL>environ = os.environ.copy() <EOL>environ['<STR_LIT>'] = self.fullname<EOL>environ['<STR_LIT>'] = self.email <EOL>environ['<STR_LIT>'] = self.fullname<EOL>environ['<STR_LIT>'] = self.email <EOL>cmd = [pipes.quote(c) for c in cmd]<EOL>cmd = "<STR_LIT:U+0020>".join(['<STR_LIT>'] + cmd)<EOL>cmd += "<STR_LIT>"<EOL>try:<EOL><INDENT>output = subprocess.check_output(cmd,<EOL>stderr=subprocess.STDOUT,<EOL>shell=True,<EOL>env=environ)<EOL><DEDENT>except subprocess.CalledProcessError as e:<EOL><INDENT>output = e.output<EOL><DEDENT>output = output.decode('<STR_LIT:utf-8>')<EOL>output = output.strip()<EOL>return output<EOL>

Helper function to run commands

Parameters
----------
cmd : list
      Arguments to git command


def _run_generic_command(self, repo, cmd):

result = None<EOL>with cd(repo.rootdir):<EOL><INDENT>output = self._run(cmd)<EOL>try:<EOL><INDENT>result = {<EOL>'<STR_LIT>': cmd,<EOL>'<STR_LIT:status>': '<STR_LIT:success>',<EOL>'<STR_LIT:message>': output,<EOL>}<EOL><DEDENT>except Exception as e:<EOL><INDENT>result = {<EOL>'<STR_LIT>': cmd,<EOL>'<STR_LIT:status>': '<STR_LIT:error>',<EOL>'<STR_LIT:message>': str(e)<EOL>}<EOL><DEDENT><DEDENT>return result<EOL>

Run a generic command within the repo. Assumes that you are
in the repo's root directory


return self._run_generic_command(repo, ["<STR_LIT>"] + args)<EOL>

Add notes to the commit

Parameters
----------

repo: Repository object
args: notes-specific args


Push to origin master

Parameters
----------

repo: Repository object
args: git-specific args


Check remote URL

Parameters
----------

repo: Repository object
args: git-specific args


Pull from origin/filesystem based master

Parameters
----------

repo: Repository object
args: git-specific args


return self._run_generic_command(repo, ["<STR_LIT:status>"] + args)<EOL>

Show status of the repo (pass thru git command)

Parameters
----------

repo: Repository object
args: git-specific args


Stash all the changes (pass thru git command)

Parameters
----------

repo: Repository object
args: git-specific args


diff two repo versions  (pass thru git command)

Parameters
----------

repo: Repository object
args: git-specific args


Show the log  (pass thru git command)

Parameters
----------

repo: Repository object
args: git-specific args


Commit the changes to the repo (pass thru git command)

Parameters
----------

repo: Repository object
args: git-specific args


Show the content of the repo (pass thru git command)

Parameters
----------

repo: Repository object
args: git-specific args


def init(self, username, reponame, force, backend=None):

key = self.key(username, reponame)<EOL>server_repodir = self.server_rootdir(username,<EOL>reponame,<EOL>create=False)<EOL>if os.path.exists(server_repodir) and not force:<EOL><INDENT>raise RepositoryExists()<EOL><DEDENT>if os.path.exists(server_repodir):<EOL><INDENT>shutil.rmtree(server_repodir)<EOL><DEDENT>os.makedirs(server_repodir)<EOL>with cd(server_repodir):<EOL><INDENT>git.init("<STR_LIT:.>", "<STR_LIT>")<EOL><DEDENT>if backend is not None:<EOL><INDENT>backend.init_repo(server_repodir)<EOL><DEDENT>repodir = self.rootdir(username, reponame, create=False)<EOL>if os.path.exists(repodir) and not force:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if os.path.exists(repodir):<EOL><INDENT>shutil.rmtree(repodir)<EOL><DEDENT>os.makedirs(repodir)<EOL>with cd(os.path.dirname(repodir)):<EOL><INDENT>git.clone(server_repodir, '<STR_LIT>')<EOL><DEDENT>url = server_repodir<EOL>if backend is not None:<EOL><INDENT>url = backend.url(username, reponame)<EOL><DEDENT>repo = Repo(username, reponame)<EOL>repo.manager = self<EOL>repo.remoteurl = url<EOL>repo.rootdir = self.rootdir(username, reponame)<EOL>self.add(repo)<EOL>return repo<EOL>

Initialize a Git repo

Parameters
----------

username, reponame : Repo name is tuple (name, reponame)
force: force initialization of the repo even if exists
backend: backend that must be used for this (e.g. s3)


<EOL>username = self.username<EOL>reponame = url.split("<STR_LIT:/>")[-<NUM_LIT:1>] <EOL>reponame = reponame.replace("<STR_LIT>","<STR_LIT>")<EOL>key = (username, reponame)<EOL>server_repodir = self.server_rootdir(username,<EOL>reponame,<EOL>create=False)<EOL>rootdir = self.rootdir(username,  reponame, create=False)<EOL>if backend is None:<EOL><INDENT>with cd(os.path.dirname(rootdir)):<EOL><INDENT>self._run(['<STR_LIT>', '<STR_LIT>', url])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not os.path.exists(server_repodir):<EOL><INDENT>backend.clone_repo(url, server_repodir)<EOL><DEDENT>with cd(os.path.dirname(rootdir)):<EOL><INDENT>self._run(['<STR_LIT>', '<STR_LIT>', server_repodir])<EOL><DEDENT><DEDENT>if True:<EOL><INDENT>configfile = os.path.join(rootdir, '<STR_LIT>', '<STR_LIT>')<EOL>content = open(configfile).read()<EOL>original = "<STR_LIT>"<EOL>replacement ="""<STR_LIT>"""<EOL>if "<STR_LIT>" not in content:<EOL><INDENT>content = content.replace(original, replacement)<EOL>with open(configfile, '<STR_LIT:w>') as fd:<EOL><INDENT>fd.write(content)<EOL><DEDENT><DEDENT>with cd(rootdir):<EOL><INDENT>self._run(['<STR_LIT>','<STR_LIT>'])<EOL><DEDENT><DEDENT>r = Repo(username, reponame)<EOL>r.rootdir = rootdir<EOL>r.remoteurl = url<EOL>r.manager = self<EOL>package = os.path.join(r.rootdir, '<STR_LIT>')<EOL>packagedata = open(package).read()<EOL>r.package = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(packagedata)<EOL>return self.add(r)<EOL>

Clone a URL

Parameters
----------

url : URL of the repo. Supports s3://, git@, http://


result = None<EOL>with cd(repo.rootdir):<EOL><INDENT>try:<EOL><INDENT>cmd = ['<STR_LIT>'] + list(args)<EOL>result = {<EOL>'<STR_LIT:status>': '<STR_LIT:success>',<EOL>'<STR_LIT:message>': self._run(cmd)<EOL>}<EOL><DEDENT>except Exception as e:<EOL><INDENT>result = {<EOL>'<STR_LIT:status>': '<STR_LIT:error>',<EOL>'<STR_LIT:message>': str(e)<EOL>}<EOL><DEDENT>return result<EOL><DEDENT>


<EOL>rootdir = repo.rootdir<EOL>if os.path.exists(rootdir):<EOL><INDENT>print("<STR_LIT>".format(rootdir))<EOL>shutil.rmtree(rootdir)<EOL><DEDENT>server_repodir = self.server_rootdir_from_repo(repo,<EOL>create=False)<EOL>if os.path.exists(server_repodir):<EOL><INDENT>print("<STR_LIT>".format(server_repodir))<EOL>shutil.rmtree(server_repodir)<EOL><DEDENT>super(GitRepoManager, self).drop(repo)<EOL>return {<EOL>'<STR_LIT:status>': '<STR_LIT:success>',<EOL>'<STR_LIT:message>': "<STR_LIT>"<EOL>}<EOL>


if not os.path.exists(path): <EOL><INDENT>return (None, None) <EOL><DEDENT>cwd = os.getcwd()<EOL>if os.path.isfile(path):<EOL><INDENT>os.chdir(os.path.dirname(path))<EOL><DEDENT>rootdir = self._run(["<STR_LIT>", "<STR_LIT>"])<EOL>if "<STR_LIT>" in rootdir:<EOL><INDENT>return (None, None) <EOL><DEDENT>os.chdir(rootdir)<EOL>relpath = os.path.relpath(path, rootdir)<EOL>sha1 = self._run(["<STR_LIT>", "<STR_LIT>", "<STR_LIT:1>", "<STR_LIT>", relpath])<EOL>remoteurl = self._run(["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"])<EOL>os.chdir(cwd)<EOL>m = re.search('<STR_LIT>', remoteurl)<EOL>if m is None:<EOL><INDENT>m = re.search('<STR_LIT>', remoteurl)<EOL><DEDENT>if m is not None:<EOL><INDENT>domain = m.group(<NUM_LIT:1>)<EOL>username = m.group(<NUM_LIT:2>)<EOL>project = m.group(<NUM_LIT:3>)<EOL>if project.endswith("<STR_LIT>"):<EOL><INDENT>project = project[:-<NUM_LIT:4>]<EOL><DEDENT>permalink = "<STR_LIT>".format(domain, username, project,<EOL>sha1, relpath)<EOL>return (relpath, permalink)<EOL><DEDENT>else:<EOL><INDENT>return (None, None)<EOL><DEDENT>

Get the permalink to command that generated the dataset


rootdir = repo.rootdir<EOL>for f in files:<EOL><INDENT>relativepath = f['<STR_LIT>']<EOL>sourcepath = f['<STR_LIT>']<EOL>if sourcepath is None:<EOL><INDENT>continue <EOL><DEDENT>targetpath = os.path.join(rootdir, relativepath)<EOL>try:<EOL><INDENT>os.makedirs(os.path.dirname(targetpath))<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>print("<STR_LIT>".format(relativepath))<EOL>shutil.copyfile(sourcepath, targetpath)<EOL>with cd(repo.rootdir):<EOL><INDENT>self._run(['<STR_LIT>', relativepath])<EOL><DEDENT><DEDENT>


def config(self, what='<STR_LIT>', params=None):

if what == '<STR_LIT>':<EOL><INDENT>return {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': [],<EOL>}<EOL><DEDENT>elif what == '<STR_LIT>':<EOL><INDENT>self.workspace = params['<STR_LIT>']['<STR_LIT>']<EOL>self.workspace = os.path.abspath(self.workspace)<EOL>self.username = params['<STR_LIT>']['<STR_LIT>']<EOL>self.fullname = params['<STR_LIT>']['<STR_LIT>']<EOL>self.email = params['<STR_LIT>']['<STR_LIT>']<EOL>repodir = os.path.join(self.workspace, '<STR_LIT>')<EOL>if not os.path.exists(repodir):<EOL><INDENT>return<EOL><DEDENT>for username in os.listdir(repodir):<EOL><INDENT>for reponame in os.listdir(os.path.join(repodir, username)):<EOL><INDENT>if self.is_my_repo(username, reponame):<EOL><INDENT>r = Repo(username, reponame)<EOL>r.rootdir = os.path.join(repodir, username, reponame)<EOL>package = os.path.join(r.rootdir, '<STR_LIT>')<EOL>if not os.path.exists(package):<EOL><INDENT>print("<STR_LIT>")<EOL>print("<STR_LIT>".format(username, reponame))<EOL>continue<EOL><DEDENT>packagedata = open(package).read()<EOL>r.package = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(packagedata)<EOL>r.manager = self<EOL>self.add(r)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Paramers:
---------

workspace: Directory to store the dataset repositories
email:


hooksdir = os.path.join(gitdir, '<STR_LIT>')<EOL>content = postreceive_template % {<EOL>'<STR_LIT>': self.client,<EOL>'<STR_LIT>': self.bucket,<EOL>'<STR_LIT>': self.s3cfg,<EOL>'<STR_LIT>': self.prefix<EOL>}<EOL>postrecv_filename =os.path.join(hooksdir, '<STR_LIT>')<EOL>with open(postrecv_filename,'<STR_LIT:w>') as fd:<EOL><INDENT>fd.write(content)<EOL><DEDENT>self.make_hook_executable(postrecv_filename)<EOL>print("<STR_LIT>", postrecv_filename)<EOL>


<EOL>if url.startswith("<STR_LIT>"):<EOL><INDENT>url = url.replace("<STR_LIT>","<STR_LIT>")<EOL><DEDENT>return os.path.exists(url)<EOL>


global config<EOL>profileini = getprofileini()<EOL>if os.path.exists(profileini):<EOL><INDENT>config = configparser.ConfigParser()<EOL>config.read(profileini)<EOL>mgr = plugins_get_mgr()<EOL>mgr.update_configs(config)<EOL>if show:<EOL><INDENT>for source in config:<EOL><INDENT>print("<STR_LIT>" %(source))<EOL>for k in config[source]:<EOL><INDENT>print("<STR_LIT>" % (k, config[source][k]))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>if not show:<EOL><INDENT>update(globalvars)<EOL><DEDENT><DEDENT>print("<STR_LIT>")<EOL>


global config<EOL>profileini = getprofileini()<EOL>config = configparser.ConfigParser()<EOL>config.read(profileini)<EOL>defaults = {}<EOL>if globalvars is not None:<EOL><INDENT>defaults = {a[<NUM_LIT:0>]: a[<NUM_LIT:1>] for a in globalvars }<EOL><DEDENT>generic_configs = [{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:description>': "<STR_LIT>",<EOL>'<STR_LIT>': ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>'],<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:value>': defaults.get('<STR_LIT>','<STR_LIT>'),<EOL>'<STR_LIT:description>': "<STR_LIT>",<EOL>'<STR_LIT>': EmailValidator()<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:value>': defaults.get('<STR_LIT>','<STR_LIT>'),<EOL>'<STR_LIT:description>': "<STR_LIT>",<EOL>'<STR_LIT>': NonEmptyValidator()<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:value>': defaults.get('<STR_LIT>', getpass.getuser()),<EOL>'<STR_LIT:description>': "<STR_LIT:Name>",<EOL>'<STR_LIT>': NonEmptyValidator()<EOL>},<EOL>}<EOL>}]<EOL>mgr = plugins_get_mgr()<EOL>extra_configs = mgr.gather_configs()<EOL>allconfigs = generic_configs + extra_configs<EOL>for c in allconfigs:<EOL><INDENT>name = c['<STR_LIT:name>']<EOL>for v in c['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>c['<STR_LIT>'][v]['<STR_LIT:value>'] = config[name][v]<EOL><DEDENT>except:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT>for c in allconfigs:<EOL><INDENT>print("<STR_LIT>")<EOL>print(c['<STR_LIT:description>'])<EOL>print("<STR_LIT>")<EOL>if len(c['<STR_LIT>']) == <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>continue<EOL><DEDENT>name = c['<STR_LIT:name>']<EOL>config[name] = {}<EOL>config[name]['<STR_LIT>'] = c['<STR_LIT>']<EOL>for v in c['<STR_LIT>']:<EOL><INDENT>value = '<STR_LIT>'<EOL>description = v + "<STR_LIT:U+0020>"<EOL>helptext = "<STR_LIT>"<EOL>validator = None<EOL>if v in c['<STR_LIT>']:<EOL><INDENT>value = c['<STR_LIT>'][v].get('<STR_LIT:value>','<STR_LIT>')<EOL>helptext = c['<STR_LIT>'][v].get("<STR_LIT:description>","<STR_LIT>")<EOL>validator = c['<STR_LIT>'][v].get('<STR_LIT>',None)<EOL><DEDENT>if helptext != "<STR_LIT>":<EOL><INDENT>description += "<STR_LIT:(>" + helptext + "<STR_LIT:)>"<EOL><DEDENT>while True:<EOL><INDENT>choice = input_with_default(description, value)<EOL>if validator is not None:<EOL><INDENT>if validator.is_valid(choice):<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>".format(validator.message))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>config[name][v] = choice<EOL>if v == '<STR_LIT>' and choice == '<STR_LIT:n>': <EOL><INDENT>break <EOL><DEDENT><DEDENT><DEDENT>with open(profileini, '<STR_LIT:w>') as fd:<EOL><INDENT>config.write(fd)<EOL><DEDENT>print("<STR_LIT>", config)<EOL>


def __init__(self, name, version, description, supported=[]):

self.enable = '<STR_LIT:y>'<EOL>self.name = name<EOL>self.version = version<EOL>self.description = description<EOL>self.support = supported + [name]<EOL>self.initialize()<EOL>

Parameters:
-----------
name: Name of the service e.g., s3
version: Version of this implementation
description: Text description of this service
supported: supported services with including name

For example, there may be multiple s3 implementations that
support different kinds of services.


return<EOL>

Called to initialize sessions, internal objects etc.


return None<EOL>


def evaluate(self, repo, spec, force, args):

return []<EOL>


Parameters:
-----------
name: Name of the backend service e.g., s3
version: Version of this implementation
description: Text description of this service
supported: supported services with including name

For example, there may be multiple s3 implementations that
support different kinds of services.


return<EOL>


return []<EOL>

Evaluate the repo

returns: A list of dictionaries with:
   target: relative path of the file
   rules: rules file used
   validator: name of the validator
   status: OK/Success/Error
   Message: Any additional information


if len(includes) == <NUM_LIT:0>: <EOL><INDENT>return [] <EOL><DEDENT>files = [f['<STR_LIT>'] for f in self.package['<STR_LIT>']]<EOL>includes = r'<STR_LIT:|>'.join([fnmatch.translate(x) for x in includes])<EOL>files = [f for f in files if re.match(includes, os.path.basename(f))] +[f for f in files if re.match(includes, f)]<EOL>files = list(set(files))<EOL>return files<EOL>

For various actions we need files that match patterns


if self.manager is None:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if cmd not in dir(self.manager):<EOL><INDENT>raise Exception("<STR_LIT>".format(cmd))<EOL><DEDENT>func = getattr(self.manager, cmd)<EOL>repo = self<EOL>return func(repo, *args)<EOL>

Run a specific command using the manager


for r in self.package['<STR_LIT>']:<EOL><INDENT>if r['<STR_LIT>'] == p:<EOL><INDENT>r['<STR_LIT>'] = os.path.join(self.rootdir, p)<EOL>return r<EOL><DEDENT><DEDENT>raise Exception("<STR_LIT>")<EOL>


def init(self, username, reponame, force):

pass<EOL>

Initialize a repo (may be fs/git/.. backed)


def lookup(self, username=None, reponame=None, key=None):

if key is None:<EOL><INDENT>key = self.key(username, reponame)<EOL><DEDENT>if key not in self.repos:<EOL><INDENT>raise UnknownRepository()<EOL><DEDENT>return self.repos[key]<EOL>


return os.listdir(os.path.join(self.workspace, '<STR_LIT>'))<EOL>


def server_rootdir(self,  username, reponame, create=True):

path = os.path.join(self.workspace,<EOL>'<STR_LIT>',<EOL>username,<EOL>reponame + "<STR_LIT>")<EOL>if create:<EOL><INDENT>try:<EOL><INDENT>os.makedirs(path)<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return path<EOL>


def rootdir(self,  username, reponame, create=True):

path = os.path.join(self.workspace,<EOL>'<STR_LIT>',<EOL>username,<EOL>reponame)<EOL>if create:<EOL><INDENT>try:<EOL><INDENT>os.makedirs(path)<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return path<EOL>


key = self.key(repo.username, repo.reponame)<EOL>repo.key = key<EOL>self.repos[key] = repo<EOL>return key<EOL>

Add repo to the internal lookup table...


key = repo.key<EOL>del self.repos[key]<EOL>


pass<EOL>

Files is a list with simple dict structure with relativepath and fullpath


def clone(self, repo, newusername, newreponame):

pass<EOL>


global pluginmgr<EOL>if pluginmgr is not None:<EOL><INDENT>plugins_close()<EOL><DEDENT>pluginmgr = PluginManager([])<EOL>

Load plugins from various sources:

- dgit/plugins
- dgit_extensions package


def plugins_show(what=None, name=None, version=None, details=False):

global pluginmgr<EOL>return pluginmgr.show(what, name, version, details)<EOL>

Show details of available plugins

Parameters
----------
what: Class of plugins e.g., backend
name: Name of the plugin e.g., s3
version: Version of the plugin
details: Show details be shown?


global pluginmgr<EOL>return pluginmgr<EOL>


for v in pkg_resources.iter_entry_points('<STR_LIT>'):<EOL><INDENT>m = v.load()<EOL>m.setup(self)<EOL><DEDENT>


<EOL>name = obj.name<EOL>version = obj.version<EOL>enable = obj.enable<EOL>if enable == '<STR_LIT:n>':<EOL><INDENT>return<EOL><DEDENT>key = Key(name, version)<EOL>self.plugins[what][key] = obj<EOL>

Registering a plugin

Params
------
what: Nature of the plugin (backend, instrumentation, repo)
obj: Instance of the plugin


def search(self, what, name=None, version=None):

filtered = {}<EOL>if what is None:<EOL><INDENT>whats = list(self.plugins.keys())<EOL><DEDENT>elif what is not None:<EOL><INDENT>if what not in self.plugins:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>whats = [what]<EOL><DEDENT>for what in whats:<EOL><INDENT>if what not in filtered:<EOL><INDENT>filtered[what] = []<EOL><DEDENT>for key in self.plugins[what].keys():<EOL><INDENT>(k_name, k_version) = key<EOL>if name is not None and k_name != name:<EOL><INDENT>continue<EOL><DEDENT>if version is not None and k_version != version:<EOL><INDENT>continue<EOL><DEDENT>if self.plugins[what][key].enable == '<STR_LIT:n>':<EOL><INDENT>continue<EOL><DEDENT>filtered[what].append(key)<EOL><DEDENT><DEDENT>return filtered<EOL>


configs = []<EOL>for what in self.order:<EOL><INDENT>for key in self.plugins[what]:<EOL><INDENT>mgr = self.plugins[what][key]<EOL>c = mgr.config(what='<STR_LIT>')<EOL>if c is not None:<EOL><INDENT>c.update({<EOL>'<STR_LIT:description>': mgr.description<EOL>})<EOL>configs.append(c)<EOL><DEDENT><DEDENT><DEDENT>return configs<EOL>

Gather configuration requirements of all plugins


for what in self.plugins:  <EOL><INDENT>for key in self.plugins[what]: <EOL><INDENT>self.plugins[what][key].config(what='<STR_LIT>', params=config)<EOL><DEDENT><DEDENT>return<EOL>


self.enable = '<STR_LIT:y>'<EOL>self.name = name<EOL>self.version = version<EOL>self.description = description<EOL>self.support = supported + [name]<EOL>self.initialize()<EOL>


return<EOL>


pass<EOL>


return False<EOL>


return<EOL>

Push a data version to the server

Parameters
----------

state: Overall state object that has dataset details
name: name of the dataset


if path is None: path = []<EOL>for key in b:<EOL><INDENT>if key in a:<EOL><INDENT>if isinstance(a[key], dict) andisinstance(b[key], dict):<EOL><INDENT>merge(a[key], b[key], path + [str(key)])<EOL><DEDENT>if isinstance(a[key], list) andisinstance(b[key], list):<EOL><INDENT>a[key].extend(b[key])<EOL><DEDENT>elif a[key] == b[key]:<EOL><INDENT>pass <EOL><DEDENT>elif type(a[key]) == type(b[key]):<EOL><INDENT>a[key] = b[key]<EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>' % '<STR_LIT:.>'.join(path + [str(key)]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>a[key] = b[key]<EOL><DEDENT><DEDENT>return a<EOL>


try:<EOL><INDENT>h = sha256()<EOL>fd = open(filename, '<STR_LIT:rb>')<EOL>while True:<EOL><INDENT>buf = fd.read(<NUM_LIT>)<EOL>if buf in [None, "<STR_LIT>"]:<EOL><INDENT>break<EOL><DEDENT>h.update(buf.encode('<STR_LIT:utf-8>'))<EOL><DEDENT>fd.close()<EOL>return h.hexdigest()<EOL><DEDENT>except:<EOL><INDENT>output = run(["<STR_LIT>", "<STR_LIT>", filename])<EOL>return output.split("<STR_LIT:U+0020>")[<NUM_LIT:0>]<EOL><DEDENT>

Try the library. If it doesnt work, use the command line..


cmd = [pipes.quote(c) for c in cmd]<EOL>cmd = "<STR_LIT:U+0020>".join(cmd)<EOL>cmd += "<STR_LIT>"<EOL>try:<EOL><INDENT>output = subprocess.check_output(cmd,<EOL>stderr=subprocess.STDOUT,<EOL>shell=True)<EOL><DEDENT>except subprocess.CalledProcessError as e:<EOL><INDENT>output = e.output<EOL><DEDENT>output = output.decode('<STR_LIT:utf-8>')<EOL>output = output.strip()<EOL>return output<EOL>


value = unicodedata.normalize('<STR_LIT>', value)<EOL>value = value.encode('<STR_LIT:ascii>', '<STR_LIT:ignore>')<EOL>value = value.decode('<STR_LIT:utf-8>')<EOL>value = re.sub(r'<STR_LIT>', '<STR_LIT:->', value).strip().lower()<EOL>return re.sub(r'<STR_LIT>', '<STR_LIT:->', value)<EOL>

Normalizes string, converts to lowercase, removes non-alpha characters,
and converts spaces to hyphens.


def _inner(*args, **kwargs):<EOL><INDENT>result = func(*args, **kwargs)         <EOL>log_action(func, result, *args, **kwargs)<EOL>return result <EOL><DEDENT>_inner.__name__ = func.__name__<EOL>_inner.__doc__ = func.__doc__<EOL>return _inner<EOL>


if module is None:<EOL><INDENT>frm = sys._getframe((stacklevel or <NUM_LIT:0>) + <NUM_LIT:1>)<EOL>name = frm.f_globals['<STR_LIT>']<EOL>glob = frm.f_globals<EOL><DEDENT>elif isinstance(module, string_types):<EOL><INDENT>frm = sys._getframe(<NUM_LIT:1>)<EOL>name = module<EOL>glob = __import__(module, frm.f_globals,<EOL>frm.f_locals, ['<STR_LIT>']).__dict__<EOL><DEDENT>else:<EOL><INDENT>name = module.__name__<EOL>glob = module.__dict__<EOL><DEDENT>return _discover_space(name, glob)<EOL>

Returns the :class:`PluginSource` for the current module or the given
    module.  The module can be provided by name (in which case an import
    will be attempted) or as a module object.

    If no plugin source can be discovered, the return value from this method
    is `None`.

    This function can be very useful if additional data has been attached
    to the plugin source.  For instance this could allow plugins to get
    access to a back reference to the application that created them.

    :param module: optionally the module to locate the plugin source of.
    :param stacklevel: defines how many levels up the module should search
                       for before it discovers the plugin frame.  The
                       default is 0.  This can be useful for writing wrappers
                       around this function.


def make_plugin_source(self, *args, **kwargs):

return PluginSource(self, *args, **kwargs)<EOL>

Creats a plugin source for this plugin base and returns it.
        All parameters are forwarded to :class:`PluginSource`.


rv = []<EOL>for _, modname, ispkg in pkgutil.iter_modules(self.mod.__path__):<EOL><INDENT>rv.append(modname)<EOL><DEDENT>return sorted(rv)<EOL>

Returns a sorted list of all plugins that are available in this
        plugin source.  This can be useful to automatically discover plugins
        that are available and is usually used together with
        :meth:`load_plugin`.


if '<STR_LIT:.>' in name:<EOL><INDENT>raise ImportError('<STR_LIT>')<EOL><DEDENT>with self:<EOL><INDENT>return __import__(self.base.package + '<STR_LIT:.>' + name,<EOL>globals(), {}, ['<STR_LIT>'])<EOL><DEDENT>

This automatically loads a plugin by the given name from the
        current source and returns the module.  This is a convenient
        alternative to the import statement and saves you from invoking
        ``__import__`` or a similar function yourself.

        :param name: the name of the plugin to load.


def open_resource(self, plugin, filename):

mod = self.load_plugin(plugin)<EOL>fn = getattr(mod, '<STR_LIT>', None)<EOL>if fn is not None:<EOL><INDENT>if fn.endswith(('<STR_LIT>', '<STR_LIT>')):<EOL><INDENT>fn = fn[:-<NUM_LIT:1>]<EOL><DEDENT>if os.path.isfile(fn):<EOL><INDENT>return open(os.path.join(os.path.dirname(fn), filename), '<STR_LIT:rb>')<EOL><DEDENT><DEDENT>buf = pkgutil.get_data(self.mod.__name__ + '<STR_LIT:.>' + plugin, filename)<EOL>if buf is None:<EOL><INDENT>raise IOError(errno.ENOEXITS, '<STR_LIT>')<EOL><DEDENT>return NativeBytesIO(buf)<EOL>

This function locates a resource inside the plugin and returns
        a byte stream to the contents of it.  If the resource cannot be
        loaded an :exc:`IOError` will be raised.  Only plugins that are
        real Python packages can contain resources.  Plain old Python
        modules do not allow this for obvious reasons.

        .. versionadded:: 0.3

        :param plugin: the name of the plugin to open the resource of.
        :param filename: the name of the file within the plugin to open.


Cleans up all loaded plugins manually.  This is necessary to
        call only if :attr:`persist` is enabled.  Otherwise this happens
        automatically when the source gets garbage collected.


self.enabled = True<EOL>

Enables the import hook which drives the plugin base system.
        This is the default.


self.enabled = False<EOL>

Disables the import hook and restores the default import system
        behavior.  This effectively breaks pluginbase but can be useful
        for testing purposes.


def instantiate(repo, name=None, filename=None):

default_transformers = repo.options.get('<STR_LIT>', {})<EOL>transformers = {}<EOL>if name is not None:<EOL><INDENT>if name in default_transformers:<EOL><INDENT>transformers = {<EOL>name : default_transformers[name]<EOL>}<EOL><DEDENT>else:<EOL><INDENT>transformers = {<EOL>name : {<EOL>'<STR_LIT>': [],<EOL>}<EOL>}<EOL><DEDENT><DEDENT>else:<EOL><INDENT>transformers = default_transformers<EOL><DEDENT>input_matching_files = None<EOL>if filename is not None:<EOL><INDENT>input_matching_files = repo.find_matching_files([filename])<EOL><DEDENT>for t in transformers:<EOL><INDENT>for k in transformers[t]:<EOL><INDENT>if "<STR_LIT>" not in k:<EOL><INDENT>continue<EOL><DEDENT>if k == "<STR_LIT>" and input_matching_files is not None:<EOL><INDENT>transformers[t][k] = input_matching_files<EOL><DEDENT>else:<EOL><INDENT>if transformers[t][k] is None or len(transformers[t][k]) == <NUM_LIT:0>:<EOL><INDENT>transformers[t][k] = []<EOL><DEDENT>else:<EOL><INDENT>matching_files = repo.find_matching_files(transformers[t][k])<EOL>transformers[t][k] = matching_files<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return transformers<EOL>

Instantiate the generator and filename specification


mgr = plugins_get_mgr()<EOL>specs = instantiate(repo, name, filename)<EOL>allresults = []<EOL>for s in specs:<EOL><INDENT>keys = mgr.search(what='<STR_LIT>',name=s)['<STR_LIT>']<EOL>for k in keys:<EOL><INDENT>t = mgr.get_by_key('<STR_LIT>', k)<EOL>result = t.evaluate(repo,<EOL>specs[s],<EOL>force,<EOL>args)<EOL>allresults.extend(result)<EOL><DEDENT><DEDENT>return allresults<EOL>

Materialize queries/other content within the repo.

Parameters
----------

repo: Repository object
name: Name of transformer, if any. If none, then all transformers specified in dgit.json will be included.
filename: Pattern that specifies files that must be processed by the generators selected. If none, then the default specification in dgit.json is used.


def add_file_normal(f, targetdir, generator,script, source):

basename = os.path.basename(f)<EOL>if targetdir != "<STR_LIT:.>":<EOL><INDENT>relativepath = os.path.join(targetdir, basename)<EOL><DEDENT>else:<EOL><INDENT>relativepath = basename<EOL><DEDENT>relpath = os.path.relpath(f, os.getcwd())<EOL>filetype = '<STR_LIT:data>'<EOL>if script:<EOL><INDENT>filetype = '<STR_LIT>'<EOL>if generator:<EOL><INDENT>filetype = '<STR_LIT>'<EOL><DEDENT><DEDENT>update = OrderedDict([<EOL>('<STR_LIT:type>', filetype),<EOL>('<STR_LIT>', generator),<EOL>('<STR_LIT>', relativepath),<EOL>('<STR_LIT:content>', "<STR_LIT>"),<EOL>('<STR_LIT:source>', source),<EOL>('<STR_LIT>', f),<EOL>('<STR_LIT>', relpath)<EOL>])<EOL>update = annotate_record(update)<EOL>return (basename, update)<EOL>


<EOL>lines = open(filename).readlines()<EOL>files = {}<EOL>lines = [l.strip() for l in lines if '<STR_LIT>' in l]<EOL>for l in lines:<EOL><INDENT>matchedfile = re.search('<STR_LIT>', l)<EOL>if matchedfile is None:<EOL><INDENT>matchedfile = re.search('<STR_LIT>', l)<EOL><DEDENT>if matchedfile is None:<EOL><INDENT>continue<EOL><DEDENT>matchedfile = matchedfile.group(<NUM_LIT:1>)<EOL>if os.path.exists(matchedfile) and os.path.isfile(matchedfile):<EOL><INDENT>action = '<STR_LIT:input>' if '<STR_LIT>' in l else '<STR_LIT>'<EOL>matchedfile = os.path.relpath(matchedfile, "<STR_LIT:.>")<EOL>for i in includes:<EOL><INDENT>if fnmatch.fnmatch(matchedfile, i):<EOL><INDENT>if '<STR_LIT>' in matchedfile:<EOL><INDENT>continue<EOL><DEDENT>if matchedfile not in files:<EOL><INDENT>files[matchedfile] = [action]<EOL><DEDENT>else:<EOL><INDENT>if action not in files[matchedfile]:<EOL><INDENT>files[matchedfile].append(action)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if len(files) == <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>return []<EOL><DEDENT>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>filenames = list(files.keys())<EOL>filenames.sort()<EOL>with tempfile.NamedTemporaryFile(suffix="<STR_LIT>") as temp:<EOL><INDENT>temp.write(yaml.dump(filenames, default_flow_style=False).encode('<STR_LIT:utf-8>'))<EOL>temp.flush()<EOL>EDITOR = os.environ.get('<STR_LIT>','<STR_LIT>')<EOL>subprocess.call("<STR_LIT>" %(EDITOR,temp.name), shell=True)<EOL>temp.seek(<NUM_LIT:0>)<EOL>data = temp.read()<EOL>selected = yaml.load(data)<EOL><DEDENT>print("<STR_LIT>", len(selected), "<STR_LIT>")<EOL>if len(selected) == <NUM_LIT:0>:<EOL><INDENT>return []<EOL><DEDENT>filenames = [f for f in filenames if f in selected]<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>input('<STR_LIT>')<EOL>prefixes = {}<EOL>for f in filenames:<EOL><INDENT>dirname = os.path.dirname(f)<EOL>if dirname == "<STR_LIT>":<EOL><INDENT>dirname = "<STR_LIT:.>"<EOL><DEDENT>prefixes[dirname] = dirname<EOL><DEDENT>while True:<EOL><INDENT>with tempfile.NamedTemporaryFile(suffix="<STR_LIT>") as temp:<EOL><INDENT>temp.write(yaml.dump(prefixes, default_flow_style=False).encode('<STR_LIT:utf-8>'))<EOL>temp.flush()<EOL>EDITOR = os.environ.get('<STR_LIT>','<STR_LIT>')<EOL>subprocess.call("<STR_LIT>" %(EDITOR,temp.name), shell=True)<EOL>temp.seek(<NUM_LIT:0>)<EOL>data = temp.read()<EOL>try:<EOL><INDENT>revised = yaml.load(data)<EOL><DEDENT>except Exception as e:<EOL><INDENT>revised = {}<EOL><DEDENT>if set(list(revised.keys())) == set(list(prefixes.keys())):<EOL><INDENT>prefixes = revised<EOL>break<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>input("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>if "<STR_LIT:.>" in prefixes:<EOL><INDENT>prefixes["<STR_LIT>"] = prefixes["<STR_LIT:.>"]<EOL><DEDENT>result = []<EOL>ts = datetime.now().isoformat()<EOL>for f in filenames:<EOL><INDENT>relativepath = prefixes[os.path.dirname(f)]<EOL>if relativepath == "<STR_LIT:.>":<EOL><INDENT>relativepath = os.path.basename(f)<EOL><DEDENT>else:<EOL><INDENT>relativepath = os.path.join(relativepath, os.path.basename(f))<EOL><DEDENT>result.append(OrderedDict([<EOL>('<STR_LIT>', relativepath),<EOL>('<STR_LIT:type>', '<STR_LIT>'),<EOL>('<STR_LIT>', files[f]),<EOL>('<STR_LIT>', mimetypes.guess_type(f)[<NUM_LIT:0>]),<EOL>('<STR_LIT:content>', open(f).read(<NUM_LIT>)),<EOL>('<STR_LIT>', compute_sha256(f)),<EOL>('<STR_LIT>', ts),<EOL>('<STR_LIT>', os.path.relpath(f, "<STR_LIT:.>")),<EOL>('<STR_LIT>', os.path.abspath(f)),<EOL>]))<EOL><DEDENT>print(json.dumps(result, indent=<NUM_LIT:4>))<EOL>return result<EOL>

Extract the files to be added based on the includes


def run_executable(repo, args, includes):

<EOL>mgr = plugins_get_mgr()<EOL>repomgr = mgr.get(what='<STR_LIT>', name='<STR_LIT>')<EOL>platform_metadata = repomgr.get_metadata()<EOL>print("<STR_LIT>")<EOL>(executable, commiturl) =find_executable_commitpath(repo, args)<EOL>tmpdir = tempfile.mkdtemp()<EOL>print("<STR_LIT>")<EOL>strace_filename = os.path.join(tmpdir,'<STR_LIT>')<EOL>cmd = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", strace_filename,<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"] + args<EOL>p = subprocess.Popen(cmd,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>out, err = p.communicate()<EOL>stdout = os.path.join(tmpdir, '<STR_LIT>')<EOL>with open(stdout, '<STR_LIT:w>') as fd:<EOL><INDENT>fd.write(out.decode('<STR_LIT:utf-8>'))<EOL><DEDENT>stderr = os.path.join(tmpdir, '<STR_LIT>')<EOL>with open(stderr, '<STR_LIT:w>') as fd:<EOL><INDENT>fd.write(err.decode('<STR_LIT:utf-8>'))<EOL><DEDENT>files = extract_files(strace_filename, includes)<EOL>execution_metadata = {<EOL>'<STR_LIT>': executable,<EOL>'<STR_LIT>': commiturl,<EOL>'<STR_LIT:args>': args,<EOL>}<EOL>execution_metadata.update(platform_metadata)<EOL>for i in range(len(files)):<EOL><INDENT>files[i]['<STR_LIT>'] = execution_metadata<EOL><DEDENT>return files<EOL>

Run the executable and capture the input and output...


def add(repo, args, targetdir,<EOL>execute=False, generator=False,<EOL>includes=[], script=False,<EOL>source=None):

<EOL>if not execute:<EOL><INDENT>files = add_files(args=args,<EOL>targetdir=targetdir,<EOL>source=source,<EOL>script=script,<EOL>generator=generator)<EOL><DEDENT>else:<EOL><INDENT>files = run_executable(repo, args, includes)<EOL><DEDENT>if files is None or len(files) == <NUM_LIT:0>:<EOL><INDENT>return repo<EOL><DEDENT>filtered_files = []<EOL>package = repo.package<EOL>for h in files:<EOL><INDENT>found = False<EOL>for i, r in  enumerate(package['<STR_LIT>']):<EOL><INDENT>if h['<STR_LIT>'] == r['<STR_LIT>']:<EOL><INDENT>found = True<EOL>if h['<STR_LIT>'] == r['<STR_LIT>']:<EOL><INDENT>change = False<EOL>for attr in ['<STR_LIT:source>']:<EOL><INDENT>if h[attr] != r[attr]:<EOL><INDENT>r[attr] = h[attr]<EOL>change = True<EOL><DEDENT><DEDENT>if change:<EOL><INDENT>filtered_files.append(h)<EOL><DEDENT>continue<EOL><DEDENT>else:<EOL><INDENT>filtered_files.append(h)<EOL>package['<STR_LIT>'][i] = h<EOL><DEDENT>break<EOL><DEDENT><DEDENT>if not found:<EOL><INDENT>filtered_files.append(h)<EOL>package['<STR_LIT>'].append(h)<EOL><DEDENT><DEDENT>if len(filtered_files) == <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>repo.manager.add_files(repo, filtered_files)<EOL>rootdir = repo.rootdir<EOL>with cd(rootdir):<EOL><INDENT>datapath = "<STR_LIT>"<EOL>with open(datapath, '<STR_LIT:w>') as fd:<EOL><INDENT>fd.write(json.dumps(package, indent=<NUM_LIT:4>))<EOL><DEDENT><DEDENT>return len(filtered_files)<EOL>

Add files to the repository by explicitly specifying them or by
specifying a pattern over files accessed during execution of an
executable.

Parameters
----------

repo: Repository

args: files or command line
     (a) If simply adding files, then the list of files that must
     be added (including any additional arguments to be passed to
     git
     (b) If files to be added are an output of a command line, then
     args is the command lined
targetdir: Target directory to store the files
execute: Args are not files to be added but scripts that must be run.
includes: patterns used to select files to
script: Is this a script?
generator: Is this a generator
source: Link to the original source of the data


def instantiate(repo, validator_name=None, filename=None, rulesfiles=None):

default_validators = repo.options.get('<STR_LIT>', {})<EOL>validators = {}<EOL>if validator_name is not None:<EOL><INDENT>if validator_name in default_validators:<EOL><INDENT>validators = {<EOL>validator_name : default_validators[validator_name]<EOL>}<EOL><DEDENT>else:<EOL><INDENT>validators = {<EOL>validator_name : {<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': {},<EOL>'<STR_LIT>': []<EOL>}<EOL>}<EOL><DEDENT><DEDENT>else:<EOL><INDENT>validators = default_validators<EOL><DEDENT>if filename is not None:<EOL><INDENT>matching_files = repo.find_matching_files([filename])<EOL>if len(matching_files) == <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>", filename)<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT>for v in validators:<EOL><INDENT>validators[v]['<STR_LIT>'] = matching_files<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for v in validators:<EOL><INDENT>if '<STR_LIT>' not in validators[v]:<EOL><INDENT>validators[v]['<STR_LIT>'] = []<EOL><DEDENT>elif len(validators[v]['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>matching_files = repo.find_matching_files(validators[v]['<STR_LIT>'])<EOL>validators[v]['<STR_LIT>'] = matching_files<EOL><DEDENT><DEDENT><DEDENT>if rulesfiles is not None:<EOL><INDENT>matching_files = repo.find_matching_files([rulesfiles])<EOL>if len(matching_files) == <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>".format(rulesfiles,v))<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT>for v in validators:<EOL><INDENT>validators[v]['<STR_LIT>'] = matching_files<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for v in validators:<EOL><INDENT>if '<STR_LIT>' not in validators[v]:<EOL><INDENT>validators[v]['<STR_LIT>'] = []<EOL><DEDENT>else:<EOL><INDENT>rulesfiles = validators[v]['<STR_LIT>']<EOL>matching_files = repo.find_matching_files(rulesfiles)<EOL>validators[v]['<STR_LIT>'] = matching_files<EOL><DEDENT><DEDENT><DEDENT>return validators<EOL>

Instantiate the validation specification


@log_repo_action<EOL>def validate(repo, <EOL>validator_name=None, <EOL>filename=None, <EOL>rulesfiles=None,<EOL>args=[]):

mgr = plugins_get_mgr()<EOL>validator_specs = instantiate(repo, validator_name, filename, rulesfiles)<EOL>allresults = []<EOL>for v in validator_specs:<EOL><INDENT>keys = mgr.search(what='<STR_LIT>',name=v)['<STR_LIT>']<EOL>for k in keys:<EOL><INDENT>validator = mgr.get_by_key('<STR_LIT>', k)<EOL>result = validator.evaluate(repo, <EOL>validator_specs[v],<EOL>args)<EOL>allresults.extend(result)<EOL><DEDENT><DEDENT>return allresults<EOL>

Validate the content of the files for consistency. Validators can
look as deeply as needed into the files. dgit treats them all as
black boxes.

Parameters
----------

repo: Repository object
validator_name: Name of validator, if any. If none, then all validators specified in dgit.json will be included.
filename: Pattern that specifies files that must be processed by the validators selected. If none, then the default specification in dgit.json is used.
rules: Pattern specifying the files that have rules that validators will use
show: Print the validation results on the terminal

Returns
-------

status: A list of dictionaries, each with target file processed, rules file applied, status of the validation and any error  message.


mgr = plugins_get_mgr()<EOL>repomgr = mgr.get(what='<STR_LIT>', name='<STR_LIT>')<EOL>repo =  repomgr.lookup(username=username,<EOL>reponame=reponame)<EOL>return repo<EOL>

Lookup a repo based on username reponame


mgr = plugins_get_mgr()<EOL>if not remote:<EOL><INDENT>repomgr = mgr.get(what='<STR_LIT>', name='<STR_LIT>')<EOL>repos = repomgr.get_repo_list()<EOL>repos.sort()<EOL>return repos<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>

List repos

Parameters
----------

remote: Flag


with cd(repo.rootdir):<EOL><INDENT>result = run(args)<EOL>return result<EOL><DEDENT>

Run a shell command within the repo's context

Parameters
----------

repo: Repository object
args: Shell command


datapath = os.path.join(repo.rootdir, "<STR_LIT>")<EOL>return os.path.exists(datapath)<EOL>


return generic_repo_cmd(repo, '<STR_LIT>', args)<EOL>

Log of the changes executed until now

Parameters
----------

repo: Repository object
args: Arguments to git command


Show commit details

Parameters
----------

repo: Repository object
args: Arguments to git command


Show remote

Parameters
----------

repo: Repository object
args: Arguments to git command


Push changes to the backend

Parameters
----------

repo: Repository object
args: Arguments to git command


Pull changes from the backend

Parameters
----------

repo: Repository object
args: Arguments to git command


Commit changes to the data repository

Parameters
----------

repo: Repository object
args: Arguments to git command


Drop the repository (new to dgit)

Parameters
----------

repo: Repository object
args: Arguments to git command


Stash the changes

Parameters
----------

repo: Repository object
args: Arguments to git command


Diff between versions

Parameters
----------

repo: Repository object
args: Arguments to git command


result = generic_repo_cmd(repo, '<STR_LIT:status>', args)<EOL>return result<EOL>

Show status of the repo

Parameters
----------

repo: Repository object (result of lookup)
details: Show internal details of the repo
args: Parameters to be passed to git status command


<EOL>result = generic_repo_cmd(repo, '<STR_LIT>', args)<EOL>if result['<STR_LIT:status>'] != '<STR_LIT:success>': <EOL><INDENT>return status <EOL><DEDENT>with cd(repo.rootdir): <EOL><INDENT>package = repo.package <EOL>resources = package['<STR_LIT>'] <EOL>cleaned_resources = []<EOL>for r in resources: <EOL><INDENT>relativepath = r['<STR_LIT>'] <EOL>sha256 = r['<STR_LIT>'] <EOL>if relativepath not in ['<STR_LIT>', None]: <EOL><INDENT>if not os.path.exists(relativepath): <EOL><INDENT>print("<STR_LIT>", relativepath) <EOL>continue <EOL><DEDENT><DEDENT>cleaned_resources.append(r) <EOL><DEDENT>package['<STR_LIT>'] = cleaned_resources <EOL>repo.package = package <EOL>with open('<STR_LIT>', '<STR_LIT:w>') as fd: <EOL><INDENT>fd.write(json.dumps(repo.package, indent=<NUM_LIT:4>))<EOL><DEDENT>return {<EOL>'<STR_LIT:status>': '<STR_LIT:success>',<EOL>'<STR_LIT:message>': '<STR_LIT>'<EOL>}<EOL><DEDENT>

Delete files

Parameters
----------

repo: Repository object
args: Arguments to git command


def bootstrap_datapackage(repo, force=False,<EOL>options=None, noinput=False):

print("<STR_LIT>")<EOL>tsprefix = datetime.now().date().isoformat()<EOL>package = OrderedDict([<EOL>('<STR_LIT:title>', '<STR_LIT>'),<EOL>('<STR_LIT:description>', '<STR_LIT>'),<EOL>('<STR_LIT:username>', repo.username),<EOL>('<STR_LIT>', repo.reponame),<EOL>('<STR_LIT:name>', str(repo)),<EOL>('<STR_LIT:title>', "<STR_LIT>"),<EOL>('<STR_LIT:description>', "<STR_LIT>"),<EOL>('<STR_LIT>', []),<EOL>('<STR_LIT>', []),<EOL>('<STR_LIT>', getpass.getuser()),<EOL>('<STR_LIT>', datetime.now().isoformat()),<EOL>('<STR_LIT>', repo.remoteurl)<EOL>])<EOL>if options is not None:<EOL><INDENT>package['<STR_LIT:title>'] = options['<STR_LIT:title>']<EOL>package['<STR_LIT:description>'] = options['<STR_LIT:description>']<EOL><DEDENT>else:<EOL><INDENT>if noinput:<EOL><INDENT>raise IncompleteParameters("<STR_LIT>")<EOL><DEDENT>for var in ['<STR_LIT:title>', '<STR_LIT:description>']:<EOL><INDENT>value = '<STR_LIT>'<EOL>while value in ['<STR_LIT>',None]:<EOL><INDENT>value = input('<STR_LIT>' + var.title() + "<STR_LIT>")<EOL>if len(value) == <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>".format(var.title()))<EOL><DEDENT><DEDENT>package[var] = value<EOL><DEDENT><DEDENT>(handle, filename) = tempfile.mkstemp()<EOL>with open(filename, '<STR_LIT:w>') as fd:<EOL><INDENT>fd.write(json.dumps(package, indent=<NUM_LIT:4>))<EOL><DEDENT>repo.package = package<EOL>return filename<EOL>


def init(username, reponame, setup,<EOL>force=False, options=None,<EOL>noinput=False):

mgr = plugins_get_mgr()<EOL>repomgr = mgr.get(what='<STR_LIT>', name='<STR_LIT>')<EOL>backendmgr = None<EOL>if setup == '<STR_LIT>':<EOL><INDENT>backendmgr = mgr.get(what='<STR_LIT>', name='<STR_LIT>')<EOL><DEDENT>repo = repomgr.init(username, reponame, force, backendmgr)<EOL>(handle, gitignore) = tempfile.mkstemp()<EOL>with open(gitignore, '<STR_LIT:w>') as fd:<EOL><INDENT>fd.write("<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>filename = bootstrap_datapackage(repo, force, options, noinput)<EOL><DEDENT>except Exception as e:<EOL><INDENT>repomgr.drop(repo,[])<EOL>os.unlink(gitignore)<EOL>raise e<EOL><DEDENT>repo.run('<STR_LIT>',<EOL>[<EOL>{<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': filename,<EOL>},<EOL>{<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': gitignore,<EOL>},<EOL>])<EOL>os.unlink(filename)<EOL>os.unlink(gitignore)<EOL>args = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>repo.run('<STR_LIT>', args)<EOL>return repo<EOL>

Initialize an empty repository with datapackage.json

Parameters
----------

username: Name of the user
reponame: Name of the repo
setup: Specify the 'configuration' (git only, git+s3 backend etc)
force: Force creation of the files
options: Dictionary with content of dgit.json, if available.
noinput: Automatic operation with no human interaction


backend = None<EOL>backendmgr = None<EOL>if url.startswith('<STR_LIT>'):<EOL><INDENT>backendtype = '<STR_LIT>'<EOL><DEDENT>elif url.startswith("<STR_LIT:http>") or url.startswith("<STR_LIT>"):<EOL><INDENT>backendtype = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>backendtype = None<EOL><DEDENT>mgr = plugins_get_mgr()<EOL>repomgr = mgr.get(what='<STR_LIT>', name='<STR_LIT>')<EOL>backendmgr = mgr.get(what='<STR_LIT>', name=backendtype)<EOL>if backendmgr is not None and not backendmgr.url_is_valid(url):<EOL><INDENT>raise InvalidParameters("<STR_LIT>")<EOL><DEDENT>key = repomgr.clone(url, backendmgr)<EOL>repo = repomgr.lookup(key=key)<EOL>if not datapackage_exists(repo):<EOL><INDENT>filename = bootstrap_datapackage(repo)<EOL>repo.run('<STR_LIT>',<EOL>[<EOL>{<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': filename,<EOL>},<EOL>])<EOL>os.unlink(filename)<EOL>args = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>repo.run('<STR_LIT>', args)<EOL><DEDENT>return repo<EOL>

Clone a URL. Examples include:

    - git@github.com:pingali/dgit.git
    - https://github.com:pingali/dgit.git
    - s3://mybucket/git/pingali/dgit.git

Parameters
----------

url: URL of the repo


def annotate_metadata_data(repo, task, patterns=["<STR_LIT:*>"], size=<NUM_LIT:0>):

mgr = plugins_get_mgr() <EOL>keys = mgr.search('<STR_LIT>')['<STR_LIT>']<EOL>representations = [mgr.get_by_key('<STR_LIT>', k) for k in keys]<EOL>matching_files = repo.find_matching_files(patterns)<EOL>package = repo.package<EOL>rootdir = repo.rootdir<EOL>files = package['<STR_LIT>']<EOL>for f in files:<EOL><INDENT>relativepath = f['<STR_LIT>']<EOL>if relativepath in matching_files:<EOL><INDENT>path = os.path.join(rootdir, relativepath)<EOL>if task == '<STR_LIT>':<EOL><INDENT>print("<STR_LIT>", relativepath)<EOL>f['<STR_LIT:content>'] = open(path).read()[:size]<EOL><DEDENT>elif task == '<STR_LIT>':<EOL><INDENT>for r in representations: <EOL><INDENT>if r.can_process(path): <EOL><INDENT>print("<STR_LIT>", path)<EOL>f['<STR_LIT>'] = r.get_schema(path)<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>

Update metadata with the content of the files


package = repo.package<EOL>package['<STR_LIT:code>'] = []<EOL>for p in files:<EOL><INDENT>matching_files = glob2.glob("<STR_LIT>".format(p))<EOL>for f in matching_files:<EOL><INDENT>absf = os.path.abspath(f)<EOL>print("<STR_LIT>".format(f))<EOL>package['<STR_LIT:code>'].append(OrderedDict([<EOL>('<STR_LIT>', f),<EOL>('<STR_LIT>', repo.manager.permalink(repo, absf)),<EOL>('<STR_LIT>', mimetypes.guess_type(absf)[<NUM_LIT:0>]),<EOL>('<STR_LIT>', compute_sha256(absf))<EOL>]))<EOL><DEDENT><DEDENT>

Update metadata with the commit information


package = repo.package    <EOL>print("<STR_LIT>")<EOL>with cd(repo.rootdir): <EOL><INDENT>filename = "<STR_LIT>"        <EOL>if os.path.exists(filename):             <EOL><INDENT>history = open(filename).readlines() <EOL>actions = []<EOL>for a in history: <EOL><INDENT>try: <EOL><INDENT>a = json.loads(a)<EOL>for x in ['<STR_LIT:code>']: <EOL><INDENT>if x not in a or a[x] == None: <EOL><INDENT>a[x] = "<STR_LIT>"<EOL><DEDENT><DEDENT>actions.append(a)<EOL><DEDENT>except:<EOL><INDENT>pass <EOL><DEDENT><DEDENT>package['<STR_LIT>'] = actions<EOL><DEDENT><DEDENT>


print("<STR_LIT>")<EOL>package = repo.package<EOL>mgr = plugins_get_mgr()<EOL>repomgr = mgr.get(what='<STR_LIT>', name='<STR_LIT>')<EOL>package['<STR_LIT>'] = repomgr.get_metadata()<EOL>


options = repo.options<EOL>if '<STR_LIT>' not in options:<EOL><INDENT>print("<STR_LIT>")<EOL>return []<EOL><DEDENT>repos = []<EOL>dependent_repos = options['<STR_LIT>']<EOL>for d in dependent_repos:<EOL><INDENT>if "<STR_LIT:/>" not in d:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>(username, reponame) = d.split("<STR_LIT:/>")<EOL>try:<EOL><INDENT>repos.append(repo.manager.lookup(username, reponame))<EOL><DEDENT>except:<EOL><INDENT>print("<STR_LIT>", d)<EOL><DEDENT><DEDENT>package = repo.package<EOL>package['<STR_LIT>'] = []<EOL>for r in repos:<EOL><INDENT>package['<STR_LIT>'].append({<EOL>'<STR_LIT:username>': r.username,<EOL>'<STR_LIT>': r.reponame,<EOL>})<EOL><DEDENT>

Collect information from the dependent repo's


mgr = plugins_get_mgr()<EOL>keys = mgr.search(what='<STR_LIT>')<EOL>keys = keys['<STR_LIT>']<EOL>if len(keys) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>if '<STR_LIT>' in repo.options:<EOL><INDENT>for name, details in repo.options['<STR_LIT>'].items():<EOL><INDENT>patterns = details['<STR_LIT>']<EOL>matching_files = repo.find_matching_files(patterns)<EOL>matching_files.sort()<EOL>details['<STR_LIT>'] = matching_files<EOL>for i, f in enumerate(matching_files):<EOL><INDENT>r = repo.get_resource(f)<EOL>if '<STR_LIT>' not in r:<EOL><INDENT>r['<STR_LIT>'] = []<EOL><DEDENT>r['<STR_LIT>'].append(name + "<STR_LIT>".format(i))<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in repo.options:<EOL><INDENT>print("<STR_LIT>")<EOL>metadata = repo.options['<STR_LIT>']<EOL>if '<STR_LIT>' in metadata and metadata['<STR_LIT>']:<EOL><INDENT>repo.package['<STR_LIT>'] = get_history(repo.rootdir)<EOL><DEDENT>if '<STR_LIT>' in metadata and metadata['<STR_LIT>']:<EOL><INDENT>annotate_metadata_action(repo) <EOL><DEDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>annotate_metadata_data(repo,<EOL>task='<STR_LIT>',<EOL>patterns=metadata['<STR_LIT>']['<STR_LIT>'],<EOL>size=metadata['<STR_LIT>']['<STR_LIT>'])<EOL><DEDENT>if (('<STR_LIT>' in metadata) and metadata['<STR_LIT>']):<EOL><INDENT>annotate_metadata_data(repo,  task='<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>annotate_metadata_code(repo, files=metadata['<STR_LIT>'])<EOL><DEDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>annotate_metadata_platform(repo)<EOL><DEDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>annotate_metadata_validation(repo)<EOL><DEDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>annotate_metadata_dependencies(repo)<EOL><DEDENT>history = repo.package.get('<STR_LIT>',None)<EOL>if (('<STR_LIT>' in metadata) and<EOL>metadata['<STR_LIT>'] and<EOL>history is not None):<EOL><INDENT>annotate_metadata_diffs(repo)<EOL><DEDENT>repo.package['<STR_LIT>'] = repo.options<EOL><DEDENT>try:<EOL><INDENT>for k in keys:<EOL><INDENT>metadatamgr = mgr.get_by_key('<STR_LIT>', k)<EOL>url = metadatamgr.url<EOL>o = urlparse(url)<EOL>print("<STR_LIT>", o.netloc)<EOL>response = metadatamgr.post(repo)<EOL>if isinstance(response, str):<EOL><INDENT>print("<STR_LIT>", response)<EOL><DEDENT>elif response.status_code in [<NUM_LIT>]:<EOL><INDENT>content = response.json()<EOL>print("<STR_LIT>")<EOL>for k in content:<EOL><INDENT>print("<STR_LIT:U+0020>", k,"<STR_LIT>", "<STR_LIT:U+002C>".join(content[k]))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>except NetworkError as e:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>except NetworkInvalidConfiguration as e:<EOL><INDENT>print("<STR_LIT>")<EOL>print(e.message)<EOL><DEDENT>except Exception as e:<EOL><INDENT>print("<STR_LIT>")<EOL>print(e)<EOL><DEDENT>

Post to metadata server

Parameters
----------

repo: Repository object (result of lookup)


files = glob.glob("<STR_LIT:*>") + glob.glob("<STR_LIT>") + glob.glob('<STR_LIT>')<EOL>files = filter(lambda f: os.path.isfile(f), files)<EOL>executable = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH<EOL>final = []<EOL>for filename in files:<EOL><INDENT>if os.path.isfile(filename):<EOL><INDENT>st = os.stat(filename)<EOL>mode = st.st_mode<EOL>if mode & executable:<EOL><INDENT>final.append(filename)<EOL>if len(final) > <NUM_LIT:5>:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return final<EOL>

Find max 5 executables that are responsible for this repo.


if os.path.exists(autofile) and not force_init:<EOL><INDENT>try:<EOL><INDENT>autooptions = json.loads(open(autofile).read())<EOL>return autooptions<EOL><DEDENT>except:<EOL><INDENT>print("<STR_LIT>")<EOL>traceback.print_exc()<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT>config = get_config()<EOL>pluginmgr = plugins_get_mgr()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>username = getpass.getuser()<EOL>revised = input("<STR_LIT>".format(username))<EOL>if revised not in ["<STR_LIT>", None]:<EOL><INDENT>username = revised<EOL><DEDENT>thisdir = os.path.abspath(os.getcwd())<EOL>reponame = os.path.basename(thisdir)<EOL>revised = input("<STR_LIT>".format(reponame))<EOL>if revised not in ["<STR_LIT>", None]:<EOL><INDENT>reponame = revised<EOL><DEDENT>keys = pluginmgr.search('<STR_LIT>')<EOL>keys = keys['<STR_LIT>']<EOL>keys = [k for k in keys if k[<NUM_LIT:0>] != "<STR_LIT>"]<EOL>remoteurl = "<STR_LIT>"<EOL>backend = None<EOL>if len(keys) > <NUM_LIT:0>:<EOL><INDENT>backend = pluginmgr.get_by_key('<STR_LIT>', keys[<NUM_LIT:0>])<EOL>candidate = backend.url(username, reponame)<EOL>revised = input("<STR_LIT>".format(candidate))<EOL>if revised in ["<STR_LIT>", None]:<EOL><INDENT>remoteurl = candidate<EOL><DEDENT>else:<EOL><INDENT>remoteurl = revised<EOL><DEDENT><DEDENT>title = "<STR_LIT>"<EOL>while title == "<STR_LIT>":<EOL><INDENT>title = input("<STR_LIT>")<EOL>if title == "<STR_LIT>":<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>description = "<STR_LIT>"<EOL>while description == "<STR_LIT>":<EOL><INDENT>description = input("<STR_LIT>")<EOL>if description == "<STR_LIT>":<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>autooptions = OrderedDict([<EOL>("<STR_LIT:username>", username),<EOL>("<STR_LIT>", reponame),<EOL>("<STR_LIT>", remoteurl),<EOL>("<STR_LIT:title>", title),<EOL>("<STR_LIT:description>", description),<EOL>("<STR_LIT>", "<STR_LIT:.>"),<EOL>('<STR_LIT>' ,OrderedDict([<EOL>('<STR_LIT>', ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>','<STR_LIT>', '<STR_LIT>', '<STR_LIT>', "<STR_LIT>", "<STR_LIT>"]),<EOL>('<STR_LIT>', ['<STR_LIT>', '<STR_LIT>', os.path.basename(autofile)]),<EOL>])),<EOL>('<STR_LIT>', False),<EOL>('<STR_LIT>' ,OrderedDict([])),<EOL>('<STR_LIT>' ,OrderedDict([<EOL>('<STR_LIT>' ,OrderedDict([<EOL>('<STR_LIT:.>', '<STR_LIT>')<EOL>]))<EOL>])),<EOL>('<STR_LIT>' ,OrderedDict([]))<EOL>])<EOL>for p in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>keys = pluginmgr.search(p)<EOL>keys = keys[p]<EOL>options = OrderedDict()<EOL>for k in keys:<EOL><INDENT>if k.name in options:<EOL><INDENT>continue<EOL><DEDENT>mgr = pluginmgr.get_by_key(p, k)<EOL>options[k.name] = mgr.autooptions()<EOL><DEDENT>autooptions[p] = options<EOL><DEDENT>keys = pluginmgr.search('<STR_LIT>')<EOL>keys = keys['<STR_LIT>']<EOL>if len(keys) > <NUM_LIT:0>:<EOL><INDENT>servers = []<EOL>for k in keys:<EOL><INDENT>server = pluginmgr.get_by_key('<STR_LIT>', k)<EOL>server = server.url.split("<STR_LIT:/>")[<NUM_LIT:2>]<EOL>servers.append(server)<EOL><DEDENT>autooptions.update(OrderedDict([<EOL>('<STR_LIT>', OrderedDict([<EOL>('<STR_LIT>', servers),<EOL>('<STR_LIT>', find_executable_files()),<EOL>('<STR_LIT>', OrderedDict([<EOL>('<STR_LIT>', <NUM_LIT>),<EOL>('<STR_LIT>', ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>])),<EOL>('<STR_LIT>', True),<EOL>('<STR_LIT>', True),<EOL>('<STR_LIT>', True),<EOL>('<STR_LIT>', True),<EOL>('<STR_LIT>', True),<EOL>('<STR_LIT>', True),<EOL>('<STR_LIT>', True),<EOL>]))]))<EOL><DEDENT>with open(autofile, '<STR_LIT:w>') as fd:<EOL><INDENT>fd.write(json.dumps(autooptions, indent=<NUM_LIT:4>))<EOL><DEDENT>print("<STR_LIT>")<EOL>print("<STR_LIT>".format(autofile))<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>sys.exit()<EOL>

Initialize a repo-specific configuration file to execute dgit

Parameters
----------

autofile: Repo-specific configuration file (dgit.json)
force_init: Flag to force to re-initialization of the configuration file


<EOL>pluginmgr = plugins_get_mgr()<EOL>repomgr = pluginmgr.get(what='<STR_LIT>', name='<STR_LIT>')<EOL>repo = None<EOL>try:<EOL><INDENT>if debug:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>repo = repomgr.lookup(username=autooptions['<STR_LIT:username>'],<EOL>reponame=autooptions['<STR_LIT>'])<EOL><DEDENT>except:<EOL><INDENT>try:<EOL><INDENT>print("<STR_LIT>")<EOL>url = autooptions['<STR_LIT>']<EOL>if debug:<EOL><INDENT>print("<STR_LIT>".format(url))<EOL><DEDENT>common_clone(url)<EOL>repo = repomgr.lookup(username=autooptions['<STR_LIT:username>'],<EOL>reponame=autooptions['<STR_LIT>'])<EOL>if debug:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT><DEDENT>except:<EOL><INDENT>yes = input("<STR_LIT>")<EOL>if yes == '<STR_LIT:y>':<EOL><INDENT>setup = "<STR_LIT>"<EOL>if autooptions['<STR_LIT>'].startswith('<STR_LIT>'):<EOL><INDENT>setup = '<STR_LIT>'<EOL><DEDENT>repo = common_init(username=autooptions['<STR_LIT:username>'],<EOL>reponame=autooptions['<STR_LIT>'],<EOL>setup=setup,<EOL>force=True,<EOL>options=autooptions)<EOL>if debug:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>repo.options = autooptions<EOL>return repo<EOL>

Automatically get repo

Parameters
----------

autooptions: dgit.json content


workingdir = autooptions['<STR_LIT>']<EOL>includes = autooptions['<STR_LIT>']['<STR_LIT>']<EOL>excludes = autooptions['<STR_LIT>']['<STR_LIT>']<EOL>includes = r'<STR_LIT:|>'.join([fnmatch.translate(x) for x in includes])<EOL>excludes = r'<STR_LIT:|>'.join([fnmatch.translate(x) for x in excludes]) or r'<STR_LIT>'<EOL>matched_files = []<EOL>for root, dirs, files in os.walk(workingdir):<EOL><INDENT>dirs[:] = [d for d in dirs if not re.match(excludes, d)]<EOL>files = [f for f in files if not re.match(excludes, f)]<EOL>files = [f for f in files if re.match(includes, f)]<EOL>files = [os.path.join(root, f) for f in files]<EOL>matched_files.extend(files)<EOL><DEDENT>return matched_files<EOL>

Look through the local directory to pick up files to check


<EOL>mapping = { "<STR_LIT:.>": "<STR_LIT>" }<EOL>if (('<STR_LIT>' in autooptions) and<EOL>('<STR_LIT>' in autooptions['<STR_LIT>'])):<EOL><INDENT>mapping = autooptions['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>keys = mapping.keys()<EOL>keys = sorted(keys, key=lambda k: len(k), reverse=True)<EOL>count = <NUM_LIT:0><EOL>params = []<EOL>for f in files:<EOL><INDENT>relativepath = f<EOL>for k in keys:<EOL><INDENT>v = mapping[k]<EOL>if f.startswith(k + "<STR_LIT:/>"):<EOL><INDENT>relativepath = f.replace(k + "<STR_LIT:/>", v)<EOL>break<EOL><DEDENT><DEDENT>count += files_add(repo=repo,<EOL>args=[f],<EOL>targetdir=os.path.dirname(relativepath))<EOL><DEDENT>return count<EOL>


cmd = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", '<STR_LIT>']<EOL>output = run(cmd)<EOL>lines = output.split("<STR_LIT:\n>")<EOL>content = "<STR_LIT>"<EOL>history = []<EOL>for l in lines:<EOL><INDENT>try:<EOL><INDENT>revisedcontent = content + l<EOL>if revisedcontent.count('<STR_LIT:">') % <NUM_LIT:2> == <NUM_LIT:0>:<EOL><INDENT>j = json.loads(revisedcontent[:-<NUM_LIT:1>])<EOL>if "<STR_LIT>" in j['<STR_LIT>']:<EOL><INDENT>content = "<STR_LIT>"<EOL>continue<EOL><DEDENT>history.append(j)<EOL>content = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>content = revisedcontent<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>print("<STR_LIT>")<EOL>print(revisedcontent)<EOL>content = "<STR_LIT>"<EOL><DEDENT><DEDENT>history.reverse()<EOL>changes = get_change()<EOL>for i in range(len(history)):<EOL><INDENT>abbrev_commit = history[i]['<STR_LIT>']<EOL>if abbrev_commit not in changes:<EOL><INDENT>raise Exception("<STR_LIT>" + abbrev_commit)<EOL><DEDENT>history[i]['<STR_LIT>'] = changes[abbrev_commit]['<STR_LIT>']<EOL><DEDENT>return history<EOL>

Get the commit history for a given dataset


<EOL>mgr = plugins_get_mgr() <EOL>keys = mgr.search('<STR_LIT>')['<STR_LIT>']<EOL>representations = [mgr.get_by_key('<STR_LIT>', k) for k in keys]<EOL>for i in range(len(history)):<EOL><INDENT>if i+<NUM_LIT:1> > len(history) - <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>prev = history[i]<EOL>curr = history[i+<NUM_LIT:1>]<EOL>for c in curr['<STR_LIT>']:<EOL><INDENT>path = c['<STR_LIT:path>']<EOL>if c['<STR_LIT:path>'].endswith('<STR_LIT>'): <EOL><INDENT>continue <EOL><DEDENT>handler = None <EOL>for r in representations: <EOL><INDENT>if r.can_process(path): <EOL><INDENT>handler = r <EOL>break <EOL><DEDENT><DEDENT>if handler is None: <EOL><INDENT>continue <EOL><DEDENT>v1_hex = prev['<STR_LIT>']<EOL>v2_hex = curr['<STR_LIT>']<EOL>temp1 = tempfile.mkdtemp(prefix="<STR_LIT>") <EOL>try: <EOL><INDENT>for h in [v1_hex, v2_hex]: <EOL><INDENT>filename = '<STR_LIT>'.format(temp1, h)<EOL>try:<EOL><INDENT>os.makedirs(os.path.dirname(filename))<EOL><DEDENT>except:<EOL><INDENT>pass <EOL><DEDENT>extractcmd = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', filename, h, path]<EOL>output = run(extractcmd)<EOL>if '<STR_LIT>' in output: <EOL><INDENT>raise Exception("<STR_LIT>") <EOL><DEDENT>with cd(os.path.dirname(filename)): <EOL><INDENT>cmd = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>output = run(cmd) <EOL>if '<STR_LIT>' in output: <EOL><INDENT>print("<STR_LIT>", temp1)<EOL>shutil.rmtree(temp1)<EOL>continue <EOL><DEDENT><DEDENT><DEDENT>path1 = os.path.join(temp1, v1_hex, path) <EOL>path2 = os.path.join(temp1, v2_hex, path) <EOL>if not os.path.exists(path1) or not os.path.exists(path2): <EOL><INDENT>shutil.rmtree(temp1)<EOL>continue <EOL><DEDENT>diff = handler.get_diff(path1, path2)<EOL>c['<STR_LIT>'] = diff<EOL><DEDENT>except Exception as e: <EOL><INDENT>shutil.rmtree(temp1)<EOL><DEDENT><DEDENT><DEDENT>

Look at files and compute the diffs intelligently


def _inner(*args, **kwargs):<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>_inner.__name__ = func.__name__<EOL>_inner.__doc__ = func.__doc__<EOL>return _inner<EOL>


ds = _get_dataset(args)<EOL>ds.validate(log=args.log)<EOL>

cldf validate <DATASET>

Validate a dataset against the CLDF specification, i.e. check
- whether required tables and columns are present
- whether values for required columns are present
- the referential integrity of the dataset


ds = _get_dataset(args)<EOL>print(ds)<EOL>md = Table('<STR_LIT:key>', '<STR_LIT:value>')<EOL>md.extend(ds.properties.items())<EOL>print(md.render(condensed=False, tablefmt=None))<EOL>print()<EOL>t = Table('<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>for p, type_, r in ds.stats():<EOL><INDENT>t.append([p, type_, r])<EOL><DEDENT>print(t.render(condensed=False, tablefmt=None))<EOL>

cldf stats <DATASET>

Print basic stats for CLDF dataset <DATASET>, where <DATASET> may be the path to
- a CLDF metadata file
- a CLDF core data file


if len(args.args) < <NUM_LIT:2>:<EOL><INDENT>raise ParserError('<STR_LIT>')<EOL><DEDENT>ds = _get_dataset(args)<EOL>db = Database(ds, fname=args.args[<NUM_LIT:1>])<EOL>db.write_from_tg()<EOL>args.log.info('<STR_LIT>'.format(ds, db.fname))<EOL>

cldf createdb <DATASET> <SQLITE_DB_PATH>

Load CLDF dataset <DATASET> into a SQLite DB, where <DATASET> may be the path to
- a CLDF metadata file
- a CLDF core data file


if len(args.args) < <NUM_LIT:2>:<EOL><INDENT>raise ParserError('<STR_LIT>')  <EOL><DEDENT>ds = _get_dataset(args)<EOL>db = Database(ds, fname=args.args[<NUM_LIT:1>])<EOL>mdpath = Path(args.args[<NUM_LIT:2>]) if len(args.args) > <NUM_LIT:2> else ds.tablegroup._fname<EOL>args.log.info('<STR_LIT>'.format(db.to_cldf(mdpath.parent, mdname=mdpath.name)))<EOL>

cldf dumpdb <DATASET> <SQLITE_DB_PATH> [<METADATA_PATH>]


if not component:<EOL><INDENT>for table in self.tables:<EOL><INDENT>self.auto_constraints(table)<EOL><DEDENT>return<EOL><DEDENT>if not component.tableSchema.primaryKey:<EOL><INDENT>idcol = component.get_column(term_uri('<STR_LIT:id>'))<EOL>if idcol:<EOL><INDENT>component.tableSchema.primaryKey = [idcol.name]<EOL><DEDENT><DEDENT>self._auto_foreign_keys(component)<EOL>try:<EOL><INDENT>table_type = self.get_tabletype(component)<EOL><DEDENT>except ValueError:<EOL><INDENT>return<EOL><DEDENT>for table in self.tables:<EOL><INDENT>self._auto_foreign_keys(table, component=component, table_type=table_type)<EOL><DEDENT>

Use CLDF reference properties to implicitely create foreign key constraints.

:param component: A Table object or `None`.


if isinstance(item, tuple):<EOL><INDENT>table, column = item<EOL><DEDENT>else:<EOL><INDENT>table, column = item, None<EOL><DEDENT>if not isinstance(table, Table):<EOL><INDENT>uri = term_uri(table, terms=TERMS.by_uri)<EOL>for t in self.tables:<EOL><INDENT>if (uri and t.common_props.get('<STR_LIT>') == uri)or t.url.string == table:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise KeyError(table)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>t = table<EOL><DEDENT>if not column:<EOL><INDENT>return t<EOL><DEDENT>uri = term_uri(column, terms=TERMS.by_uri)<EOL>for c in t.tableSchema.columns:<EOL><INDENT>if (c.propertyUrl and c.propertyUrl.uri == uri) or c.header == column:<EOL><INDENT>return c<EOL><DEDENT><DEDENT>raise KeyError(column)<EOL>

Access to tables and columns.

If a pair (table-spec, column-spec) is passed as `item`, a Column will be
returned, otherwise `item` is assumed to be a table-spec.

A table-spec may be
- a CLDF ontology URI matching the dc:conformsTo property of a table
- the local name of a CLDF ontology URI, where the complete URI matches the
  the dc:conformsTo property of a table
- a filename matching the `url` property of a table

A column-spec may be
- a CLDF ontology URI matching the propertyUrl of a column
- the local name of a CLDF ontology URI, where the complete URI matches the
  propertyUrl of a column
- the name of a column


def get_subsequence(self, cognate, form=None):

return resolve_slices(<EOL>cognate,<EOL>self,<EOL>('<STR_LIT>', "<STR_LIT>"),<EOL>('<STR_LIT>', "<STR_LIT>"),<EOL>'<STR_LIT>',<EOL>target_row=form)<EOL>

Compute the subsequence of the morphemes of a form which is specified in a partial
cognate assignment.

:param partial_cognate:
:return:


def to_cldf(self, dest, mdname='<STR_LIT>'):

dest = Path(dest)<EOL>if not dest.exists():<EOL><INDENT>dest.mkdir()<EOL><DEDENT>data = self.read()<EOL>if data[self.source_table_name]:<EOL><INDENT>sources = Sources()<EOL>for src in data[self.source_table_name]:<EOL><INDENT>sources.add(Source(<EOL>src['<STR_LIT>'],<EOL>src['<STR_LIT:id>'],<EOL>**{k: v for k, v in src.items() if k not in ['<STR_LIT:id>', '<STR_LIT>']}))<EOL><DEDENT>sources.write(dest / self.dataset.properties.get('<STR_LIT>', '<STR_LIT>'))<EOL><DEDENT>for table_type, items in data.items():<EOL><INDENT>try:<EOL><INDENT>table = self.dataset[table_type]<EOL>table.common_props['<STR_LIT>'] = table.write(<EOL>[self.retranslate(table, item) for item in items],<EOL>base=dest)<EOL><DEDENT>except KeyError:<EOL><INDENT>assert table_type == self.source_table_name, table_type<EOL><DEDENT><DEDENT>return self.dataset.write_metadata(dest / mdname)<EOL>

Write the data from the db to a CLDF dataset according to the metadata in `self.dataset`.

:param dest:
:param mdname:
:return: path of the metadata file


for entry in entries:<EOL><INDENT>if isinstance(entry, string_types):<EOL><INDENT>self._add_entries(database.parse_string(entry, bib_format='<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>self._add_entries(entry)<EOL><DEDENT><DEDENT>

Add a source, either specified by glottolog reference id, or as bibtex record.


classes = super(MainNavigationBaseTab, self).tab_classes[:]<EOL>if self.current_tab.request.user.is_authenticated():<EOL><INDENT>classes += ['<STR_LIT>']<EOL><DEDENT>return classes<EOL>

If user is logged in, set ``logged_in_only`` class.


@decorators.method_decorator(login_required)<EOL><INDENT>def dispatch(self, *args, **kwargs):<DEDENT>

return super(HamTab, self).dispatch(*args, **kwargs)<EOL>

Make sure only authenticated users can access this tab.


return self.current_tab.request.user.is_authenticated()<EOL>

Show tab only if current user is logged in.