if is_atlas2anat_inverted:<EOL><INDENT>anat_to_mni_nl_inv = atlas2anat_nonlin_xfm_filepath<EOL><DEDENT>else:<EOL><INDENT>output_dir         = op.abspath   (op.dirname(atlasinanat_out_filepath))<EOL>ext                = get_extension(atlas2anat_nonlin_xfm_filepath)<EOL>anat_to_mni_nl_inv = op.join(output_dir, remove_ext(op.basename(atlas2anat_nonlin_xfm_filepath)) + '<STR_LIT>' + ext)<EOL><DEDENT>invwarp_cmd   = op.join('<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>applywarp_cmd = op.join('<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>fslsub_cmd    = op.join('<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>if parallel:<EOL><INDENT>invwarp_cmd   = fslsub_cmd + '<STR_LIT:U+0020>' + invwarp_cmd<EOL>applywarp_cmd = fslsub_cmd + '<STR_LIT:U+0020>' + applywarp_cmd<EOL><DEDENT>if rewrite or (not is_atlas2anat_inverted and not op.exists(anat_to_mni_nl_inv)):<EOL><INDENT>log.debug('<STR_LIT>'.format(anat_to_mni_nl_inv))<EOL>cmd  = invwarp_cmd + '<STR_LIT:U+0020>'<EOL>cmd += '<STR_LIT>'.format(atlas2anat_nonlin_xfm_filepath)<EOL>cmd += '<STR_LIT>'.format(anat_to_mni_nl_inv)<EOL>cmd += '<STR_LIT>'.format(anatbrain_filepath)<EOL>log.debug('<STR_LIT>'.format(cmd))<EOL>check_call(cmd)<EOL><DEDENT>if rewrite or not op.exists(atlasinanat_out_filepath):<EOL><INDENT>log.debug('<STR_LIT>'.format(atlasinanat_out_filepath))<EOL>cmd  = applywarp_cmd + '<STR_LIT:U+0020>'<EOL>cmd += '<STR_LIT>'.format(atlas_filepath)<EOL>cmd += '<STR_LIT>'.format(anatbrain_filepath)<EOL>cmd += '<STR_LIT>'.format(anat_to_mni_nl_inv)<EOL>cmd += '<STR_LIT>'.format(interp)<EOL>cmd += '<STR_LIT>'.format(atlasinanat_out_filepath)<EOL>log.debug('<STR_LIT>'.format(cmd))<EOL>check_call(cmd)<EOL><DEDENT>if rewrite or not op.exists(atlasinfunc_out_filepath):<EOL><INDENT>log.debug('<STR_LIT>'.format(atlasinfunc_out_filepath))<EOL>cmd  = applywarp_cmd + '<STR_LIT:U+0020>'<EOL>cmd += '<STR_LIT>'.format(atlasinanat_out_filepath)<EOL>cmd += '<STR_LIT>'.format(meanfunc_filepath)<EOL>cmd += '<STR_LIT>'.format(anat2func_lin_xfm_filepath)<EOL>cmd += '<STR_LIT>'.format(interp)<EOL>cmd += '<STR_LIT>'.format(atlasinfunc_out_filepath)<EOL>log.debug('<STR_LIT>'.format(cmd))<EOL>check_call(cmd)<EOL><DEDENT>

Call FSL tools to apply transformations to a given atlas to a functional image.
    Given the transformation matrices.

    Parameters
    ----------
    atlas_filepath: str
        Path to the 3D atlas volume file.

    anatbrain_filepath: str
        Path to the anatomical brain volume file (skull-stripped and registered to the same space as the atlas,
        e.g., MNI).

    meanfunc_filepath: str
        Path to the average functional image to be used as reference in the last applywarp step.

    atlas2anat_nonlin_xfm_filepath: str
        Path to the atlas to anatomical brain linear transformation .mat file.
        If you have the inverse transformation, i.e., anatomical brain to atlas, set is_atlas2anat_inverted to True.

    is_atlas2anat_inverted: bool
        If False will have to calculate the inverse atlas2anat transformation to apply the transformations.
        This step will be performed with FSL invwarp.

    anat2func_lin_xfm_filepath: str
        Path to the anatomical to functional .mat linear transformation file.

    atlasinanat_out_filepath: str
        Path to output file which will contain the 3D atlas in the subject anatomical space.

    atlasinfunc_out_filepath: str
        Path to output file which will contain the 3D atlas in the subject functional space.

    verbose: bool
        If verbose will show DEBUG log info.

    rewrite: bool
        If True will re-run all the commands overwriting any existing file. Otherwise will check if
        each file exists and if it does won't run the command.

    parallel: bool
        If True will launch the commands using ${FSLDIR}/fsl_sub to use the cluster infrastructure you have setup
        with FSL (SGE or HTCondor).


if is_img(div2):<EOL><INDENT>return img1.get_data()/div2.get_data()<EOL><DEDENT>elif isinstance(div2, (float, int)):<EOL><INDENT>return img1.get_data()/div2<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError('<STR_LIT>'<EOL>'<STR_LIT>'.format(type(img1),<EOL>img1,<EOL>type(div2),<EOL>div2))<EOL><DEDENT>

Pixelwise division or divide by a number


bool_img = read_img(img).get_data() > <NUM_LIT:0><EOL>return bool_img.astype(int)<EOL>

Return an image with the positive voxels of the data of `img`.


def filter_icc(icc, mask=None, thr=<NUM_LIT:2>, zscore=True, mode="<STR_LIT:+>"):

if zscore:<EOL><INDENT>icc_filt = thr_img(icc_img_to_zscore(icc), thr=thr, mode=mode)<EOL><DEDENT>else:<EOL><INDENT>icc_filt = thr_img(icc, thr=thr, mode=mode)<EOL><DEDENT>if mask is not None:<EOL><INDENT>icc_filt = apply_mask(icc_filt, mask)<EOL><DEDENT>return icc_filt<EOL>

Threshold then mask an IC correlation map.
    Parameters
    ----------
    icc: img-like
        The 'raw' ICC map.

    mask: img-like
        If not None. Will apply this masks in the end of the process.

    thr: float
        The threshold value.

    zscore: bool
        If True will calculate the z-score of the ICC before thresholding.

    mode: str
        Choices: '+' for positive threshold,
                 '+-' for positive and negative threshold and
                 '-' for negative threshold.

    Returns
    -------
    icc_filt: nibabel.NiftiImage
        Thresholded and masked ICC.


@nifti_out<EOL>def thr_img(img, thr=<NUM_LIT>, mode='<STR_LIT:+>'):

vol  = read_img(img).get_data()<EOL>if mode == '<STR_LIT:+>':<EOL><INDENT>mask = vol > thr<EOL><DEDENT>elif mode == '<STR_LIT>' or mode == '<STR_LIT>':<EOL><INDENT>mask = np.abs(vol) > thr<EOL><DEDENT>elif mode == '<STR_LIT:->':<EOL><INDENT>mask = vol < -thr<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>".format(mode))<EOL><DEDENT>return vol * mask<EOL>

Use the given magic function name `func` to threshold with value `thr`
    the data of `img` and return a new nibabel.Nifti1Image.
    Parameters
    ----------
    img: img-like

    thr: float or int
        The threshold value.

    mode: str
        Choices: '+' for positive threshold,
                 '+-' for positive and negative threshold and
                 '-' for negative threshold.
    Returns
    -------
    thr_img: nibabel.Nifti1Image
        Thresholded image


bool_img = np.abs(read_img(img).get_data())<EOL>return bool_img.astype(int)<EOL>

Return an image with the binarised version of the data of `img`.


bool_img = read_img(img).get_data() < <NUM_LIT:0><EOL>return bool_img.astype(int)<EOL>

Return an image with the negative voxels of the data of `img`.


warnings.warn("<STR_LIT>",<EOL>DeprecationWarning)<EOL>return get_img_info(img_file)<EOL>


warnings.warn("<STR_LIT>",<EOL>DeprecationWarning)<EOL>return get_img_data(nii_file)<EOL>


def new_img_like(ref_niimg, data, affine=None, copy_header=False):

<EOL>if not (hasattr(ref_niimg, '<STR_LIT>')<EOL>and hasattr(ref_niimg,'<STR_LIT>')):<EOL><INDENT>if isinstance(ref_niimg, _basestring):<EOL><INDENT>ref_niimg = nib.load(ref_niimg)<EOL><DEDENT>elif operator.isSequenceType(ref_niimg):<EOL><INDENT>ref_niimg = nib.load(ref_niimg[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>raise TypeError(('<STR_LIT>'<EOL>'<STR_LIT>') % ref_niimg )<EOL><DEDENT><DEDENT>if affine is None:<EOL><INDENT>affine = ref_niimg.get_affine()<EOL><DEDENT>if data.dtype == bool:<EOL><INDENT>default_dtype = np.int8<EOL>if (LooseVersion(nib.__version__) >= LooseVersion('<STR_LIT>') and<EOL>isinstance(ref_niimg, nib.freesurfer.mghformat.MGHImage)):<EOL><INDENT>default_dtype = np.uint8<EOL><DEDENT>data = as_ndarray(data, dtype=default_dtype)<EOL><DEDENT>header = None<EOL>if copy_header:<EOL><INDENT>header = copy.copy(ref_niimg.get_header())<EOL>header['<STR_LIT>'] = <NUM_LIT:0.><EOL>header['<STR_LIT>'] = <NUM_LIT:0.><EOL>header['<STR_LIT>'] = <NUM_LIT:0.><EOL>header['<STR_LIT>'] = np.max(data) if data.size > <NUM_LIT:0> else <NUM_LIT:0.><EOL>header['<STR_LIT>'] = np.min(data) if data.size > <NUM_LIT:0> else <NUM_LIT:0.><EOL><DEDENT>return ref_niimg.__class__(data, affine, header=header)<EOL>

Create a new image of the same class as the reference image

    Parameters
    ----------
    ref_niimg: image
        Reference image. The new image will be of the same type.

    data: numpy array
        Data to be stored in the image

    affine: 4x4 numpy array, optional
        Transformation matrix

    copy_header: boolean, optional
        Indicated if the header of the reference image should be used to
        create the new image

    Returns
    -------
    new_img: image
        A loaded image with the same type (and header) as the reference image.


shape = img.shape<EOL>if len(shape) == <NUM_LIT:3>:<EOL><INDENT>return img<EOL><DEDENT>elif len(shape) == <NUM_LIT:4> and shape[<NUM_LIT:3>] == <NUM_LIT:1>:<EOL><INDENT>return img[:, :, :, <NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>raise TypeError('<STR_LIT>'.format(shape))<EOL><DEDENT>

Enforce that img is a 3D img-like object, if it is not, raise a TypeError.
    i.e., remove dimensions of size 1.

    Parameters
    ----------
    img: numpy.ndarray
        Image data array

    Returns
    -------
    3D numpy ndarray object


def write_mhd_file(filename, data, shape=None, meta_dict=None):

<EOL>ext = get_extension(filename)<EOL>fname = op.basename(filename)<EOL>if ext != '<STR_LIT>' or ext != '<STR_LIT>':<EOL><INDENT>mhd_filename = fname + '<STR_LIT>'<EOL>raw_filename = fname + '<STR_LIT>'<EOL><DEDENT>elif ext == '<STR_LIT>':<EOL><INDENT>mhd_filename = fname<EOL>raw_filename = remove_ext(fname) + '<STR_LIT>'<EOL><DEDENT>elif ext == '<STR_LIT>':<EOL><INDENT>mhd_filename = remove_ext(fname) + '<STR_LIT>'<EOL>raw_filename = fname<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'.format(ext, filename))<EOL><DEDENT>if meta_dict is None:<EOL><INDENT>meta_dict = {}<EOL><DEDENT>if shape is None:<EOL><INDENT>shape = data.shape<EOL><DEDENT>meta_dict['<STR_LIT>']             = meta_dict.get('<STR_LIT>',             '<STR_LIT>')<EOL>meta_dict['<STR_LIT>']             = meta_dict.get('<STR_LIT>',             '<STR_LIT:True>' )<EOL>meta_dict['<STR_LIT>'] = meta_dict.get('<STR_LIT>', '<STR_LIT:False>')<EOL>meta_dict['<STR_LIT>']            = meta_dict.get('<STR_LIT>',            NUMPY_TO_MHD_TYPE[data.dtype.type])<EOL>meta_dict['<STR_LIT>']                  = meta_dict.get('<STR_LIT>',                  str(len(shape)))<EOL>meta_dict['<STR_LIT>']                = meta_dict.get('<STR_LIT>',                '<STR_LIT:U+0020>'.join([str(i) for i in shape]))<EOL>meta_dict['<STR_LIT>']        = meta_dict.get('<STR_LIT>',        raw_filename)<EOL>mhd_filename = op.join(op.dirname(filename), mhd_filename)<EOL>raw_filename = op.join(op.dirname(filename), raw_filename)<EOL>write_meta_header(mhd_filename, meta_dict)<EOL>dump_raw_data(raw_filename, data)<EOL>return mhd_filename, raw_filename<EOL>

Write the `data` and `meta_dict` in two files with names
    that use `filename` as a prefix.

    Parameters
    ----------
    filename: str
        Path to the output file.
        This is going to be used as a preffix.
        Two files will be created, one with a '.mhd' extension
        and another with '.raw'. If `filename` has any of these already
        they will be taken into account to build the filenames.

    data: numpy.ndarray
        n-dimensional image data array.

    shape: tuple
        Tuple describing the shape of `data`
        Default: data.shape

    meta_dict: dict
        Dictionary with the fields of the metadata .mhd file
        Default: {}

    Returns
    -------
    mhd_filename: str
        Path to the .mhd file

    raw_filename: str
        Path to the .raw file


header = '<STR_LIT>'<EOL>for tag in MHD_TAGS:<EOL><INDENT>if tag in meta_dict.keys():<EOL><INDENT>header += '<STR_LIT>'.format(tag, meta_dict[tag])<EOL><DEDENT><DEDENT>with open(filename, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(header)<EOL><DEDENT>

Write the content of the `meta_dict` into `filename`.

    Parameters
    ----------
    filename: str
        Path to the output file

    meta_dict: dict
        Dictionary with the fields of the metadata .mhd file


<EOL>if not op.exists(src):<EOL><INDENT>raise IOError('<STR_LIT>'.format(src))<EOL><DEDENT>ext = get_extension(src)<EOL>if ext != '<STR_LIT>':<EOL><INDENT>msg = '<STR_LIT>'.format(src)<EOL>raise ValueError(msg)<EOL><DEDENT>meta_src = _read_meta_header(src)<EOL>src_raw = meta_src['<STR_LIT>']<EOL>if not op.isabs(src_raw):<EOL><INDENT>src_raw = op.join(op.dirname(src), src_raw)<EOL><DEDENT>if op.isdir(dst):<EOL><INDENT>shutil.copyfile(src, dst)<EOL>shutil.copyfile(src_raw, dst)<EOL>return dst<EOL><DEDENT>dst_raw = op.join(op.dirname(dst), remove_ext(op.basename(dst))) + '<STR_LIT>'<EOL>if get_extension(dst) != '<STR_LIT>':<EOL><INDENT>dst += '<STR_LIT>'<EOL><DEDENT>log.debug('<STR_LIT>'.format(src,     dst))<EOL>log.debug('<STR_LIT>'.format(src_raw, dst_raw))<EOL>shutil.copyfile(src, dst)<EOL>shutil.copyfile(src_raw, dst_raw)<EOL>if op.basename(dst) != op.basename(src):<EOL><INDENT>log.debug('<STR_LIT>'.format(dst, src_raw,<EOL>op.basename(dst_raw)))<EOL>meta_dst = _read_meta_header(dst)<EOL>meta_dst['<STR_LIT>'] = op.basename(dst_raw)<EOL>write_meta_header(dst, meta_dst)<EOL><DEDENT>return dst<EOL>

Copy .mhd and .raw files to dst.

    If dst is a folder, won't change the file, but if dst is another filepath,
    will modify the ElementDataFile field in the .mhd to point to the
    new renamed .raw file.

    Parameters
    ----------
    src: str
        Path to the .mhd file to be copied

    dst: str
        Path to the destination of the .mhd and .raw files.
        If a new file name is given, the extension will be ignored.

    Returns
    -------
    dst: str


if data.ndim == <NUM_LIT:3>:<EOL><INDENT>data = data.reshape([data.shape[<NUM_LIT:0>], data.shape[<NUM_LIT:1>]*data.shape[<NUM_LIT:2>]])<EOL><DEDENT>a = array.array('<STR_LIT:f>')<EOL>for o in data:<EOL><INDENT>a.fromlist(list(o.flatten()))<EOL><DEDENT>with open(filename, '<STR_LIT:wb>') as rawf:<EOL><INDENT>a.tofile(rawf)<EOL><DEDENT>

Write the data into a raw format file. Big endian is always used.

    Parameters
    ----------
    filename: str
        Path to the output file

    data: numpy.ndarray
        n-dimensional image data array.


meta_dict = _read_meta_header(filename)<EOL>dim       = int(meta_dict['<STR_LIT>'])<EOL>assert (meta_dict['<STR_LIT>'] in MHD_TO_NUMPY_TYPE)<EOL>arr = [int(i) for i in meta_dict['<STR_LIT>'].split()]<EOL>volume = reduce(lambda x, y: x*y, arr[<NUM_LIT:0>:dim-<NUM_LIT:1>], <NUM_LIT:1>)<EOL>pwd       = op.dirname(filename)<EOL>raw_file  = meta_dict['<STR_LIT>']<EOL>data_file = op.join(pwd, raw_file)<EOL>ndtype    = MHD_TO_NUMPY_TYPE[meta_dict['<STR_LIT>']]<EOL>arrtype   = NDARRAY_TO_ARRAY_TYPE[ndtype]<EOL>with open(data_file, '<STR_LIT:rb>') as fid:<EOL><INDENT>binvalues = array.array(arrtype)<EOL>binvalues.fromfile(fid, volume*arr[dim-<NUM_LIT:1>])<EOL><DEDENT>data = np.array  (binvalues, ndtype)<EOL>data = np.reshape(data, (arr[dim-<NUM_LIT:1>], volume))<EOL>if dim >= <NUM_LIT:3>:<EOL><INDENT>dimensions = [int(i) for i in meta_dict['<STR_LIT>'].split()]<EOL>data       = data.reshape(dimensions)<EOL><DEDENT>return data, meta_dict<EOL>

Return a dictionary of meta data from meta header file.

    Parameters
    ----------
    filename: str
        Path to a .mhd file

    Returns
    -------
    data: numpy.ndarray
        n-dimensional image data array.

    meta_dict: dict
        A dictionary with the .mhd header content.


return tabulate(self, tablefmt=tablefmt)<EOL>

:param tablefmt: string
 Supported table formats are:
"plain"
"simple"
"grid"
"pipe"
"orgtbl"
"rst"
"mediawiki"
"latex"

:return: tabulate
Tabulated content


return tabulate(list(self.items()), tablefmt=tablefmt)<EOL>


if smooth_fwhm <= <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>old_smooth_fwhm   = self._smooth_fwhm<EOL>self._smooth_fwhm = smooth_fwhm<EOL>try:<EOL><INDENT>data = self.get_data(smoothed=True, masked=True, safe_copy=True)<EOL><DEDENT>except ValueError as ve:<EOL><INDENT>self._smooth_fwhm = old_smooth_fwhm<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>self._smooth_fwhm = smooth_fwhm<EOL>return data<EOL><DEDENT>

Set self._smooth_fwhm and then smooths the data.
        See boyle.nifti.smooth.smooth_imgs.

        Returns
        -------
        the smoothed data deepcopied.


self.set_mask(mask_img)<EOL>return self.get_data(masked=True, smoothed=True, safe_copy=True)<EOL>

First set_mask and the get_masked_data.

        Parameters
        ----------
        mask_img:  nifti-like image, NeuroImage or str
            3D mask array: True where a voxel should be used.
            Can either be:
            - a file path to a Nifti image
            - any object with get_data() and get_affine() methods, e.g., nibabel.Nifti1Image.
            If niimg is a string, consider it as a path to Nifti image and
            call nibabel.load on it. If it is an object, check if get_data()
            and get_affine() methods are present, raise TypeError otherwise.

        Returns
        -------
        The masked data deepcopied


if not self.has_mask() and not self.is_smoothed():<EOL><INDENT>save_niigz(outpath, self.img)<EOL><DEDENT>else:<EOL><INDENT>save_niigz(outpath, self.get_data(masked=True, smoothed=True),<EOL>self.get_header(), self.get_affine())<EOL><DEDENT>

Save this object instance in outpath.

        Parameters
        ----------
        outpath: str
            Output file path


<EOL>if not op.exists(filepath):<EOL><INDENT>raise IOError('<STR_LIT>'.format(filepath))<EOL><DEDENT>def open_nifti_file(filepath):<EOL><INDENT>return NiftiImage(filepath)<EOL><DEDENT>def open_mhd_file(filepath):<EOL><INDENT>return MedicalImage(filepath)<EOL>vol_data, hdr_data = load_raw_data_with_mhd(filepath)<EOL>return vol_data, hdr_data<EOL><DEDENT>def open_mha_file(filepath):<EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT>def _load_file(filepath, loader):<EOL><INDENT>return loader(filepath)<EOL><DEDENT>filext_loader = {<EOL>'<STR_LIT>': open_nifti_file,<EOL>'<STR_LIT>': open_mhd_file,<EOL>'<STR_LIT>': open_mha_file,<EOL>}<EOL>ext = get_extension(filepath)<EOL>loader = None<EOL>for e in filext_loader:<EOL><INDENT>if ext in e:<EOL><INDENT>loader = filext_loader[e]<EOL><DEDENT><DEDENT>if loader is None:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(filepath))<EOL><DEDENT>return _load_file(filepath, loader)<EOL>

Open a volumetric file using the tools following the file extension.

    Parameters
    ----------
    filepath: str
        Path to a volume file

    Returns
    -------
    volume_data: np.ndarray
        Volume data

    pixdim: 1xN np.ndarray
        Vector with the description of the voxels physical size (usually in mm) for each volume dimension.

    Raises
    ------
    IOError
        In case the file is not found.


updated_msg = msg<EOL>for k, v in sorted(kwargs.items()):<EOL><INDENT>if isinstance(v, _basestring):  <EOL><INDENT>updated_msg += "<STR_LIT:\n>" + k + "<STR_LIT>" + v<EOL><DEDENT><DEDENT>return updated_msg<EOL>

Append key-value pairs to msg, for display.

    Parameters
    ----------
    msg: string
        arbitrary message
    kwargs: dict
        arbitrary dictionary

    Returns
    -------
    updated_msg: string
        msg, with "key: value" appended. Only string values are appended.

    Example
    -------
    >>> compose_err_msg('Error message with arguments...', arg_num=123, \
        arg_str='filename.nii', arg_bool=True)
    'Error message with arguments...\\narg_str: filename.nii'
    >>>


def insert_unique(self, table_name, data, unique_fields=None, *, raise_if_found=False):

return insert_unique(table=self.table(table_name),<EOL>data=_to_string(data),<EOL>unique_fields=unique_fields,<EOL>raise_if_found=raise_if_found)<EOL>

Insert `data` into `table` ensuring that data has unique values
        in `table` for the fields listed in `unique_fields`.

        If `raise_if_found` is True, will raise an NotUniqueItemError if
        another item with the same `unique_fields` values are found
        previously in `table`.
        If False, will return the `eid` from the item found.

        Parameters
        ----------
        table_name: str

        data: dict

        unique_fields: list of str
            Name of fields (keys) from `data` which are going to be used to build
            a sample to look for exactly the same values in the database.
            If None, will use every key in `data`.

        raise_if_found: bool

        Returns
        -------
        eid: int
            Id of the object inserted or the one found with same `unique_fields`.

        Raises
        ------
        MoreThanOneItemError
            Raise even with `raise_with_found` == False if it finds more than one item
            with the same values as the sample.

        NotUniqueItemError
            If `raise_if_found` is True and an item with the same `unique_fields`
            values from `data` is found in `table`.


utc = tzutc()<EOL>if dt.tzinfo:<EOL><INDENT>dt = dt.astimezone(utc).replace(tzinfo=None)<EOL><DEDENT>return dt.isoformat() + '<STR_LIT>'<EOL>

Serialize a date/time value into an ISO8601 text representation
adjusted (if needed) to UTC timezone.

For instance:
>>> serialize_date(datetime(2012, 4, 10, 22, 38, 20, 604391))
'2012-04-10T22:38:20.604391Z'


def search_unique(table, sample, unique_fields=None):

if unique_fields is None:<EOL><INDENT>unique_fields = list(sample.keys())<EOL><DEDENT>query = _query_data(sample, field_names=unique_fields, operators='<STR_LIT>')<EOL>items = table.search(query)<EOL>if len(items) == <NUM_LIT:1>:<EOL><INDENT>return items[<NUM_LIT:0>]<EOL><DEDENT>if len(items) == <NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>raise MoreThanOneItemError('<STR_LIT>'<EOL>'<STR_LIT>'.format(len(items)))<EOL>

Search for items in `table` that have the same field sub-set values as in `sample`.
    Expecting it to be unique, otherwise will raise an exception.

    Parameters
    ----------
    table: tinydb.table
    sample: dict
        Sample data

    Returns
    -------
    search_result: tinydb.database.Element
        Unique item result of the search.

    Raises
    ------
    KeyError:
        If the search returns for more than one entry.


def insert_unique(table, data, unique_fields=None, *, raise_if_found=False):

item = find_unique(table, data, unique_fields)<EOL>if item is not None:<EOL><INDENT>if raise_if_found:<EOL><INDENT>raise NotUniqueItemError('<STR_LIT>'<EOL>'<STR_LIT>'.format(unique_fields,<EOL>data,<EOL>table.get(eid=item),<EOL>item))<EOL><DEDENT>else:<EOL><INDENT>return item<EOL><DEDENT><DEDENT>return table.insert(data)<EOL>

Insert `data` into `table` ensuring that data has unique values
    in `table` for the fields listed in `unique_fields`.

    If `raise_if_found` is True, will raise an NotUniqueItemError if
    another item with the same `unique_fields` values are found
    previously in `table`.
    If False, will return the `eid` from the item found.

    Parameters
    ----------
    table: tinydb.Table

    data: dict

    unique_fields: list of str
        Name of fields (keys) from `data` which are going to be used to build
        a sample to look for exactly the same values in the database.
        If None, will use every key in `data`.

    raise_if_found: bool

    Returns
    -------
    eid: int
        Id of the object inserted or the one found with same `unique_fields`.

    Raises
    ------
    MoreThanOneItemError
        Raise even with `raise_with_found` == False if it finds more than one item
        with the same values as the sample.

    NotUniqueItemError
        If `raise_if_found` is True and an item with the same `unique_fields`
        values from `data` is found in `table`.


def search_sample(self, table_name, sample):

return search_sample(table=self.table(table_name),<EOL>sample=sample)<EOL>

Search for items in `table` that have the same field sub-set values as in `sample`.

        Parameters
        ----------
        table_name: str

        sample: dict
            Sample data

        Returns
        -------
        search_result: list of dict
            List of the items found. The list is empty if no item is found.


def _query_data(data, field_names=None, operators='<STR_LIT>'):

if field_names is None:<EOL><INDENT>field_names = list(data.keys())<EOL><DEDENT>if isinstance(field_names, str):<EOL><INDENT>field_names = [field_names]<EOL><DEDENT>sample = OrderedDict([(fn, data[fn]) for fn in field_names])<EOL>return _query_sample(sample, operators=operators)<EOL>

Create a tinyDB Query object that looks for items that confirms the correspondent operator
    from `operators` for each `field_names` field values from `data`.

    Parameters
    ----------
    data: dict
        The data sample

    field_names: str or list of str
        The name of the fields in `data` that will be used for the query.

    operators: str or list of str
        A list of comparison operations for each field value in `field_names`.
        If this is a str, will use the same operator for all `field_names`.
        If you want different operators for each field, remember to use an OrderedDict for `data`.
        Check TinyDB.Query class for possible choices.

    Returns
    -------
    query: tinydb.database.Query


install_deps = []<EOL>try:<EOL><INDENT>for fpath in args:<EOL><INDENT>install_deps.extend([str(d.req or d.url) for d in parse_requirements(fpath)])<EOL><DEDENT><DEDENT>except:<EOL><INDENT>print('<STR_LIT>'.format(fpath))<EOL><DEDENT>return [dep for dep in install_deps if dep != '<STR_LIT:None>']<EOL>

Parse all requirements files given and return a list of the dependencies


@baker.command(default=True,<EOL>shortopts={'<STR_LIT>': '<STR_LIT:c>',<EOL>'<STR_LIT>': '<STR_LIT:d>',<EOL>'<STR_LIT>': '<STR_LIT:s>',<EOL>'<STR_LIT>': '<STR_LIT:o>'})<EOL>def copy(configfile='<STR_LIT>', destpath='<STR_LIT>', overwrite=False, sub_node='<STR_LIT>'):

log.info('<STR_LIT>'.format(os.path.basename(__file__),<EOL>whoami(),<EOL>locals()))<EOL>assert(os.path.isfile(configfile))<EOL>if os.path.exists(destpath):<EOL><INDENT>if os.listdir(destpath):<EOL><INDENT>raise FolderAlreadyExists('<STR_LIT>'<EOL>'<STR_LIT>'.format(destpath))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.info('<STR_LIT>'.format(destpath))<EOL>path(destpath).makedirs_p()<EOL><DEDENT>from boyle.files.file_tree_map import FileTreeMap<EOL>file_map = FileTreeMap()<EOL>try:<EOL><INDENT>file_map.from_config_file(configfile)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise FileTreeMapError(str(e))<EOL><DEDENT>if sub_node:<EOL><INDENT>sub_map = file_map.get_node(sub_node)<EOL>if not sub_map:<EOL><INDENT>raise FileTreeMapError('<STR_LIT>'<EOL>'<STR_LIT>'.format(sub_node))<EOL><DEDENT>file_map._filetree = {}<EOL>file_map._filetree[sub_node] = sub_map<EOL><DEDENT>try:<EOL><INDENT>file_map.copy_to(destpath, overwrite=overwrite)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise FileTreeMapError(str(e))<EOL><DEDENT>

Copies the files in the built file tree map
    to despath.

    :param configfile: string
     Path to the FileTreeMap config file

    :param destpath: string
     Path to the files destination

    :param overwrite: bool
     Overwrite files if they already exist.

    :param sub_node: string
     Tree map configuration sub path.
     Will copy only the contents within this sub-node


def print_compare_idsets_one_ref(self, idset1_name, idset2_name):

try:<EOL><INDENT>idset1 = self[idset1_name]<EOL>idset2 = self[idset2_name]<EOL><DEDENT>except KeyError as ke:<EOL><INDENT>log.error('<STR_LIT>'.format(idset1_name,<EOL>idset2_name))<EOL>import sys, pdb<EOL>pdb.post_mortem(sys.exc_info()[<NUM_LIT:2>])<EOL>raise<EOL><DEDENT>assert(isinstance(idset1, idset_with_reference))<EOL>assert(isinstance(idset2, idset))<EOL>self._print_general_vs_table(idset1, idset2)<EOL>self._print_foreign_repetition_table(idset1, idset2)<EOL>

idset1_name: string
key of an idset_with_reference

idset2_name: string
key of an idset


@baker.command(name='<STR_LIT>',<EOL>params={"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": """<STR_LIT>""",<EOL>"<STR_LIT>": """<STR_LIT>"""},<EOL>shortopts={'<STR_LIT>': '<STR_LIT:i>', '<STR_LIT>': '<STR_LIT:o>', <EOL>'<STR_LIT>': '<STR_LIT:m>', '<STR_LIT>': '<STR_LIT:t>'})<EOL>def convert_sav(inputfile, outputfile=None, method='<STR_LIT>', otype='<STR_LIT>'):

assert(os.path.isfile(inputfile))<EOL>assert(method=='<STR_LIT>' or method=='<STR_LIT>')<EOL>if method == '<STR_LIT>':<EOL><INDENT>df = sav_to_pandas_rpy2(inputfile)<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>df = sav_to_pandas_savreader(inputfile)<EOL><DEDENT>otype_exts = {'<STR_LIT>': '<STR_LIT>', <EOL>'<STR_LIT>': '<STR_LIT>', <EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:html>': '<STR_LIT>'}<EOL>if outputfile is None:<EOL><INDENT>outputfile = inputfile.replace(path(inputfile).ext, '<STR_LIT>')<EOL><DEDENT>outputfile = add_extension_if_needed(outputfile, otype_exts[otype])<EOL>if otype == '<STR_LIT>':<EOL><INDENT>df.to_csv(outputfile)<EOL><DEDENT>elif otype == '<STR_LIT>':<EOL><INDENT>df.to_hdf(outputfile, os.path.basename(outputfile))<EOL><DEDENT>elif otype == '<STR_LIT>':<EOL><INDENT>df.to_stata(outputfile)<EOL><DEDENT>elif otype == '<STR_LIT>':<EOL><INDENT>df.to_json(outputfile)<EOL><DEDENT>elif otype == '<STR_LIT>':<EOL><INDENT>df.to_pickle(outputfile)<EOL><DEDENT>elif otype == '<STR_LIT>':<EOL><INDENT>df.to_excel(outputfile)<EOL><DEDENT>elif otype == '<STR_LIT:html>':<EOL><INDENT>df.to_html(outputfile)<EOL><DEDENT>else:<EOL><INDENT>df.to_csv(outputfile)<EOL><DEDENT>

Transforms the input .sav SPSS file into other format.
    If you don't specify an outputfile, it will use the
    inputfile and change its extension to .csv


try:<EOL><INDENT>b'<STR_LIT>'.decode(encoding)<EOL><DEDENT>except:<EOL><INDENT>raise ValueError('<STR_LIT>' %(str(encoding), ))<EOL><DEDENT>global defaultIREncoding<EOL>defaultIREncoding = encoding<EOL>

setDefaultIREncoding - Sets the default encoding used by IndexedRedis.
  This will be the default encoding used for field data. You can override this on a
  per-field basis by using an IRField (such as IRUnicodeField or IRRawField)

@param encoding - An encoding (like utf-8)


return IrNullBaseType.__new__(self, '<STR_LIT>')<EOL>


if self.obj is None:<EOL><INDENT>if not self.pk:<EOL><INDENT>return None<EOL><DEDENT>self.obj = self.foreignModel.objects.get(self.pk)<EOL><DEDENT>return self.obj<EOL>

getObj - Fetch (if not fetched) and return the obj associated with this data.


def __init__(self, pk=None, foreignModel=None, obj=None):

self.pk = pk<EOL>self.obj = obj<EOL>if foreignModel is not None:<EOL><INDENT>if issubclass(foreignModel.__class__, weakref.ReferenceType):<EOL><INDENT>foreignModel = foreignModel()<EOL><DEDENT>self._foreignModel = weakref.ref(foreignModel)<EOL><DEDENT>else:<EOL><INDENT>self._foreignModel = None<EOL><DEDENT>

__init__ - Create a ForeignLinkData object

@param pk <int> - The primary key of the foreign object
@param obj <None/IndexedRedisModel> - The resolved object, or None if not yet resolved


return False<EOL>

isMulti - Returns True if this is a MultiLink object (expects lists), otherwise False (expects object)

@return <bool>


return not bool(self.obj is None)<EOL>

isFetched - Check if the associated obj has been fetched or not.


return self.getObj()<EOL>


def __init__(self, name='<STR_LIT>', foreignModel=None):

IRField.__init__(self, name, valueType=int, defaultValue=irNull)<EOL>if foreignModel:<EOL><INDENT>if not isinstance(foreignModel, type):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if not hasattr(foreignModel, '<STR_LIT>'):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>self._foreignModel = weakref.ref(foreignModel)<EOL>

__init__ - Create an IRForeignLinkField. Only takes a name

@param name <str> - Field name

This field type does not support indexing.


def __init__(self, name='<STR_LIT>', defaultValue=irNull):

self.valueType = None<EOL>self.defaultValue = defaultValue<EOL>

__init__ - Create an IRPickleField

@param name <str> - Field name

@param defaultValue - The default value of this field

Because even with the same format, python2 and python3 can output different pickle strings for the same object,
  as well as different host configurations may lead to different output, this field type is not indexable.


def __init__(self, name='<STR_LIT>', decimalPlaces=<NUM_LIT:5>, defaultValue=irNull):

self.decimalPlaces = decimalPlaces<EOL>if isinstance(defaultValue, int):<EOL><INDENT>defaultValue = float(defaultValue)<EOL><DEDENT>elif isinstance(defaultValue, float):<EOL><INDENT>defaultValue = round(defaultValue, decimalPlaces)<EOL><DEDENT>self.defaultValue = defaultValue<EOL>

__init__ - Create this object.

@param name <str> - Field name (or blank if used in an IRFieldChain)

@param decimalPlaces <int> - The number of decimal places to use (precision). Values will be rounded to this many places, and always have
  this many digits after the decimal point.

@param defaultValue - The default value for this field

An IRFixedPointField is indexable, and has no option to hash the index.


if self._isIrNull(value):<EOL><INDENT>ret = IR_NULL_STR<EOL><DEDENT>else:<EOL><INDENT>ret = self._toIndex(value)<EOL><DEDENT>if self.isIndexHashed is False:<EOL><INDENT>return ret<EOL><DEDENT>return md5(tobytes(ret)).hexdigest()<EOL>

toIndex - An optional method which will return the value prepped for index.

By default, "toStorage" will be called. If you provide "hashIndex=True" on the constructor,
the field will be md5summed for indexing purposes. This is useful for large strings, etc.


if value == irNull or None:<EOL><INDENT>return IR_NULL_STR<EOL><DEDENT>return self._toStorage(value)<EOL>

toStorage - Convert the value to a string representation for storage.

  The default implementation will work here for basic types.

@param value - The value of the item to convert
@return A string value suitable for storing.


ret = []<EOL>if getattr(self, '<STR_LIT>', None) is not None:<EOL><INDENT>ret.append('<STR_LIT>' %(self.valueType.__name__, ))<EOL><DEDENT>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>ret.append('<STR_LIT>' %(self.hashIndex, ))<EOL><DEDENT>return ret<EOL>

_getReprProperties - Get the properties of this field to display in repr().

        These should be in the form of $propertyName=$propertyRepr

        The default IRField implementation handles just the "hashIndex" property.

        defaultValue is part of "__repr__" impl. You should just extend this method
        with your object's properties instead of rewriting repr.


return to_unicode(value)<EOL>

_toStorage - Convert the value to a string for storage.

The default implementation works for most valueTypes within IRField, override this for extending types.

You don't need to handle null

@param value - Value of item to convert

@return - A string value suitable for storing


return str(self)<EOL>

name - Property, return this field's name

@return <str> - Field name


return self.__class__(name=self.name, valueType=self.valueType, defaultValue=self.defaultValue, hashIndex=self.hashIndex)<EOL>

copy - Create a copy of this IRField.

  Each subclass should implement this, as you'll need to pass in the args to constructor.

@return <IRField (or subclass)> - Another IRField that has all the same values as this one.


return bool(self.hashIndex)<EOL>

isIndexHashed - Returns if the index value should be hashed

@return <bool> - True if this field should be hashed before indexing / filtering


if value == irNull:<EOL><INDENT>return irNull<EOL><DEDENT>return self._fromInput(value)<EOL>

fromInput - Convert the value from input (like assigning this through constructor or as an item assignment on the object

@param value - Value to convert

@return - Converted value


return self.valueType(value)<EOL>

_fromStorage - Convert the value from storage to the value type.

  This default impl works fine for most value types, should be implemented by extending types.

  @param value - Value to convert

  @return - Converted value


if value in IR_NULL_STRINGS:<EOL><INDENT>return irNull<EOL><DEDENT>return self._fromStorage(value)<EOL>

fromStorage - Convert the value from storage to the value type.

@param value - Value to convert

@return - The converted value


if not self.encoding:<EOL><INDENT>return getDefaultIREncoding()<EOL><DEDENT>return self.encoding<EOL>

getEncoding - Get the encoding codec associated with this field.

        If you provided None, this will return the defaultIREncoding

@return <str> - Encoding


def __init__(self, name='<STR_LIT>', defaultValue=irNull, encoding=None):

self.valueType = None<EOL>self.defaultValue = defaultValue<EOL>self.encoding = encoding<EOL>

__init__ - Create an IRBytesField object

@param name <str> - Field name

@param defaultValue <any> default irNull - Default value for this field

@param encoding <None/str> - If None, defaultIREncoding will be used when converting to bytes,
  otherwise you can provide an explicit encoding

An IRBytesField is indexable, and the index is forced to be hashed.


if len(self) == <NUM_LIT:0>:<EOL><INDENT>return []<EOL><DEDENT>ret = []<EOL>for obj in self:<EOL><INDENT>res = None<EOL>try:<EOL><INDENT>res = obj.reload()<EOL><DEDENT>except Exception as e:<EOL><INDENT>res = e<EOL><DEDENT>ret.append(res)<EOL><DEDENT>return ret<EOL>

reload - Reload all objects in this list.
        Updates in-place. To just fetch all these objects again, use "refetch"

@return - List (same order as current objects) of either exception (KeyError) if operation failed,
  or a dict of fields changed -> (old, new)


if not hasattr(mdl, '<STR_LIT>'):<EOL><INDENT>raise ValueError('<STR_LIT>' %(str(mdl.__class__.__name__),))<EOL><DEDENT>

__validate_model - Internal function to check that model is of correct type.

Uses a class variable that has been defined for IndexedRedisModel s for a long time, not the type itself, to prevent circular imports etc.

@param mdl - type to validate


if len(self) == <NUM_LIT:0>:<EOL><INDENT>return IRQueryableList()<EOL><DEDENT>mdl = self.getModel()<EOL>pks = [item._id for item in self if item._id]<EOL>return mdl.objects.getMultiple(pks)<EOL>

refetch - Fetch a fresh copy of all items in this list.
        Returns a new list. To update in-place, use "reload".

@return IRQueryableList<IndexedRedisModel> - List of fetched items


if len(self) == <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>mdl = self.getModel()<EOL>return mdl.deleter.deleteMultiple(self)<EOL>

delete - Delete all objects in this list.

@return <int> - Number of objects deleted


def _add_id_to_keys(self, pk, conn=None):

if conn is None:<EOL><INDENT>conn = self._get_connection()<EOL><DEDENT>conn.sadd(self._get_ids_key(), pk)<EOL>

_add_id_to_keys - Adds primary key to table
internal


def diff(firstObj, otherObj, includeMeta=False):

if not isIndexedRedisModel(firstObj): <EOL><INDENT>raise ValueError('<STR_LIT>' %( type(firstObj).__name__ , ) )<EOL><DEDENT>if not isIndexedRedisModel(otherObj): <EOL><INDENT>raise ValueError('<STR_LIT>' %( type(otherObj).__name__ , ) )<EOL><DEDENT>firstObj.validateModel()<EOL>otherObj.validateModel()<EOL>if getattr(firstObj, '<STR_LIT>') != getattr(otherObj, '<STR_LIT>'):<EOL><INDENT>raise ValueError('<STR_LIT>' %( firstObj.__class__, otherObj.__class__) )<EOL><DEDENT>diffFields = {}<EOL>for thisField in firstObj.FIELDS:<EOL><INDENT>thisFieldStr = str(thisField)<EOL>firstVal = object.__getattribute__( firstObj, thisFieldStr )<EOL>otherVal = object.__getattribute__( otherObj, thisFieldStr )<EOL>if firstVal != otherVal:<EOL><INDENT>diffFields[ thisFieldStr ] = ( (firstVal, otherVal) )<EOL><DEDENT><DEDENT>if includeMeta:<EOL><INDENT>firstPk = firstObj.getPk()<EOL>otherPk = otherObj.getPk()<EOL>if firstPk != otherPk:<EOL><INDENT>diffFields['<STR_LIT>'] = ( firstPk, otherPk )<EOL><DEDENT><DEDENT>return diffFields<EOL>

diff - Compare the field values on two IndexedRedisModels.

@param firstObj <IndexedRedisModel instance> - First object (or self)

@param otherObj <IndexedRedisModel instance> - Second object

@param includeMeta <bool> - If meta information (like pk) should be in the diff results.

@return <dict> - Dict of  'field' : ( value_firstObjForField, value_otherObjForField ).

        Keys are names of fields with different values.
        Value is a tuple of ( value_firstObjForField, value_otherObjForField )

Can be called statically, like: IndexedRedisModel.diff ( obj1, obj2 )

  or in reference to an obj   : obj1.diff(obj2)


return IndexedRedisSave(cls)<EOL>

saver - Get an IndexedRedisSave associated with this model


<EOL>return self.save(objs)<EOL>

saveMultiple - Save a list of objects using a pipeline.

@param objs < list<IndexedRedisModel> > - List of objects to save


global _defaultRedisConnectionParams<EOL>return copy.copy(_defaultRedisConnectionParams)<EOL>

getDefaultRedisConnectionParams - Gets A COPY OF the default Redis connection params.

@see setDefaultRedisConnectionParams for more info

@return <dict> - copy of default Redis connection parameters


if type(redisCon) == dict:<EOL><INDENT>conn = redis.Redis(**redisCon)<EOL><DEDENT>elif hasattr(conn, '<STR_LIT>') and issubclass(conn.__class__, redis.Redis):<EOL><INDENT>conn = redisCon<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>saver = self.saver<EOL>forceID = saver._getNextID(conn) <EOL>myCopy = self.copy(False)<EOL>return saver.save(myCopy, usePipeline=True, forceID=forceID, conn=conn)<EOL>

saveToExternal - Saves this object to a different Redis than that specified by REDIS_CONNECTION_PARAMS on this model.

@param redisCon <dict/redis.Redis> - Either a dict of connection params, a la REDIS_CONNECTION_PARAMS, or an existing Redis connection.
        If you are doing a lot of bulk copies, it is recommended that you create a Redis connection and pass it in rather than establish a new
        connection with each call.

@note - You will generate a new primary key relative to the external Redis environment. If you need to reference a "shared" primary key, it is better
                to use an indexed field than the internal pk.


matchedKeys = self.getPrimaryKeys()<EOL>if matchedKeys:<EOL><INDENT>return self.getMultiple(matchedKeys, cascadeFetch=cascadeFetch)<EOL><DEDENT>return IRQueryableList([], mdl=self.mdl)<EOL>

all - Get the underlying objects which match the filter criteria.

Example:   objs = Model.objects.filter(field1='value', field2='value2').all()

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

@return - Objects of the Model instance associated with this query.


oga = object.__getattribute__<EOL>if keyName not in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>fields = oga(self, '<STR_LIT>')<EOL>try:<EOL><INDENT>idx = fields.index(keyName)<EOL><DEDENT>except:<EOL><INDENT>idx = -<NUM_LIT:1><EOL><DEDENT>if idx != -<NUM_LIT:1>:<EOL><INDENT>value = fields[idx].fromInput(value)<EOL><DEDENT><DEDENT>object.__setattr__(self, keyName, value)<EOL>

__setattr__ - Will be used to set an attribute on this object.

  If the attribute is a field (in self.FIELDS), it will be converted via the field type's #fromInput method.

  Otherwise, it will just set the attribute on this object.


_id = self._id<EOL>if not _id:<EOL><INDENT>raise KeyError('<STR_LIT>')<EOL><DEDENT>currentData = self.asDict(False, forStorage=False)<EOL>newDataObj = self.objects.get(_id)<EOL>if not newDataObj:<EOL><INDENT>raise KeyError('<STR_LIT>' %(_id,))<EOL><DEDENT>newData = newDataObj.asDict(False, forStorage=False)<EOL>if currentData == newData and not self.foreignFields:<EOL><INDENT>return []<EOL><DEDENT>updatedFields = {}<EOL>for thisField, newValue in newData.items():<EOL><INDENT>defaultValue = thisField.getDefaultValue()<EOL>currentValue = currentData.get(thisField, defaultValue)<EOL>fieldIsUpdated = False<EOL>if currentValue != newValue:<EOL><INDENT>fieldIsUpdated = True<EOL><DEDENT>elif cascadeObjects is True and issubclass(thisField.__class__, IRForeignLinkFieldBase):<EOL><INDENT>if currentValue.isFetched():<EOL><INDENT>oldObjs = currentValue.getObjs()<EOL>newObjs = newValue.getObjs()<EOL>if oldObjs != newObjs: <EOL><INDENT>fieldIsUpdated = True<EOL><DEDENT>else:<EOL><INDENT>for i in range(len(oldObjs)):<EOL><INDENT>if not oldObjs[i].hasSameValues(newObjs[i], cascadeObjects=True):<EOL><INDENT>fieldIsUpdated = True<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if fieldIsUpdated is True:<EOL><INDENT>updatedFields[thisField] = ( currentValue, newValue) <EOL>setattr(self, thisField, newValue)<EOL>self._origData[thisField] = newDataObj._origData[thisField]<EOL><DEDENT><DEDENT>return updatedFields<EOL>

reload - Reload this object from the database, overriding any local changes and merging in any updates.

    @param cascadeObjects <bool> Default True. If True, foreign-linked objects will be reloaded if their values have changed
      since last save/fetch. If False, only if the pk changed will the foreign linked objects be reloaded.

    @raises KeyError - if this object has not been saved (no primary key)

    @return - Dict with the keys that were updated. Key is field name that was updated,
       and value is tuple of (old value, new value).

    NOTE: Currently, this will cause a fetch of all Foreign Link objects, one level


saver = IndexedRedisSave(self.__class__)<EOL>return saver.save(self, cascadeSave=cascadeSave)<EOL>

save - Save this object.

Will perform an "insert" if this object had not been saved before,
  otherwise will update JUST the fields changed on THIS INSTANCE of the model.

  i.e. If you have two processes fetch the same object and change different fields, they will not overwrite
  eachother, but only save the ones each process changed.

If you want to save multiple objects of type MyModel in a single transaction,
and you have those objects in a list, myObjs, you can do the following:

        MyModel.saver.save(myObjs)

@param cascadeSave <bool> Default True - If True, any Foreign models linked as attributes that have been altered
   or created will be saved with this object. If False, only this object (and the reference to an already-saved foreign model) will be saved.

@see #IndexedRedisSave.save

@return <list> - Single element list, id of saved object (if successful)


return cls.connectAlt(redisConnectionParams)<EOL>

connect - DEPRECATED NAME - @see connectAlt
  Create a class of this model which will use an alternate connection than the one specified by REDIS_CONNECTION_PARAMS on this model.

@param redisConnectionParams <dict> - Dictionary of arguments to redis.Redis, same as REDIS_CONNECTION_PARAMS.

@return - A class that can be used in all the same ways as the existing IndexedRedisModel, but that connects to a different instance.


matchedKeys = list(self.getPrimaryKeys())<EOL>obj = None<EOL>while matchedKeys and not obj:<EOL><INDENT>key = matchedKeys.pop(random.randint(<NUM_LIT:0>, len(matchedKeys)-<NUM_LIT:1>))<EOL>obj = self.get(key, cascadeFetch=cascadeFetch)<EOL><DEDENT>return obj<EOL>

Random - Returns a random record in current filterset.

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

@return - Instance of Model object, or None if no items math current filters


obj.validateModel()<EOL>if not obj.foreignFields:<EOL><INDENT>return<EOL><DEDENT>NOTE: Currently this fetches using one transaction per object. Implementation for actual resolution is in<EOL><INDENT>IndexedRedisModel.__getattribute__ <EOL><DEDENT>for foreignField in obj.foreignFields:<EOL><INDENT>subObjsData = object.__getattribute__(obj, foreignField)<EOL>if not subObjsData:<EOL><INDENT>setattr(obj, str(foreignField), irNull)<EOL>continue<EOL><DEDENT>subObjs = subObjsData.getObjs()<EOL>for subObj in subObjs:<EOL><INDENT>if isIndexedRedisModel(subObj):<EOL><INDENT>IndexedRedisQuery._doCascadeFetch(subObj)<EOL><DEDENT><DEDENT><DEDENT>

_doCascadeFetch - Takes an object and performs a cascading fetch on all foreign links, and all theirs, and so on.

@param obj <IndexedRedisModel> - A fetched model


copyNum = _modelCopyMap[mdl]<EOL>_modelCopyMap[mdl] += <NUM_LIT:1><EOL>mdlCopy = type(mdl.__name__ + '<STR_LIT>' + str(copyNum), mdl.__bases__, copy.deepcopy(dict(mdl.__dict__)))<EOL>mdlCopy.FIELDS = [field.copy() for field in mdl.FIELDS]<EOL>mdlCopy.INDEXED_FIELDS = [str(idxField) for idxField in mdl.INDEXED_FIELDS] <EOL>mdlCopy.validateModel()<EOL>return mdlCopy<EOL>

copyModel - Copy this model, and return that copy.

  The copied model will have all the same data, but will have a fresh instance of the FIELDS array and all members,
    and the INDEXED_FIELDS array.

  This is useful for converting, like changing field types or whatever, where you can load from one model and save into the other.

@return <IndexedRedisModel> - A copy class of this model class with a unique name.


def getMultipleOnlyFields(self, pks, fields, cascadeFetch=False):

if type(pks) == set:<EOL><INDENT>pks = list(pks)<EOL><DEDENT>if len(pks) == <NUM_LIT:1>:<EOL><INDENT>return IRQueryableList([self.getOnlyFields(pks[<NUM_LIT:0>], fields, cascadeFetch=cascadeFetch)], mdl=self.mdl)<EOL><DEDENT>conn = self._get_connection()<EOL>pipeline = conn.pipeline()<EOL>for pk in pks:<EOL><INDENT>key = self._get_key_for_id(pk)<EOL>pipeline.hmget(key, fields)<EOL><DEDENT>res = pipeline.execute()<EOL>ret = IRQueryableList(mdl=self.mdl)<EOL>pksLen = len(pks)<EOL>i = <NUM_LIT:0><EOL>numFields = len(fields)<EOL>while i < pksLen:<EOL><INDENT>objDict = {}<EOL>anyNotNone = False<EOL>thisRes = res[i]<EOL>if thisRes is None or type(thisRes) != list:<EOL><INDENT>ret.append(None)<EOL>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>j = <NUM_LIT:0><EOL>while j < numFields:<EOL><INDENT>objDict[fields[j]] = thisRes[j]<EOL>if thisRes[j] != None:<EOL><INDENT>anyNotNone = True<EOL><DEDENT>j += <NUM_LIT:1><EOL><DEDENT>if anyNotNone is False:<EOL><INDENT>ret.append(None)<EOL>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>objDict['<STR_LIT>'] = pks[i]<EOL>obj = self._redisResultToObj(objDict)<EOL>ret.append(obj)<EOL>i += <NUM_LIT:1><EOL><DEDENT>if cascadeFetch is True:<EOL><INDENT>for obj in ret:<EOL><INDENT>self._doCascadeFetch(obj)<EOL><DEDENT><DEDENT>return ret<EOL>

getMultipleOnlyFields - Gets only certain fields from a list of  primary keys. For working on entire filter set, see allOnlyFields

@param pks list<str> - Primary Keys

@param fields list<str> - List of fields

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

return - List of partial objects with only fields applied


deleter = IndexedRedisDelete(self.__class__)<EOL>return deleter.deleteOne(self)<EOL>


def save(self, obj, usePipeline=True, forceID=False, cascadeSave=True, conn=None):

if conn is None:<EOL><INDENT>conn = self._get_connection()<EOL><DEDENT>if usePipeline is True:<EOL><INDENT>idConn = conn<EOL><DEDENT>else:<EOL><INDENT>idConn = self._get_new_connection()<EOL><DEDENT>if issubclass(obj.__class__, (list, tuple)):<EOL><INDENT>objs = obj<EOL><DEDENT>else:<EOL><INDENT>objs = [obj]<EOL><DEDENT>if usePipeline is True:<EOL><INDENT>pipeline = conn.pipeline()<EOL><DEDENT>else:<EOL><INDENT>pipeline = conn<EOL><DEDENT>oga = object.__getattribute__<EOL>if cascadeSave is True:<EOL>ignPipelines = OrderedDict()<EOL><INDENT>foreignSavers = {}<EOL>for thisObj in objs:<EOL><INDENT>if not thisObj.foreignFields:<EOL><INDENT>continue<EOL><DEDENT>foreignFields = thisObj.foreignFields<EOL>for foreignField in foreignFields:<EOL><INDENT>rawObj = oga(thisObj, str(foreignField))<EOL>if rawObj in (None, irNull) or not rawObj.isFetched():<EOL><INDENT>continue<EOL><DEDENT>foreignObjects = oga(thisObj, str(foreignField)).getObjs()<EOL>for foreignObject in foreignObjects:<EOL><INDENT>doSaveForeign = False<EOL>if getattr(foreignObject, '<STR_LIT>', None):<EOL><INDENT>if foreignObject.hasUnsavedChanges(cascadeObjects=True):<EOL><INDENT>doSaveForeign = True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>doSaveForeign = True<EOL><DEDENT>if doSaveForeign is True:<EOL><INDENT>if foreignField not in foreignSavers:<EOL><DEDENT><DEDENT><DEDENT>foreignPipelines[foreignField] = self._get_new_connection().pipeline()<EOL><INDENT>foreignSavers[foreignField] = IndexedRedisSave(foreignObject.__class__)<EOL>

save - Save an object / objects associated with this model.

You probably want to just do object.save() instead of this, but to save multiple objects at once in a single transaction,
   you can use:

        MyModel.saver.save(myObjs)

@param obj <IndexedRedisModel or list<IndexedRedisModel> - The object to save, or a list of objects to save

@param usePipeline - Use a pipeline for saving. You should always want this, unless you are calling this function from within an existing pipeline.

@param forceID - if not False, force ID to this. If obj is list, this is also list. Forcing IDs also forces insert. Up to you to ensure ID will not clash.
@param cascadeSave <bool> Default True - If True, any Foreign models linked as attributes that have been altered
   or created will be saved with this object. If False, only this object (and the reference to an already-saved foreign model) will be saved.

@param conn - A connection or None

@note - if no ID is specified

@return - List of pks


if not isinstance(redisConnectionParams, dict):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>hashVal = hashDictOneLevel(redisConnectionParams)<EOL>modelDictCopy = copy.deepcopy(dict(cls.__dict__))<EOL>modelDictCopy['<STR_LIT>'] = redisConnectionParams<EOL>ConnectedIndexedRedisModel = type('<STR_LIT>' + cls.__name__ + str(hashVal), cls.__bases__, modelDictCopy)<EOL>return ConnectedIndexedRedisModel<EOL>

connectAlt - Create a class of this model which will use an alternate connection than the one specified by REDIS_CONNECTION_PARAMS on this model.

@param redisConnectionParams <dict> - Dictionary of arguments to redis.Redis, same as REDIS_CONNECTION_PARAMS.

@return - A class that can be used in all the same ways as the existing IndexedRedisModel, but that connects to a different instance.

  The fields and key will be the same here, but the connection will be different. use #copyModel if you want an independent class for the model


<EOL>if type(self) != type(other):<EOL><INDENT>return False<EOL><DEDENT>if not self.hasSameValues(other):<EOL><INDENT>return False<EOL><DEDENT>if getattr(self, '<STR_LIT>', None) != getattr(other, '<STR_LIT>', None):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

__eq__ - Check if two IndexedRedisModels are equal.

They are equal if they have the same type and same field values (including id).

To check if two models have the same values (but can have different ids), use #hasSameValues method.


myDict = self.asDict(True, forStorage=False, strKeys=True)<EOL>myClassName = self.__class__.__name__<EOL>ret = [myClassName, '<STR_LIT:(>']<EOL>_id = myDict.pop('<STR_LIT>', '<STR_LIT>')<EOL>if _id:<EOL><INDENT>ret += ['<STR_LIT>', to_unicode(_id), '<STR_LIT>']<EOL><DEDENT>key = None<EOL>for key, value in myDict.items():<EOL><INDENT>ret += [key, '<STR_LIT:=>', repr(value), '<STR_LIT:U+002CU+0020>']<EOL><DEDENT>if key is not None or not _id:<EOL><INDENT>ret.pop()<EOL><DEDENT>ret.append('<STR_LIT:)>')<EOL>return '<STR_LIT>'.join(ret)<EOL>

__repr__ - Returns a string of the constructor/params to recreate this object.
    Example: objCopy = eval(repr(obj))

    @return - String of python init call to recreate this object


def _rem_id_from_index(self, indexedField, pk, val, conn=None):

if conn is None:<EOL><INDENT>conn = self._get_connection()<EOL><DEDENT>conn.srem(self._get_key_for_index(indexedField, val), pk)<EOL>

_rem_id_from_index - Removes an id from an index
internal


return IndexedRedisQuery._filter(self, **kwargs)<EOL>

filterInline - @see IndexedRedisQuery.filter. This is the same as filter, but works inline on this object instead of creating a copy.
        Use this is you do not need to retain the previous filter object.


return self._id<EOL>

getPk - Gets the internal primary key associated with this object


if self._connection is None:<EOL><INDENT>self._connection = self._get_new_connection() <EOL><DEDENT>return self._connection<EOL>

_get_connection - Maybe get a new connection, or reuse if passed in.
        Will share a connection with a model
internal


def asDict(self, includeMeta=False, forStorage=False, strKeys=False):

ret = {}<EOL>for thisField in self.FIELDS:<EOL><INDENT>val = object.__getattribute__(self, thisField)<EOL>if forStorage is True:<EOL><INDENT>val = thisField.toStorage(val)<EOL><DEDENT>if strKeys:<EOL><INDENT>ret[str(thisField)] = val<EOL><DEDENT>else:<EOL><INDENT>ret[thisField] = val<EOL><DEDENT><DEDENT>if includeMeta is True:<EOL><INDENT>ret['<STR_LIT>'] = getattr(self, '<STR_LIT>', None)<EOL><DEDENT>return ret<EOL>

toDict / asDict - Get a dictionary representation of this model.

@param includeMeta - Include metadata in return. For now, this is only pk stored as "_id"

@param convertValueTypes <bool> - default True. If False, fields with fieldValue defined will be converted to that type.
        Use True when saving, etc, as native type is always either str or bytes.

@param strKeys <bool> Default False - If True, just the string value of the field name will be used as the key.
        Otherwise, the IRField itself will be (although represented and indexed by string)

@return - Dictionary reprensetation of this object and all fields


self.__class__.validateModel()<EOL>for key, value in stateDict.items():<EOL><INDENT>setattr(self, key, value)<EOL><DEDENT>self._origData = stateDict['<STR_LIT>']<EOL>


pool = getRedisPool(self.mdl.REDIS_CONNECTION_PARAMS)<EOL>return redis.Redis(connection_pool=pool)<EOL>

_get_new_connection - Get a new connection
internal


matchedKeys = self.getPrimaryKeys(sortByAge=True)<EOL>if matchedKeys:<EOL><INDENT>return self.getMultiple(matchedKeys, cascadeFetch=cascadeFetch)<EOL><DEDENT>return IRQueryableList([], mdl=self.mdl)<EOL>

allByAge - Get the underlying objects which match the filter criteria, ordered oldest -> newest
        If you are doing a queue or just need the head/tail, consider .first() and .last() instead.

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

@return - Objects of the Model instance associated with this query, sorted oldest->newest


mdl.validateModel()<EOL>self.mdl = mdl<EOL>self.keyName = mdl.KEY_NAME<EOL>fields = mdl.FIELDS<EOL>self.fields = mdl.FIELDS<EOL>self.indexedFields = [fields[fieldName] for fieldName in mdl.INDEXED_FIELDS]<EOL>self._connection = None<EOL>

Internal constructor

@param mdl - IndexedRedisModel implementer


def hasSameValues(self, other, cascadeObject=True):

if self.FIELDS != other.FIELDS:<EOL><INDENT>return False<EOL><DEDENT>oga = object.__getattribute__<EOL>for field in self.FIELDS:<EOL><INDENT>thisVal = oga(self, field)<EOL>otherVal = oga(other, field)<EOL>if thisVal != otherVal:<EOL><INDENT>return False<EOL><DEDENT>if cascadeObject is True and issubclass(field.__class__, IRForeignLinkFieldBase):<EOL><INDENT>if thisVal and thisVal.isFetched():<EOL><INDENT>if otherVal and otherVal.isFetched():<EOL><INDENT>theseForeign = thisVal.getObjs()<EOL>othersForeign = otherVal.getObjs()<EOL>for i in range(len(theseForeign)):<EOL><INDENT>if not theseForeign[i].hasSameValues(othersForeign[i]):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>theseForeign = thisVal.getObjs()<EOL>for i in range(len(theseForeign)):<EOL><INDENT>if theseForeign[i].hasUnsavedChanges(cascadeObjects=True):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if otherVal and otherVal.isFetched():<EOL><INDENT>othersForeign = otherVal.getObjs()<EOL>for i in range(len(othersForeign)):<EOL><INDENT>if othersForeign[i].hasUnsavedChanges(cascadeObjects=True):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return True<EOL>

hasSameValues - Check if this and another model have the same fields and values.

This does NOT include id, so the models can have the same values but be different objects in the database.

@param other <IndexedRedisModel> - Another model

@param cascadeObject <bool> default True - If True, foreign link values with changes will be considered a difference.
        Otherwise, only the immediate values are checked.

@return <bool> - True if all fields have the same value, otherwise False


conn = self._get_connection()<EOL>numFilters = len(self.filters)<EOL>numNotFilters = len(self.notFilters)<EOL>if numFilters + numNotFilters == <NUM_LIT:0>:<EOL><INDENT>return conn.scard(self._get_ids_key())<EOL><DEDENT>if numNotFilters == <NUM_LIT:0>:<EOL><INDENT>if numFilters == <NUM_LIT:1>:<EOL><INDENT>(filterFieldName, filterValue) = self.filters[<NUM_LIT:0>]<EOL>return conn.scard(self._get_key_for_index(filterFieldName, filterValue))<EOL><DEDENT>indexKeys = [self._get_key_for_index(filterFieldName, filterValue) for filterFieldName, filterValue in self.filters]<EOL>return len(conn.sinter(indexKeys))<EOL><DEDENT>notIndexKeys = [self._get_key_for_index(filterFieldName, filterValue) for filterFieldName, filterValue in self.notFilters]<EOL>if numFilters == <NUM_LIT:0>:<EOL><INDENT>return len(conn.sdiff(self._get_ids_key(), *notIndexKeys))<EOL><DEDENT>indexKeys = [self._get_key_for_index(filterFieldName, filterValue) for filterFieldName, filterValue in self.filters]<EOL>tempKey = self._getTempKey()<EOL>pipeline = conn.pipeline()<EOL>pipeline.sinterstore(tempKey, *indexKeys)<EOL>pipeline.sdiff(tempKey, *notIndexKeys)<EOL>pipeline.delete(tempKey)<EOL>pks = pipeline.execute()[<NUM_LIT:1>] <EOL>return len(pks)<EOL>

count - gets the number of records matching the filter criteria

Example:
        theCount = Model.objects.filter(field1='value').count()


def setDefaultRedisConnectionParams( connectionParams ):

global _defaultRedisConnectionParams<EOL>_defaultRedisConnectionParams.clear()<EOL>for key, value in connectionParams.items():<EOL><INDENT>_defaultRedisConnectionParams[key] = value<EOL><DEDENT>clearRedisPools()<EOL>

setDefaultRedisConnectionParams - Sets the default parameters used when connecting to Redis.

  This should be the args to redis.Redis in dict (kwargs) form.

  @param connectionParams <dict> - A dict of connection parameters.
    Common keys are:

       host <str> - hostname/ip of Redis server (default '127.0.0.1')
       port <int> - Port number			(default 6379)
       db  <int>  - Redis DB number		(default 0)

   Omitting any of those keys will ensure the default value listed is used.

  This connection info will be used by default for all connections to Redis, unless explicitly set otherwise.
  The common way to override is to define REDIS_CONNECTION_PARAMS on a model, or use AltConnectedModel = MyModel.connectAlt( PARAMS )

  Any omitted fields in these connection overrides will inherit the value from the global default.

  For example, if your global default connection params define host = 'example.com', port=15000, and db=0,
    and then one of your models has

       REDIS_CONNECTION_PARAMS = { 'db' : 1 }

    as an attribute, then that model's connection will inherit host='example.com" and port=15000 but override db and use db=1

    NOTE: Calling this function will clear the connection_pool attribute of all stored managed connections, disconnect all managed connections,
      and close-out the connection pool.
     It may not be safe to call this function while other threads are potentially hitting Redis (not that it would make sense anyway...)

     @see clearRedisPools   for more info


IndexedRedisQuery._doCascadeFetch(self)<EOL>

cascadeFetch - Immediately fetch all foreign links on this field, and all their links, etc.

  Normally, this would be done on access of the foreign members, or at .all() time by passing cascadeFetch=True into
   the fetch function

   e.x.    MyModel.objects.filter(...).all(cascadeFetch=True)


def allOnlyFields(self, fields, cascadeFetch=False):

matchedKeys = self.getPrimaryKeys()<EOL>if matchedKeys:<EOL><INDENT>return self.getMultipleOnlyFields(matchedKeys, fields, cascadeFetch=cascadeFetch)<EOL><DEDENT>return IRQueryableList([], mdl=self.mdl)<EOL>

allOnlyFields - Get the objects which match the filter criteria, only fetching given fields.

@param fields - List of fields to fetch

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

@return - Partial objects with only the given fields fetched


if model == IndexedRedisModel:<EOL><INDENT>import re<EOL>if re.match('<STR_LIT>', sys.argv[<NUM_LIT:0>]):<EOL><INDENT>return<EOL><DEDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>global validatedModels<EOL>keyName = model.KEY_NAME<EOL>if not keyName:<EOL><INDENT>raise InvalidModelException('<STR_LIT>' %(str(model.__name__), ) )<EOL><DEDENT>if model in validatedModels:<EOL><INDENT>return True<EOL><DEDENT>failedValidationStr = '<STR_LIT>' %(str(model.__name__), ) <EOL>fieldSet = set(model.FIELDS)<EOL>indexedFieldSet = set(model.INDEXED_FIELDS)<EOL>if not fieldSet:<EOL><INDENT>raise InvalidModelException('<STR_LIT>' %(failedValidationStr,))<EOL><DEDENT>if hasattr(model, '<STR_LIT>'):<EOL><INDENT>raise InvalidModelException('<STR_LIT>')<EOL><DEDENT>if hasattr(model, '<STR_LIT>'):<EOL><INDENT>raise InvalidModelException('<STR_LIT>')<EOL><DEDENT>newFields = []<EOL>updatedFields = []<EOL>mustUpdateFields = False<EOL>foreignFields = []<EOL>for thisField in fieldSet:<EOL><INDENT>if thisField == '<STR_LIT>':<EOL><INDENT>raise InvalidModelException('<STR_LIT>' %(failedValidationStr,))<EOL><DEDENT>try:<EOL><INDENT>codecs.ascii_encode(thisField)<EOL><DEDENT>except UnicodeDecodeError as e:<EOL><INDENT>raise InvalidModelException('<STR_LIT>' %(failedValidationStr, to_unicode(thisField), str(e)))<EOL><DEDENT>if issubclass(thisField.__class__, IRForeignLinkFieldBase):<EOL><INDENT>foreignFields.append(thisField)<EOL><DEDENT>if issubclass(thisField.__class__, IRField):<EOL><INDENT>newFields.append(thisField)<EOL><DEDENT>else:<EOL><INDENT>mustUpdateFields = True<EOL>newField = IRClassicField(thisField)<EOL>newFields.append(newField)<EOL>updatedFields.append(thisField)<EOL>thisField = newField<EOL><DEDENT>if str(thisField) == '<STR_LIT>':<EOL><INDENT>raise InvalidModelException('<STR_LIT>' %(failedValidationStr, str(type(thisField)), repr(thisField)   ) )<EOL><DEDENT>if thisField in indexedFieldSet and thisField.CAN_INDEX is False:<EOL><INDENT>raise InvalidModelException('<STR_LIT>' %(failedValidationStr, str(thisField.__class__.__name__), repr(thisField)))<EOL><DEDENT>if hasattr(IndexedRedisModel, thisField) is True:<EOL><INDENT>raise InvalidModelException('<STR_LIT>' %(failedValidationStr, str(thisField)))<EOL><DEDENT><DEDENT>if mustUpdateFields is True:<EOL><INDENT>model.FIELDS = newFields<EOL>deprecatedMessage('<STR_LIT>' %(model.__name__, repr(updatedFields)), '<STR_LIT>' + model.__name__)<EOL><DEDENT>model.FIELDS = KeyList(model.FIELDS)<EOL>if bool(indexedFieldSet - fieldSet):<EOL><INDENT>raise InvalidModelException('<STR_LIT>' %(failedValidationStr, str(list(indexedFieldSet - fieldSet)), ) )<EOL><DEDENT>model.foreignFields = foreignFields<EOL>validatedModels.add(model)<EOL>return True<EOL>

validateModel - Class method that validates a given model is implemented correctly. Will only be validated once, on first model instantiation.

@param model - Implicit of own class

@return - True

@raises - InvalidModelException if there is a problem with the model, and the message contains relevant information.


obj = None<EOL>matchedKeys = self.getPrimaryKeys(sortByAge=True)<EOL>if matchedKeys:<EOL><INDENT>while matchedKeys and obj is None:<EOL><INDENT>obj = self.get(matchedKeys.pop(), cascadeFetch=cascadeFetch)<EOL><DEDENT><DEDENT>return obj<EOL>

Last - Returns the newest record (highest primary key) with current filters.
        This makes an efficient queue, as it only fetches a single object.

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

@return - Instance of Model object, or None if no items match current filters


def _rem_id_from_keys(self, pk, conn=None):

if conn is None:<EOL><INDENT>conn = self._get_connection()<EOL><DEDENT>conn.srem(self._get_ids_key(), pk)<EOL>

_rem_id_from_keys - Remove primary key from table
internal


global RedisPools<EOL>global _defaultRedisConnectionParams<EOL>global _redisManagedConnectionParams<EOL>if not params:<EOL><INDENT>params = _defaultRedisConnectionParams<EOL>isDefaultParams = True<EOL><DEDENT>else:<EOL><INDENT>isDefaultParams = bool(params is _defaultRedisConnectionParams)<EOL><DEDENT>if '<STR_LIT>' in params:<EOL><INDENT>return params['<STR_LIT>']<EOL><DEDENT>hashValue = hashDictOneLevel(params)<EOL>if hashValue in RedisPools:<EOL><INDENT>params['<STR_LIT>'] = RedisPools[hashValue]<EOL>return RedisPools[hashValue]<EOL><DEDENT>if not isDefaultParams:<EOL><INDENT>origParams = params<EOL>params = copy.copy(params)<EOL><DEDENT>else:<EOL><INDENT>origParams = params<EOL><DEDENT>checkAgain = False<EOL>if '<STR_LIT:host>' not in params:<EOL><INDENT>if not isDefaultParams and '<STR_LIT:host>' in _defaultRedisConnectionParams:<EOL><INDENT>params['<STR_LIT:host>'] = _defaultRedisConnectionParams['<STR_LIT:host>']<EOL><DEDENT>else:<EOL><INDENT>params['<STR_LIT:host>'] = '<STR_LIT:127.0.0.1>'<EOL><DEDENT>checkAgain = True<EOL><DEDENT>if '<STR_LIT:port>' not in params:<EOL><INDENT>if not isDefaultParams and '<STR_LIT:port>' in _defaultRedisConnectionParams:<EOL><INDENT>params['<STR_LIT:port>'] = _defaultRedisConnectionParams['<STR_LIT:port>']<EOL><DEDENT>else:<EOL><INDENT>params['<STR_LIT:port>'] = <NUM_LIT><EOL><DEDENT>checkAgain = True<EOL><DEDENT>if '<STR_LIT>' not in params:<EOL><INDENT>if not isDefaultParams and '<STR_LIT>' in _defaultRedisConnectionParams:<EOL><INDENT>params['<STR_LIT>'] = _defaultRedisConnectionParams['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>params['<STR_LIT>'] = <NUM_LIT:0><EOL><DEDENT>checkAgain = True<EOL><DEDENT>if not isDefaultParams:<EOL><INDENT>otherGlobalKeys = set(_defaultRedisConnectionParams.keys()) - set(params.keys())<EOL>for otherKey in otherGlobalKeys:<EOL><INDENT>if otherKey == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>params[otherKey] = _defaultRedisConnectionParams[otherKey]<EOL>checkAgain = True<EOL><DEDENT><DEDENT>if checkAgain:<EOL><INDENT>hashValue = hashDictOneLevel(params)<EOL>if hashValue in RedisPools:<EOL><INDENT>params['<STR_LIT>'] = RedisPools[hashValue]<EOL>return RedisPools[hashValue]<EOL><DEDENT><DEDENT>connectionPool = redis.ConnectionPool(**params)<EOL>origParams['<STR_LIT>'] = params['<STR_LIT>'] = connectionPool<EOL>RedisPools[hashValue] = connectionPool<EOL>origParamsHash = hashDictOneLevel(origParams)<EOL>if origParamsHash not in _redisManagedConnectionParams:<EOL><INDENT>_redisManagedConnectionParams[origParamsHash] = [origParams]<EOL><DEDENT>elif origParams not in _redisManagedConnectionParams[origParamsHash]:<EOL><INDENT>_redisManagedConnectionParams[origParamsHash].append(origParams)<EOL><DEDENT>return connectionPool<EOL>

getRedisPool - Returns and possibly also creates a Redis connection pool
        based on the REDIS_CONNECTION_PARAMS passed in.

        The goal of this method is to keep a small connection pool rolling
        to each unique Redis instance, otherwise during network issues etc
        python-redis will leak connections and in short-order can exhaust
        all the ports on a system. There's probably also some minor
        performance gain in sharing Pools.

        Will modify "params", if "host" and/or "port" are missing, will fill
        them in with defaults, and prior to return will set "connection_pool"
        on params, which will allow immediate return on the next call,
        and allow access to the pool directly from the model object.

        @param params <dict> - REDIS_CONNECTION_PARAMS - kwargs to redis.Redis

        @return redis.ConnectionPool corrosponding to this unique server.


def getOnlyFields(self, pk, fields, cascadeFetch=False):

conn = self._get_connection()<EOL>key = self._get_key_for_id(pk)<EOL>res = conn.hmget(key, fields)<EOL>if type(res) != list or not len(res):<EOL><INDENT>return None<EOL><DEDENT>objDict = {}<EOL>numFields = len(fields)<EOL>i = <NUM_LIT:0><EOL>anyNotNone = False<EOL>while i < numFields:<EOL><INDENT>objDict[fields[i]] = res[i]<EOL>if res[i] != None:<EOL><INDENT>anyNotNone = True<EOL><DEDENT>i += <NUM_LIT:1><EOL><DEDENT>if anyNotNone is False:<EOL><INDENT>return None<EOL><DEDENT>objDict['<STR_LIT>'] = pk<EOL>ret = self._redisResultToObj(objDict)<EOL>if cascadeFetch is True:<EOL><INDENT>self._doCascadeFetch(ret)<EOL><DEDENT>return ret<EOL>

getOnlyFields - Gets only certain fields from a paticular primary key. For working on entire filter set, see allOnlyFields

@param pk <int> - Primary Key

@param fields list<str> - List of fields

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

return - Partial objects with only fields applied


pprint.pprint(self.asDict(includeMeta=True, forStorage=False, strKeys=True), stream=stream)<EOL>

pprint - Pretty-print a dict representation of this object.

@param stream <file/None> - Either a stream to output, or None to default to sys.stdout


def _get_key_for_index(self, indexedField, val):

<EOL>if hasattr(indexedField, '<STR_LIT>'):<EOL><INDENT>val = indexedField.toIndex(val)<EOL><DEDENT>else:<EOL><INDENT>val = self.fields[indexedField].toIndex(val)<EOL><DEDENT>return '<STR_LIT>'.join( [INDEXED_REDIS_PREFIX, self.keyName, '<STR_LIT>', indexedField, '<STR_LIT::>', val] )<EOL>

_get_key_for_index - Returns the key name that would hold the indexes on a value
Internal - does not validate that indexedFields is actually indexed. Trusts you. Don't let it down.

@param indexedField - string of field name
@param val - Value of field

@return - Key name string, potentially hashed.


myData = self.asDict(True, forStorage=False)<EOL>myData['<STR_LIT>'] = self._origData<EOL>return myData<EOL>


def deprecatedMessage(msg, key=None, printStack=False):

if __deprecatedMessagesEnabled is False:<EOL><INDENT>return<EOL><DEDENT>if not _alreadyWarned:<EOL><INDENT>sys.stderr.write('<STR_LIT>')<EOL><DEDENT>if key is None:<EOL><INDENT>from .compat_str import tobytes<EOL>key = md5(tobytes(msg)).hexdigest()<EOL><DEDENT>if key not in _alreadyWarned:<EOL><INDENT>_alreadyWarned[key] = True<EOL>sys.stderr.write('<STR_LIT>' %(msg, ))<EOL>if printStack:<EOL><INDENT>sys.stderr.write('<STR_LIT>')<EOL>curStack = traceback.extract_stack()<EOL>sys.stderr.write('<STR_LIT:U+0020>' + '<STR_LIT>'.join(traceback.format_list(curStack[:-<NUM_LIT:2>])).replace('<STR_LIT:\t>', '<STR_LIT:U+0020>') + '<STR_LIT:\n>')<EOL><DEDENT><DEDENT>

deprecatedMessage - Print a deprecated messsage (unless they are toggled off). Will print a message only once (based on "key")

@param msg <str> - Deprecated message to possibly print

@param key <anything> - A key that is specific to this message.
        If None is provided (default), one will be generated from the md5 of the message.
        However, better to save cycles and provide a unique key if at all possible.
        The decorator uses the function itself as the key.

@param printStack <bool> Default False, if True print a stack trace


keys = [str(x) for x in myDict.keys()]<EOL>keys.sort()<EOL>lst = []<EOL>for key in keys:<EOL><INDENT>lst.append(str(myDict[key]) + '<STR_LIT>')<EOL><DEDENT>return '<STR_LIT>'.join(lst).__hash__()<EOL>

A function which can generate a hash of a one-level
  dict containing strings (like REDIS_CONNECTION_PARAMS)

@param myDict <dict> - Dict with string keys and values

@return <long> - Hash of myDict


def raw_from_delimited(msgs: DelimitedMsg) -> RawMsgs:

delim = _rindex(msgs, b'<STR_LIT>')<EOL>return tuple(msgs[:delim]), tuple(msgs[delim + <NUM_LIT:1>:])<EOL>

\
    From a message consisting of header frames, delimiter frame, and payload frames, return a tuple `(header, payload)`.
    The payload frames may be returned as sequences of bytes (raw) or as `Message`s.


def parse(self, data: RawMessage) -> Message:

try:<EOL><INDENT>return self.receiver.parse(data)<EOL><DEDENT>except KeyError as err:<EOL><INDENT>raise UnknownCommandError from err<EOL><DEDENT>except DecodeError as err:<EOL><INDENT>raise UnknownCommandError(f"<STR_LIT>") from err<EOL><DEDENT>

\
        Parses a binary protobuf message into a Message object.


async def get_ltd_product(session, slug=None, url=None):

if url is None:<EOL><INDENT>url = '<STR_LIT>'.format(slug)<EOL><DEDENT>async with session.get(url) as response:<EOL><INDENT>data = await response.json()<EOL><DEDENT>return data<EOL>

Get the product resource (JSON document) from the LSST the Docs API.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    slug : `str`, optional
        Slug identfying the product. This is the same as the subdomain.
        For example, ``'ldm-151'`` is the slug for ``ldm-151.lsst.io``.
        A full product URL can be provided instead, see ``url``.
    url : `str`, optional
        The full LTD Keeper URL for the product resource. For example,
        ``'https://keeper.lsst.codes/products/ldm-151'``. The ``slug``
        can be provided instead.

    Returns
    -------
    product : `dict`
        Product dataset. See
        https://ltd-keeper.lsst.io/products.html#get--products-(slug)
        for fields.


product_url = '<STR_LIT>'<EOL>async with session.get(product_url) as response:<EOL><INDENT>data = await response.json()<EOL><DEDENT>return data['<STR_LIT>']<EOL>

Get URLs for LSST the Docs (LTD) products from the LTD Keeper API.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.

    Returns
    -------
    product_urls : `list`
        List of product URLs.


logger = logging.getLogger(__name__)<EOL>@functools.wraps(func)<EOL>def _install_and_run(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>result = func(*args, **kwargs)<EOL><DEDENT>except OSError:<EOL><INDENT>message = "<STR_LIT>"<EOL>logger.warning(message)<EOL>pypandoc.download_pandoc(version='<STR_LIT>')<EOL>logger.debug("<STR_LIT>")<EOL>result = func(*args, **kwargs)<EOL><DEDENT>return result<EOL><DEDENT>return _install_and_run<EOL>

Decorate a function that uses pypandoc to ensure that pandoc is
    installed if necessary.


for linker in self._linkers:<EOL><INDENT>tex_source = linker(tex_source)<EOL><DEDENT>return tex_source<EOL>

r"""Convert citations in LaTeX source to Hyperref links.

        Parameters
        ----------
        tex_source : `str`
            LaTeX document source.

        Returns
        -------
        processed_tex : `str`
            LaTeX document source with all citation commands converted to
            ``\hyperref`` commands.


<EOL>return re.sub(r'<STR_LIT>', r'<STR_LIT>', tex_source, flags=re.M)<EOL>

Delete latex comments from TeX source.

    Parameters
    ----------
    tex_source : str
        TeX source content.

    Returns
    -------
    tex_source : str
        TeX source without comments.


<EOL>return re.sub(r'<STR_LIT>', '<STR_LIT>', tex_source, flags=re.M)<EOL>

Delete trailing whitespace from TeX source.

    Parameters
    ----------
    tex_source : str
        TeX source content.

    Returns
    -------
    tex_source : str
        TeX source without trailing whitespace.


logger = logging.getLogger(__name__)<EOL>def _sub_line(match):<EOL><INDENT>"""<STR_LIT>"""<EOL>fname = match.group('<STR_LIT:filename>')<EOL>if not fname.endswith('<STR_LIT>'):<EOL><INDENT>full_fname = "<STR_LIT:.>".join((fname, '<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>full_fname = fname<EOL><DEDENT>full_path = os.path.abspath(os.path.join(root_dir, full_fname))<EOL>try:<EOL><INDENT>included_source = read_tex_file(full_path, root_dir=root_dir)<EOL><DEDENT>except IOError:<EOL><INDENT>logger.error("<STR_LIT>".format(full_path))<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>return included_source<EOL><DEDENT><DEDENT>tex_source = input_include_pattern.sub(_sub_line, tex_source)<EOL>return tex_source<EOL>

r"""Insert referenced TeX file contents (from  ``\input`` and ``\include``
    commands) into the source.

    Parameters
    ----------
    tex_source : `str`
        TeX source where referenced source files will be found and inserted.
    root_dir : `str`, optional
        Name of the directory containing the TeX project's root file. Files
        referenced by TeX ``\input`` and ``\include`` commands are relative to
        this directory. If not set, the current working directory is assumed.

    Returns
    -------
    tex_source : `str`
        TeX source.

    See also
    --------
    `read_tex_file`
        Recommended API for reading a root TeX source file and inserting
        referenced files.


command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._handle = None<EOL>self._series = None<EOL>self._serial = None<EOL>return<EOL><DEDENT>self._handle = parsed['<STR_LIT>']<EOL>try:<EOL><INDENT>self._series, self._serial = self._handle.split('<STR_LIT:->', <NUM_LIT:1>)<EOL><DEDENT>except ValueError:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>', self._handle)<EOL>self._series = None<EOL>self._serial = None<EOL><DEDENT>

Parse the document handle.

        Sets the ``_series``, ``_serial``, and ``_handle`` attributes.


command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._abstract = None<EOL>return<EOL><DEDENT>try:<EOL><INDENT>content = parsed['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._abstract = None<EOL>return<EOL><DEDENT>content = content.strip()<EOL>self._abstract = content<EOL>

Parse the abstract from the TeX source.

        Sets the ``_abstract`` attribute.


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_title()<EOL><DEDENT>return self._title<EOL>


return self.format_abstract(format='<STR_LIT>', deparagraph=False,<EOL>mathjax=False, smart=True)<EOL>

HTML5-formatted document abstract (`str`).


doc_datetime = None<EOL>if not self.is_draft:<EOL><INDENT>date_command = LatexCommand(<EOL>'<STR_LIT:date>',<EOL>{'<STR_LIT:name>': '<STR_LIT:content>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(date_command.parse(self._tex))<EOL>command_content = parsed['<STR_LIT:content>'].strip()<EOL><DEDENT>except StopIteration:<EOL><INDENT>command_content = None<EOL>self._logger.warning('<STR_LIT>')<EOL><DEDENT>if command_content is not None and command_content != r'<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>doc_datetime = datetime.datetime.strptime(command_content,<EOL>'<STR_LIT>')<EOL>project_tz = timezone('<STR_LIT>')<EOL>localized_datetime = project_tz.localize(doc_datetime)<EOL>doc_datetime = localized_datetime.astimezone(pytz.utc)<EOL>self._revision_datetime_source = '<STR_LIT>'<EOL><DEDENT>except ValueError:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>command_content)<EOL><DEDENT><DEDENT><DEDENT>if doc_datetime is None:<EOL><INDENT>content_extensions = ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>try:<EOL><INDENT>doc_datetime = get_content_commit_date(<EOL>content_extensions,<EOL>root_dir=self._root_dir)<EOL>self._revision_datetime_source = '<STR_LIT>'<EOL><DEDENT>except RuntimeError:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>self._root_dir)<EOL><DEDENT><DEDENT>if doc_datetime is None:<EOL><INDENT>doc_datetime = pytz.utc.localize(datetime.datetime.now())<EOL>self._revision_datetime_source = '<STR_LIT>'<EOL><DEDENT>self._datetime = doc_datetime<EOL>

r"""Parse the ``\date`` command, falling back to getting the
        most recent Git commit date and the current datetime.

        Result is available from the `revision_datetime` attribute.


def format_short_title(self, format='<STR_LIT>', deparagraph=True,<EOL>mathjax=False, smart=True, extra_args=None):

if self.short_title is None:<EOL><INDENT>return None<EOL><DEDENT>output_text = convert_lsstdoc_tex(<EOL>self.short_title, '<STR_LIT>',<EOL>deparagraph=deparagraph,<EOL>mathjax=mathjax,<EOL>smart=smart,<EOL>extra_args=extra_args)<EOL>return output_text<EOL>

Get the document short title in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content or `None` if the short title is not available in
            the document.


return self.format_title(format='<STR_LIT>', deparagraph=True,<EOL>mathjax=False, smart=True)<EOL>

Plain-text-formatted document title (`str`).


command = LatexCommand(<EOL>'<STR_LIT:title>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': False, '<STR_LIT>': '<STR_LIT:[>'},<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._title = None<EOL>self._short_title = None<EOL><DEDENT>self._title = parsed['<STR_LIT>']<EOL>try:<EOL><INDENT>self._short_title = parsed['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._short_title = None<EOL><DEDENT>

Parse the title from TeX source.

        Sets these attributes:

        - ``_title``
        - ``_short_title``


def format_authors(self, format='<STR_LIT>', deparagraph=True, mathjax=False,<EOL>smart=True, extra_args=None):

formatted_authors = []<EOL>for latex_author in self.authors:<EOL><INDENT>formatted_author = convert_lsstdoc_tex(<EOL>latex_author, format,<EOL>deparagraph=deparagraph,<EOL>mathjax=mathjax,<EOL>smart=smart,<EOL>extra_args=extra_args)<EOL>formatted_author = formatted_author.strip()<EOL>formatted_authors.append(formatted_author)<EOL><DEDENT>return formatted_authors<EOL>

Get the document authors in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `list` of `str`
            Sequence of author names in the specified output markup format.


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_documentclass()<EOL><DEDENT>if '<STR_LIT>' in self._document_options:<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Document is a draft if ``'lsstdoc'`` is included in the
        documentclass options (`bool`).


return self.format_content(format='<STR_LIT>', mathjax=False, smart=True)<EOL>

Plain-text-formatted document content (`str`).


def format_abstract(self, format='<STR_LIT>', deparagraph=False, mathjax=False,<EOL>smart=True, extra_args=None):

if self.abstract is None:<EOL><INDENT>return None<EOL><DEDENT>abstract_latex = self._prep_snippet_for_pandoc(self.abstract)<EOL>output_text = convert_lsstdoc_tex(<EOL>abstract_latex, format,<EOL>deparagraph=deparagraph,<EOL>mathjax=mathjax,<EOL>smart=smart,<EOL>extra_args=extra_args)<EOL>return output_text<EOL>

Get the document abstract in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content or `None` if the title is not available in
            the document.


return self.format_short_title(format='<STR_LIT>', deparagraph=True,<EOL>mathjax=False, smart=True)<EOL>

Plaintext-formatted document short title (`str`).


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_author()<EOL><DEDENT>return self._authors<EOL>

LaTeX-formatted authors (`list` of `str`).


<EOL>command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL>bib_names = [n.strip() for n in parsed['<STR_LIT>'].split('<STR_LIT:U+002C>')]<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>bib_names = []<EOL><DEDENT>custom_bib_names = [n for n in bib_names<EOL>if n not in KNOWN_LSSTTEXMF_BIB_NAMES]<EOL>custom_bibs = []<EOL>for custom_bib_name in custom_bib_names:<EOL><INDENT>custom_bib_path = os.path.join(<EOL>os.path.join(self._root_dir),<EOL>custom_bib_name + '<STR_LIT>'<EOL>)<EOL>if not os.path.exists(custom_bib_path):<EOL><INDENT>self._logger.warning('<STR_LIT>',<EOL>custom_bib_path)<EOL>continue<EOL><DEDENT>with open(custom_bib_path, '<STR_LIT:r>') as file_handle:<EOL><INDENT>custom_bibs.append(file_handle.read())<EOL><DEDENT><DEDENT>if len(custom_bibs) > <NUM_LIT:0>:<EOL><INDENT>custom_bibtex = '<STR_LIT>'.join(custom_bibs)<EOL><DEDENT>else:<EOL><INDENT>custom_bibtex = None<EOL><DEDENT>db = get_bibliography(bibtex=custom_bibtex)<EOL>self._bib_db = db<EOL>

r"""Load the BibTeX bibliography referenced by the document.

        This method triggered by the `bib_db` attribute and populates the
        `_bib_db` private attribute.

        The ``\bibliography`` command is parsed to identify the bibliographies
        referenced by the document.


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_abstract()<EOL><DEDENT>return self._abstract<EOL>


<EOL>command_pattern = r'<STR_LIT>' + name + r'<STR_LIT>'<EOL>command_match = re.search(command_pattern, source)<EOL>if command_match is not None:<EOL><INDENT>source = source[command_match.end(<NUM_LIT:1>):]<EOL><DEDENT>pattern = r'<STR_LIT>'<EOL>match = re.search(pattern, source)<EOL>if match is None:<EOL><INDENT>message = (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>)<EOL>raise CommandParserError(message.format(name))<EOL><DEDENT>content = match.group('<STR_LIT:content>')<EOL>content.strip()<EOL>return content<EOL>

r"""Attempt to parse a single token on the first line of this source.

        This method is used for parsing whitespace-delimited arguments, like
        ``\input file``. The source should ideally contain `` file`` along
        with a newline character.

        >>> source = 'Line 1\n' r'\input test.tex' '\nLine 2'
        >>> LatexCommand._parse_whitespace_argument(source, 'input')
        'test.tex'

        Bracket delimited arguments (``\input{test.tex}``) are handled in
        the normal logic of `_parse_command`.


command_regex = self._make_command_regex(self.name)<EOL>for match in re.finditer(command_regex, source):<EOL><INDENT>self._logger.debug(match)<EOL>start_index = match.start(<NUM_LIT:0>)<EOL>yield self._parse_command(source, start_index)<EOL><DEDENT>

Parse command content from the LaTeX source.

        Parameters
        ----------
        source : `str`
            The full source of the tex document.

        Yields
        ------
        parsed_command : `ParsedCommand`
            Yields parsed commands instances for each occurence of the command
            in the source.


def _format_last(person):<EOL><INDENT>"""<STR_LIT>"""<EOL>return '<STR_LIT:U+0020>'.join([n.strip('<STR_LIT:{}>') for n in person.last_names])<EOL><DEDENT>if len(entry.persons['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>persons = entry.persons['<STR_LIT>']<EOL><DEDENT>elif len(entry.persons['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>persons = entry.persons['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>raise AuthorYearError<EOL><DEDENT>try:<EOL><INDENT>year = entry.fields['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>raise AuthorYearError<EOL><DEDENT>if paren and len(persons) == <NUM_LIT:1>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author=_format_last(persons[<NUM_LIT:0>]),<EOL>year=year)<EOL><DEDENT>elif not paren and len(persons) == <NUM_LIT:1>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author=_format_last(persons[<NUM_LIT:0>]),<EOL>year=year)<EOL><DEDENT>elif paren and len(persons) == <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author1=_format_last(persons[<NUM_LIT:0>]),<EOL>author2=_format_last(persons[<NUM_LIT:1>]),<EOL>year=year)<EOL><DEDENT>elif not paren and len(persons) == <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author1=_format_last(persons[<NUM_LIT:0>]),<EOL>author2=_format_last(persons[<NUM_LIT:1>]),<EOL>year=year)<EOL><DEDENT>elif not paren and len(persons) > <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author=_format_last(persons[<NUM_LIT:0>]),<EOL>year=year)<EOL><DEDENT>elif paren and len(persons) > <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author=_format_last(persons[<NUM_LIT:0>]),<EOL>year=year)<EOL><DEDENT>

Get and format author-year text from a pybtex entry to emulate
    natbib citations.

    Parameters
    ----------
    entry : `pybtex.database.Entry`
        A pybtex bibliography entry.
    parens : `bool`, optional
        Whether to add parentheses around the year. Default is `False`.

    Returns
    -------
    authoryear : `str`
        The author-year citation text.


logger = logging.getLogger(__name__)<EOL>if bibtex_filenames is None:<EOL><INDENT>bibtex_names = KNOWN_LSSTTEXMF_BIB_NAMES<EOL><DEDENT>else:<EOL><INDENT>bibtex_names = []<EOL>for filename in bibtex_filenames:<EOL><INDENT>name = os.path.basename(os.path.splitext(filename)[<NUM_LIT:0>])<EOL>if name not in KNOWN_LSSTTEXMF_BIB_NAMES:<EOL><INDENT>logger.warning('<STR_LIT>',<EOL>name)<EOL>continue<EOL><DEDENT>bibtex_names.append(name)<EOL><DEDENT><DEDENT>uncached_names = [name for name in bibtex_names<EOL>if name not in _LSSTTEXMF_BIB_CACHE]<EOL>if len(uncached_names) > <NUM_LIT:0>:<EOL><INDENT>loop = asyncio.get_event_loop()<EOL>future = asyncio.ensure_future(_download_lsst_bibtex(uncached_names))<EOL>loop.run_until_complete(future)<EOL>for name, text in zip(bibtex_names, future.result()):<EOL><INDENT>_LSSTTEXMF_BIB_CACHE[name] = text<EOL><DEDENT><DEDENT>return {name: _LSSTTEXMF_BIB_CACHE[name] for name in bibtex_names}<EOL>

Get content of lsst-texmf bibliographies.

    BibTeX content is downloaded from GitHub (``master`` branch of
    https://github.com/lsst/lsst-texmf or retrieved from an in-memory cache.

    Parameters
    ----------
    bibtex_filenames : sequence of `str`, optional
        List of lsst-texmf BibTeX files to retrieve. These can be the filenames
        of lsst-bibtex files (for example, ``['lsst.bib', 'lsst-dm.bib']``)
        or names without an extension (``['lsst', 'lsst-dm']``). The default
        (recommended) is to get *all* lsst-texmf bibliographies:

        .. code-block:: python

           ['lsst', 'lsst-dm', 'refs', 'books', 'refs_ads']

    Returns
    -------
    bibtex : `dict`
        Dictionary with keys that are bibtex file names (such as ``'lsst'``,
        ``'lsst-dm'``). Values are the corresponding bibtex file content
        (`str`).


api_root = '<STR_LIT>'<EOL>url = '<STR_LIT>'.format(<EOL>api_root=api_root,<EOL>id_=installation_id)<EOL>headers = {<EOL>'<STR_LIT>': '<STR_LIT>'.format(integration_jwt.decode('<STR_LIT:utf-8>')),<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>resp = requests.post(url, headers=headers)<EOL>resp.raise_for_status()<EOL>return resp.json()<EOL>

Create a GitHub token for an integration installation.

    Parameters
    ----------
    installation_id : `int`
        Installation ID. This is available in the URL of the integration's
        **installation** ID.
    integration_jwt : `bytes`
        The integration's JSON Web Token (JWT). You can create this with
        `create_jwt`.

    Returns
    -------
    token_obj : `dict`
        GitHub token object. Includes the fields:

        - ``token``: the token string itself.
        - ``expires_at``: date time string when the token expires.

    Example
    -------
    The typical workflow for authenticating to an integration installation is:

    .. code-block:: python

       from dochubadapter.github import auth
       jwt = auth.create_jwt(integration_id, private_key_path)
       token_obj = auth.get_installation_token(installation_id, jwt)
       print(token_obj['token'])

    Notes
    -----
    See
    https://developer.github.com/early-access/integrations/authentication/#as-an-installation
    for more information