<EOL>units_for_skip_conn = []<EOL>conv_net_params = {'<STR_LIT>': filter_width,<EOL>'<STR_LIT>': use_batch_norm,<EOL>'<STR_LIT>': training_ph}<EOL>for n_hidden in n_hidden_list:<EOL><INDENT>units = stacked_cnn(units, [n_hidden], **conv_net_params)<EOL>units_for_skip_conn.append(units)<EOL>units = tf.layers.max_pooling1d(units, pool_size=<NUM_LIT:2>, strides=<NUM_LIT:2>, padding='<STR_LIT>')<EOL><DEDENT>units = stacked_cnn(units, [n_hidden], **conv_net_params)<EOL>for down_step, n_hidden in enumerate(n_hidden_list[::-<NUM_LIT:1>]):<EOL><INDENT>units = tf.expand_dims(units, axis=<NUM_LIT:2>)<EOL>units = tf.layers.conv2d_transpose(units, n_hidden, filter_width, strides=(<NUM_LIT:2>, <NUM_LIT:1>), padding='<STR_LIT>')<EOL>units = tf.squeeze(units, axis=<NUM_LIT:2>)<EOL>skip_units = units_for_skip_conn[-(down_step + <NUM_LIT:1>)]<EOL>if skip_units.get_shape().as_list()[-<NUM_LIT:1>] != n_hidden:<EOL><INDENT>skip_units = tf.layers.dense(skip_units, n_hidden)<EOL><DEDENT>units = skip_units + units<EOL>units = stacked_cnn(units, [n_hidden], **conv_net_params)<EOL><DEDENT>return units<EOL>

Network architecture inspired by One Hundred layer Tiramisu.
        https://arxiv.org/abs/1611.09326. U-Net like.

        Args:
            units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
            n_hidden_list: list with number of hidden units at the ouput of each layer
            filter_width: width of the kernel in tokens
            use_batch_norm: whether to use batch normalization between layers
            training_ph: boolean placeholder determining whether is training phase now or not.
                It is used only for batch normalization to determine whether to use
                current batch average (std) or memory stored average (std)
        Returns:
            units: tensor at the output of the last convolutional layer
                    with dimensionality [None, n_tokens, n_hidden_list[-1]]


def variational_dropout(units, keep_prob, fixed_mask_dims=(<NUM_LIT:1>,)):

units_shape = tf.shape(units)<EOL>noise_shape = [units_shape[n] for n in range(len(units.shape))]<EOL>for dim in fixed_mask_dims:<EOL><INDENT>noise_shape[dim] = <NUM_LIT:1><EOL><DEDENT>return tf.nn.dropout(units, keep_prob, noise_shape)<EOL>

Dropout with the same drop mask for all fixed_mask_dims

    Args:
        units: a tensor, usually with shapes [B x T x F], where
            B - batch size
            T - tokens dimension
            F - feature dimension
        keep_prob: keep probability
        fixed_mask_dims: in these dimensions the mask will be the same

    Returns:
        dropped units tensor


assert axis in (<NUM_LIT:1>, <NUM_LIT:2>)<EOL>n_time_steps = tf.shape(units)[<NUM_LIT:1>]<EOL>repetitions = [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>]<EOL>repetitions[axis] = n_time_steps<EOL>return tf.tile(tf.expand_dims(units, axis), repetitions)<EOL>

Expand and tile tensor along given axis
    Args:
        units: tf tensor with dimensions [batch_size, time_steps, n_input_features]
        axis: axis along which expand and tile. Must be 1 or 2


def attention_gen_block(hidden_for_sketch, hidden_for_attn_alignment, key, attention_depth):

with tf.name_scope('<STR_LIT>'):<EOL><INDENT>sketch_dims = tf.shape(hidden_for_sketch)<EOL>batch_size = sketch_dims[<NUM_LIT:0>]<EOL>num_tokens = sketch_dims[<NUM_LIT:1>]<EOL>hidden_size = sketch_dims[<NUM_LIT:2>]<EOL>attn_alignment_dims = tf.shape(hidden_for_attn_alignment)<EOL>attn_alignment_hidden_size = attn_alignment_dims[<NUM_LIT:2>]<EOL>sketches = [tf.zeros(shape=[batch_size, hidden_size], dtype=tf.float32)]<EOL>aligned_hiddens = []<EOL>cum_att = tf.zeros(shape=[batch_size, num_tokens])  <EOL>for i in range(attention_depth):<EOL><INDENT>sketch, cum_att_, aligned_hidden = attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketches[-<NUM_LIT:1>], key, cum_att)<EOL>sketches.append(sketch) <EOL>aligned_hiddens.append(aligned_hidden) <EOL>cum_att += cum_att_<EOL><DEDENT>final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>]),[<NUM_LIT:1>, attention_depth, attn_alignment_hidden_size])<EOL><DEDENT>return final_aligned_hiddens<EOL>

It is a implementation of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        attention_depth: Number of usage csoftmax
    Returns:
        final_aligned_hiddens: Tensor at the output with dimensionality [1, attention_depth, hidden_size_for_attn_alignment]


shape_ten = tensor.shape<EOL>shape_cum = inv_cumulative_att.shape<EOL>merge_tensor = [tensor, inv_cumulative_att]<EOL>cs, _ = tf.map_fn(csoftmax_for_slice, merge_tensor, dtype=[tf.float32, tf.float32])  <EOL>return cs<EOL>

It is a implementation of the constrained softmax (csoftmax).
        Based on the paper:
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        tensor: A tensorflow tensor is score. This tensor have dimensionality [None, n_tokens]
        inv_cumulative_att: A inverse cumulative attention tensor with dimensionality [None, n_tokens]
    Returns:
        cs: Tensor at the output with dimensionality [None, n_tokens]


def bahdanau_attention(key, context, hidden_size, projected_align=False):

if hidden_size % <NUM_LIT:2> != <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>batch_size = tf.shape(context)[<NUM_LIT:0>]<EOL>max_num_tokens, token_size = context.get_shape().as_list()[-<NUM_LIT:2>:]<EOL>r_context = tf.reshape(context, shape=[-<NUM_LIT:1>, max_num_tokens, token_size])<EOL>projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())<EOL>r_projected_key =tf.tile(tf.reshape(projected_key, shape=[-<NUM_LIT:1>, <NUM_LIT:1>, hidden_size]),<EOL>[<NUM_LIT:1>, max_num_tokens, <NUM_LIT:1>])<EOL>lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>(output_fw, output_bw), states =tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,<EOL>cell_bw=lstm_bw_cell,<EOL>inputs=r_context,<EOL>dtype=tf.float32)<EOL>bilstm_output = tf.concat([output_fw, output_bw], -<NUM_LIT:1>)<EOL>concat_h_state = tf.concat([r_projected_key, output_fw, output_bw], -<NUM_LIT:1>)<EOL>projected_state =tf.layers.dense(concat_h_state, hidden_size, use_bias=False,<EOL>kernel_initializer=xav())<EOL>score =tf.layers.dense(tf.tanh(projected_state), units=<NUM_LIT:1>, use_bias=False,<EOL>kernel_initializer=xav())<EOL>attn = tf.nn.softmax(score, dim=<NUM_LIT:1>)<EOL>if projected_align:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(bilstm_output, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, hidden_size])<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(r_context, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, token_size])<EOL><DEDENT>return output<EOL>

It is a implementation of the Bahdanau et al. attention mechanism. Based on the paper:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]


def multiplicative_self_attention(units, n_hidden=None, n_output_features=None, activation=None):

n_input_features = K.int_shape(units)[<NUM_LIT:2>]<EOL>if n_hidden is None:<EOL><INDENT>n_hidden = n_input_features<EOL><DEDENT>if n_output_features is None:<EOL><INDENT>n_output_features = n_input_features<EOL><DEDENT>exp1 = Lambda(lambda x: expand_tile(x, axis=<NUM_LIT:1>))(units)<EOL>exp2 = Lambda(lambda x: expand_tile(x, axis=<NUM_LIT:2>))(units)<EOL>queries = Dense(n_hidden)(exp1)<EOL>keys = Dense(n_hidden)(exp2)<EOL>scores = Lambda(lambda x: K.sum(queries * x, axis=<NUM_LIT:3>, keepdims=True))(keys)<EOL>attention = Lambda(lambda x: softmax(x, axis=<NUM_LIT:2>))(scores)<EOL>mult = Multiply()([attention, exp1])<EOL>attended_units = Lambda(lambda x: K.sum(x, axis=<NUM_LIT:2>))(mult)<EOL>output = Dense(n_output_features, activation=activation)(attended_units)<EOL>return output<EOL>

Compute multiplicative self attention for time series of vectors (with batch dimension)
the formula: score(h_i, h_j) = <W_1 h_i,  W_2 h_j>,  W_1 and W_2 are learnable matrices
with dimensionality [n_hidden, n_input_features]

Args:
    units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
    n_hidden: number of units in hidden representation of similarity measure
    n_output_features: number of features in output dense layer
    activation: activation at the output

Returns:
    output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]


def additive_self_attention(units, n_hidden=None, n_output_features=None, activation=None):

n_input_features = K.int_shape(units)[<NUM_LIT:2>]<EOL>if n_hidden is None:<EOL><INDENT>n_hidden = n_input_features<EOL><DEDENT>if n_output_features is None:<EOL><INDENT>n_output_features = n_input_features<EOL><DEDENT>exp1 = Lambda(lambda x: expand_tile(x, axis=<NUM_LIT:1>))(units)<EOL>exp2 = Lambda(lambda x: expand_tile(x, axis=<NUM_LIT:2>))(units)<EOL>units_pairs = Concatenate(axis=<NUM_LIT:3>)([exp1, exp2])<EOL>query = Dense(n_hidden, activation="<STR_LIT>")(units_pairs)<EOL>attention = Dense(<NUM_LIT:1>, activation=lambda x: softmax(x, axis=<NUM_LIT:2>))(query)<EOL>attended_units = Lambda(lambda x: K.sum(attention * x, axis=<NUM_LIT:2>))(exp1)<EOL>output = Dense(n_output_features, activation=activation)(attended_units)<EOL>return output<EOL>

Compute additive self attention for time series of vectors (with batch dimension)
        the formula: score(h_i, h_j) = <v, tanh(W_1 h_i + W_2 h_j)>
        v is a learnable vector of n_hidden dimensionality,
        W_1 and W_2 are learnable [n_hidden, n_input_features] matrices

Args:
    units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
    n_hidden: number of2784131 units in hidden representation of similarity measure
    n_output_features: number of features in output dense layer
    activation: activation at the output

Returns:
    output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]


def interact_model(config: Union[str, Path, dict]) -> None:

model = build_model(config)<EOL>while True:<EOL><INDENT>args = []<EOL>for in_x in model.in_x:<EOL><INDENT>args.append((input('<STR_LIT>'.format(in_x)),))<EOL>if args[-<NUM_LIT:1>][<NUM_LIT:0>] in {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:q>'}:<EOL><INDENT>return<EOL><DEDENT><DEDENT>pred = model(*args)<EOL>if len(model.out_params) > <NUM_LIT:1>:<EOL><INDENT>pred = zip(*pred)<EOL><DEDENT>print('<STR_LIT>', *pred)<EOL><DEDENT>

Start interaction with the model described in corresponding configuration file.


def initialize_params_in_config(self, basic_config: dict, paths: List[list]) -> dict:

config = deepcopy(basic_config)<EOL>for path_ in paths:<EOL><INDENT>param_name = path_[-<NUM_LIT:1>]<EOL>value = self.get_value_from_config(basic_config, path_)<EOL>if isinstance(value, dict):<EOL><INDENT>if (value.get(self.prefix + "<STR_LIT>") or<EOL>value.get(self.prefix + "<STR_LIT>") or<EOL>value.get(self.prefix + "<STR_LIT>")):<EOL><INDENT>self.insert_value_or_dict_into_config(<EOL>config, path_,<EOL>self.sample_params(**{param_name: deepcopy(value)})[param_name])<EOL><DEDENT><DEDENT><DEDENT>return config<EOL>

Randomly initialize all the changable parameters in config

Args:
    basic_config: config where changable parameters are dictionaries with keys
        ``prefix`_range`, ``prefix`_bool`, ``prefix`_choice`
    paths: list of paths to changable parameters

Returns:
    config


def sample_params(self, **params) -> dict:

if not params:<EOL><INDENT>return {}<EOL><DEDENT>else:<EOL><INDENT>params_copy = deepcopy(params)<EOL><DEDENT>params_sample = dict()<EOL>for param, param_val in params_copy.items():<EOL><INDENT>if isinstance(param_val, dict):<EOL><INDENT>if self.prefix + '<STR_LIT>' in param_val and param_val[self.prefix + '<STR_LIT>']:<EOL><INDENT>sample = bool(random.choice([True, False]))<EOL><DEDENT>elif self.prefix + '<STR_LIT>' in param_val:<EOL><INDENT>sample = self._sample_from_ranges(param_val)<EOL><DEDENT>elif self.prefix + '<STR_LIT>' in param_val:<EOL><INDENT>sample = random.choice(param_val[self.prefix + '<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>sample = param_val<EOL><DEDENT>params_sample[param] = sample<EOL><DEDENT>else:<EOL><INDENT>params_sample[param] = params_copy[param]<EOL><DEDENT><DEDENT>return params_sample<EOL>

Sample parameters according to the given possible values

Args:
    **params: dictionary like {"param_0": {"`prefix`_range": [0, 10]},
                               "param_1": {"`prefix`_range": [0, 10], "discrete": true},
                               "param_2": {"`prefix`_range": [0, 1], "scale": "log"},
                               "param_3": {"`prefix`_bool": true},
                               "param_4": {"`prefix`_choice": [0, 1, 2, 3]}}

Returns:
    dictionary with randomly sampled parameters


def get_metric_by_name(name: str) -> Callable[..., Any]:

if name not in _REGISTRY:<EOL><INDENT>raise ConfigError(f'<STR_LIT>')<EOL><DEDENT>return fn_from_str(_REGISTRY[name])<EOL>

Returns a metric callable with a corresponding name.


def read(self, data_path: str, *args, **kwargs) -> Dict[str, List[Tuple[Any, Any]]]:

raise NotImplementedError<EOL>

Reads a file from a path and returns data as a list of tuples of inputs and correct outputs
         for every data type in ``train``, ``valid`` and ``test``.


scheme, netloc, path, query_string, fragment = urlsplit(url)<EOL>path += '<STR_LIT>'<EOL>return urlunsplit((scheme, netloc, path, query_string, fragment))<EOL>

Given a file URL, return a md5 query of the file

    Args:
        url: a given URL
    Returns:
        URL of the md5 file


def set_query_parameter(url, param_name, param_value):

scheme, netloc, path, query_string, fragment = urlsplit(url)<EOL>query_params = parse_qs(query_string)<EOL>query_params[param_name] = [param_value]<EOL>new_query_string = urlencode(query_params, doseq=True)<EOL>return urlunsplit((scheme, netloc, path, new_query_string, fragment))<EOL>

Given a URL, set or replace a query parameter and return the modified URL.

    Args:
        url: a given  URL
        param_name: the parameter name to add
        param_value: the parameter value
    Returns:
        URL with the added parameter


<EOL>agent = self.agent_generator()<EOL>return agent<EOL>

Initiates Alexa skill agent from agent generator


def verify_cert(signature_chain_url: str) -> Optional[crypto.X509]:

try:<EOL><INDENT>certs_chain_get = requests.get(signature_chain_url)<EOL><DEDENT>except requests.exceptions.ConnectionError as e:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL>return None<EOL><DEDENT>certs_chain_txt = certs_chain_get.text<EOL>certs_chain = extract_certs(certs_chain_txt)<EOL>amazon_cert: crypto.X509 = certs_chain.pop(<NUM_LIT:0>)<EOL>sc_url_verification = verify_sc_url(signature_chain_url)<EOL>if not sc_url_verification:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL><DEDENT>expired_verification = not amazon_cert.has_expired()<EOL>if not expired_verification:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL><DEDENT>sans_verification = verify_sans(amazon_cert)<EOL>if not sans_verification:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL><DEDENT>chain_verification = verify_certs_chain(certs_chain, amazon_cert)<EOL>if not chain_verification:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL><DEDENT>result = (sc_url_verification and expired_verification and sans_verification and chain_verification)<EOL>return amazon_cert if result else None<EOL>

Conducts series of Alexa SSL certificate verifications against Amazon Alexa requirements.

    Args:
        signature_chain_url: Signature certificate URL from SignatureCertChainUrl HTTP header.
    Returns:
        result: Amazon certificate if verification was successful, None if not.


def verify_sans(amazon_cert: crypto.X509) -> bool:

cert_extentions = [amazon_cert.get_extension(i) for i in range(amazon_cert.get_extension_count())]<EOL>subject_alt_names = '<STR_LIT>'<EOL>for extention in cert_extentions:<EOL><INDENT>if '<STR_LIT>' in str(extention.get_short_name()):<EOL><INDENT>subject_alt_names = extention.__str__()<EOL>break<EOL><DEDENT><DEDENT>result = '<STR_LIT>' in subject_alt_names<EOL>return result<EOL>

Verifies Subject Alternative Names (SANs) for Amazon certificate.

    Args:
        amazon_cert: Pycrypto X509 Amazon certificate.

    Returns:
        result: True if verification was successful, False if not.


def handle_request(self, request: dict) -> dict:

request_type = request['<STR_LIT>']['<STR_LIT:type>']<EOL>request_id = request['<STR_LIT>']['<STR_LIT>']<EOL>log.debug(f'<STR_LIT>')<EOL>if request_type in self.handled_requests.keys():<EOL><INDENT>response: dict = self.handled_requests[request_type](request)<EOL><DEDENT>else:<EOL><INDENT>response: dict = self.handled_requests['<STR_LIT>'](request)<EOL>log.warning(f'<STR_LIT>')<EOL><DEDENT>self._rearm_self_destruct()<EOL>return response<EOL>

Routes Alexa requests to appropriate handlers.

        Args:
            request: Alexa request.
        Returns:
            response: Response conforming Alexa response specification.


self.timer = Timer(self.config['<STR_LIT>'], self.self_destruct_callback)<EOL>self.timer.start()<EOL>


@overrides<EOL><INDENT>def gen_batches(self, batch_size: int, shuffle: bool = None)-> Generator[Tuple[List[str], List[int]], Any, None]:<DEDENT>

if shuffle is None:<EOL><INDENT>shuffle = self.shuffle<EOL><DEDENT>if shuffle:<EOL><INDENT>_doc_ids = self.random.sample(self.doc_ids, len(self.doc_ids))<EOL><DEDENT>else:<EOL><INDENT>_doc_ids = self.doc_ids<EOL><DEDENT>if batch_size > <NUM_LIT:0>:<EOL><INDENT>batches = [_doc_ids[i:i + batch_size] for i in<EOL>range(<NUM_LIT:0>, len(_doc_ids), batch_size)]<EOL><DEDENT>else:<EOL><INDENT>batches = [_doc_ids]<EOL><DEDENT>for i, doc_ids in enumerate(batches):<EOL><INDENT>docs = [self.get_doc_content(doc_id) for doc_id in doc_ids]<EOL>doc_nums = [self.doc2index[doc_id] for doc_id in doc_ids]<EOL>yield docs, zip(doc_ids, doc_nums)<EOL><DEDENT>

Gen batches of documents.

        Args:
            batch_size: a number of samples in a single batch
            shuffle: whether to shuffle data during batching

        Yields:
            generated tuple of documents and their ids


cursor = self.connect.cursor()<EOL>cursor.execute("<STR_LIT>")<EOL>assert cursor.arraysize == <NUM_LIT:1><EOL>name = cursor.fetchone()[<NUM_LIT:0>]<EOL>cursor.close()<EOL>return name<EOL>

Get DB name.

        Returns:
            DB name


def map_doc2idx(self) -> Dict[int, Any]:

doc2idx = {doc_id: i for i, doc_id in enumerate(self.doc_ids)}<EOL>logger.info(<EOL>"<STR_LIT>".format(len(doc2idx)))<EOL>return doc2idx<EOL>

Map DB ids to integer ids.

        Returns:
            a dictionary of document titles and correspondent integer indices


@overrides<EOL><INDENT>def get_doc_ids(self) -> List[Any]:<DEDENT>

cursor = self.connect.cursor()<EOL>cursor.execute('<STR_LIT>'.format(self.db_name))<EOL>ids = [ids[<NUM_LIT:0>] for ids in cursor.fetchall()]<EOL>cursor.close()<EOL>return ids<EOL>

Get document ids.

        Returns:
            document ids


doc_ids = list(self.doc_ids)<EOL>docs = [self.get_doc_content(doc_id) for doc_id in doc_ids]<EOL>doc_nums = [self.doc2index[doc_id] for doc_id in doc_ids]<EOL>return docs, zip(doc_ids, doc_nums)<EOL>


@overrides<EOL><INDENT>def get_doc_content(self, doc_id: Any) -> Optional[str]:<DEDENT>

cursor = self.connect.cursor()<EOL>cursor.execute(<EOL>"<STR_LIT>".format(self.db_name),<EOL>(doc_id,)<EOL>)<EOL>result = cursor.fetchone()<EOL>cursor.close()<EOL>return result if result is None else result[<NUM_LIT:0>]<EOL>

Get document content by id.

        Args:
            doc_id: a document id

        Returns:
            document content if success, else raise Exception


@staticmethod<EOL><INDENT>def _extract_cqas(data: Dict[str, Any]) -> List[Tuple[Tuple[str, str], Tuple[List[str], List[int]]]]:<DEDENT>

cqas = []<EOL>if data:<EOL><INDENT>for article in data['<STR_LIT:data>']:<EOL><INDENT>for par in article['<STR_LIT>']:<EOL><INDENT>context = par['<STR_LIT>']<EOL>for qa in par['<STR_LIT>']:<EOL><INDENT>q = qa['<STR_LIT>']<EOL>ans_text = []<EOL>ans_start = []<EOL>for answer in qa['<STR_LIT>']:<EOL><INDENT>ans_text.append(answer['<STR_LIT:text>'])<EOL>ans_start.append(answer['<STR_LIT>'])<EOL><DEDENT>cqas.append(((context, q), (ans_text, ans_start)))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return cqas<EOL>

Extracts context, question, answer, answer_start from SQuAD data

        Args:
            data: data in squad format

        Returns:
            list of (context, question), (answer_text, answer_start)
            answer text and answer_start are lists


def gen_batches(self, batch_size: int, data_type: str = '<STR_LIT:train>',<EOL>shuffle: bool = None, return_indexes: bool = False) -> Iterator[tuple]:

if shuffle is None:<EOL><INDENT>shuffle = self.shuffle<EOL><DEDENT>data = self.data[data_type]<EOL>if shuffle:<EOL><INDENT>random.shuffle(data)<EOL><DEDENT>lengths = [len(x[<NUM_LIT:0>]) for x in data]<EOL>indexes = np.argsort(lengths)<EOL>L = len(data)<EOL>if batch_size < <NUM_LIT:0>:<EOL><INDENT>batch_size = L<EOL><DEDENT>for start in range(<NUM_LIT:0>, L, batch_size):<EOL><INDENT>indexes_to_yield = indexes[start:start+batch_size]<EOL>data_to_yield = tuple(list(x) for x in zip(*([data[i] for i in indexes_to_yield])))<EOL>if return_indexes:<EOL><INDENT>yield indexes_to_yield, data_to_yield<EOL><DEDENT>else:<EOL><INDENT>yield data_to_yield<EOL><DEDENT><DEDENT>

Generate batches of inputs and expected output to train neural networks

        Args:
            batch_size: number of samples in batch
            data_type: can be either 'train', 'test', or 'valid'
            shuffle: whether to shuffle dataset before batching
            return_indexes: whether to return indexes of batch elements in initial dataset

        Yields:
            a tuple of a batch of inputs and a batch of expected outputs.
            If `return_indexes` is True, also yields indexes of batch elements.


args = parser.parse_args()<EOL>path = get_settings_path()<EOL>if args.default:<EOL><INDENT>if populate_settings_dir(force=True):<EOL><INDENT>print(f'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print(f'<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print(f'<STR_LIT>')<EOL><DEDENT>

DeepPavlov console configuration utility.


reqs_path = os.path.join(__location__, '<STR_LIT>')<EOL>with open(reqs_path, encoding='<STR_LIT:utf8>') as f:<EOL><INDENT>reqs = [line.strip() for line in f if not line.strip().startswith('<STR_LIT:#>')]<EOL><DEDENT>names = []<EOL>links = []<EOL>for req in reqs:<EOL><INDENT>if '<STR_LIT>' in req:<EOL><INDENT>links.append(req)<EOL><DEDENT>else:<EOL><INDENT>names.append(req)<EOL><DEDENT><DEDENT>return {'<STR_LIT>': names, '<STR_LIT>': links}<EOL>

parses requirements from requirements.txt


def is_effective_member(self, group_id, netid):

self._valid_group_id(group_id)<EOL>netid = re.sub('<STR_LIT>', '<STR_LIT>', netid)<EOL>url = "<STR_LIT>".format(self.API,<EOL>group_id,<EOL>netid)<EOL>try:<EOL><INDENT>data = self._get_resource(url)<EOL>return True  <EOL><DEDENT>except DataFailureException as ex:<EOL><INDENT>if ex.status == <NUM_LIT>:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>

Returns True if the netid is in the group, False otherwise.


self._valid_group_id(group_id)<EOL>url = "<STR_LIT>".format(self.API, group_id)<EOL>data = self._get_resource(url)<EOL>return self._group_from_json(data.get("<STR_LIT:data>"))<EOL>

Returns a restclients.Group object for the group identified by the
passed group ID.


msg = self.format(record)<EOL>if not isinstance(msg, dict):<EOL><INDENT>msg = json.loads(msg)<EOL><DEDENT>self.collection.insert(msg)<EOL>


return {<EOL>ARM32:   (KS_ARCH_ARM,     KS_MODE_ARM),<EOL>ARM64:   (KS_ARCH_ARM64,   KS_MODE_LITTLE_ENDIAN),<EOL>ARM_TB:  (KS_ARCH_ARM,     KS_MODE_THUMB),<EOL>HEXAGON: (KS_ARCH_HEXAGON, KS_MODE_BIG_ENDIAN),<EOL>MIPS32:  (KS_ARCH_MIPS,    KS_MODE_MIPS32),<EOL>MIPS64:  (KS_ARCH_MIPS,    KS_MODE_MIPS64),<EOL>PPC32:   (KS_ARCH_PPC,     KS_MODE_PPC32),<EOL>PPC64:   (KS_ARCH_PPC,     KS_MODE_PPC64),<EOL>SPARC32: (KS_ARCH_SPARC,   KS_MODE_SPARC32),<EOL>SPARC64: (KS_ARCH_SPARC,   KS_MODE_SPARC64),<EOL>SYSTEMZ: (KS_ARCH_SYSTEMZ, KS_MODE_BIG_ENDIAN),<EOL>X86_16:  (KS_ARCH_X86,     KS_MODE_16),<EOL>X86_32:  (KS_ARCH_X86,     KS_MODE_32),<EOL>X86_64:  (KS_ARCH_X86,     KS_MODE_64),<EOL>}<EOL>

Initialize the dictionary of architectures for assembling via keystone


if log.handlers:<EOL><INDENT>return<EOL><DEDENT>handler = logging.StreamHandler()<EOL>formatter = logging.Formatter(<EOL>'<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>log.addHandler(handler)<EOL>

If there's no log configuration, set up a default handler.


try:<EOL><INDENT>request_data = _build_request_data(request)<EOL><DEDENT>except Exception as e:<EOL><INDENT>log.exception("<STR_LIT>", e)<EOL><DEDENT>else:<EOL><INDENT>if request_data:<EOL><INDENT>_filter_ip(request_data, SETTINGS['<STR_LIT>'])<EOL>data['<STR_LIT>'] = request_data<EOL><DEDENT><DEDENT>

Attempts to build request data; if successful, sets the 'request' key on `data`.


for k, v in iteritems(data):<EOL><INDENT>data[k] = _transform(v, key=(k,))<EOL><DEDENT>payload = {<EOL>'<STR_LIT>': SETTINGS['<STR_LIT>'],<EOL>'<STR_LIT:data>': data<EOL>}<EOL>return payload<EOL>


def init(access_token, environment='<STR_LIT>', scrub_fields=None, url_fields=None, **kw):

global SETTINGS, agent_log, _initialized, _transforms, _serialize_transform, _threads<EOL>if scrub_fields is not None:<EOL><INDENT>SETTINGS['<STR_LIT>'] = list(scrub_fields)<EOL><DEDENT>if url_fields is not None:<EOL><INDENT>SETTINGS['<STR_LIT>'] = list(url_fields)<EOL><DEDENT>SETTINGS = dict_merge(SETTINGS, kw)<EOL>if _initialized:<EOL><INDENT>if not SETTINGS.get('<STR_LIT>'):<EOL><INDENT>log.warning('<STR_LIT>')<EOL><DEDENT>return<EOL><DEDENT>SETTINGS['<STR_LIT>'] = access_token<EOL>SETTINGS['<STR_LIT>'] = environment<EOL>if SETTINGS.get('<STR_LIT>'):<EOL><INDENT>logging.basicConfig()<EOL><DEDENT>if SETTINGS.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>agent_log = _create_agent_log()<EOL><DEDENT>_serialize_transform = SerializableTransform(safe_repr=SETTINGS['<STR_LIT>']['<STR_LIT>'],<EOL>whitelist_types=SETTINGS['<STR_LIT>']['<STR_LIT>'])<EOL>_transforms = [<EOL>ScrubRedactTransform(),<EOL>_serialize_transform,<EOL>ScrubTransform(suffixes=[(field,) for field in SETTINGS['<STR_LIT>']], redact_char='<STR_LIT:*>'),<EOL>ScrubUrlTransform(suffixes=[(field,) for field in SETTINGS['<STR_LIT>']], params_to_scrub=SETTINGS['<STR_LIT>'])<EOL>]<EOL>shortener_keys = [<EOL>('<STR_LIT>', '<STR_LIT:POST>'),<EOL>('<STR_LIT>', '<STR_LIT>'),<EOL>('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT:POST>'),<EOL>('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>'),<EOL>]<EOL>if SETTINGS['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>shortener_keys.append(('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:*>', '<STR_LIT:code>'))<EOL>shortener_keys.append(('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:*>', '<STR_LIT:args>', '<STR_LIT:*>'))<EOL>shortener_keys.append(('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:*>', '<STR_LIT>', '<STR_LIT:*>'))<EOL>shortener_keys.append(('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:*>', '<STR_LIT>', '<STR_LIT:*>'))<EOL><DEDENT>shortener_keys.extend(SETTINGS['<STR_LIT>'])<EOL>shortener = ShortenerTransform(safe_repr=SETTINGS['<STR_LIT>']['<STR_LIT>'],<EOL>keys=shortener_keys,<EOL>**SETTINGS['<STR_LIT>']['<STR_LIT>'])<EOL>_transforms.append(shortener)<EOL>_threads = queue.Queue()<EOL>events.reset()<EOL>filters.add_builtin_filters(SETTINGS)<EOL>_initialized = True<EOL>

Saves configuration variables in this module's SETTINGS.

access_token: project access token. Get this from the Rollbar UI:
              - click "Settings" in the top nav
              - click "Projects" in the left nav
              - copy-paste the appropriate token.
environment: environment name. Can be any string; suggestions: 'production', 'development',
             'staging', 'yourname'
**kw: provided keyword arguments will override keys in SETTINGS.


def _report_message(message, level, request, extra_data, payload_data):

if not _check_config():<EOL><INDENT>return<EOL><DEDENT>filtered_message = events.on_message(message,<EOL>request=request,<EOL>extra_data=extra_data,<EOL>payload_data=payload_data,<EOL>level=level)<EOL>if filtered_message is False:<EOL><INDENT>return<EOL><DEDENT>data = _build_base_data(request, level=level)<EOL>data['<STR_LIT:body>'] = {<EOL>'<STR_LIT:message>': {<EOL>'<STR_LIT:body>': filtered_message<EOL>}<EOL>}<EOL>if extra_data:<EOL><INDENT>extra_data = extra_data<EOL>data['<STR_LIT:body>']['<STR_LIT:message>'].update(extra_data)<EOL><DEDENT>request = _get_actual_request(request)<EOL>_add_request_data(data, request)<EOL>_add_person_data(data, request)<EOL>_add_lambda_context_data(data)<EOL>data['<STR_LIT>'] = _build_server_data()<EOL>if payload_data:<EOL><INDENT>data = dict_merge(data, payload_data)<EOL><DEDENT>payload = _build_payload(data)<EOL>send_payload(payload, payload.get('<STR_LIT>'))<EOL>return data['<STR_LIT>']<EOL>


log_file = SETTINGS['<STR_LIT>']<EOL>if not log_file.endswith('<STR_LIT>'):<EOL><INDENT>log.error("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>log_file = DEFAULTS['<STR_LIT>']<EOL><DEDENT>retval = logging.getLogger('<STR_LIT>')<EOL>handler = logging.FileHandler(log_file, '<STR_LIT:a>', '<STR_LIT:utf-8>')<EOL>formatter = logging.Formatter('<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>retval.addHandler(handler)<EOL>retval.setLevel(logging.WARNING)<EOL>return retval<EOL>

Creates .rollbar log file for use with rollbar-agent


def search_items(title, return_fields=None, access_token=None, endpoint=None, **search_fields):

if not title:<EOL><INDENT>return []<EOL><DEDENT>if return_fields is not None:<EOL><INDENT>return_fields = '<STR_LIT:U+002C>'.join(return_fields)<EOL><DEDENT>return _get_api('<STR_LIT>',<EOL>title=title,<EOL>fields=return_fields,<EOL>access_token=access_token,<EOL>endpoint=endpoint,<EOL>**search_fields)<EOL>

Searches a project for items that match the input criteria.

title: all or part of the item's title to search for.
return_fields: the fields that should be returned for each item.
        e.g. ['id', 'project_id', 'status'] will return a dict containing
             only those fields for each item.
access_token: a project access token. If this is not provided,
              the one provided to init() will be used instead.
search_fields: additional fields to include in the search.
        currently supported: status, level, environment


self.notify_level = _checkLevel(level)<EOL>

Override so we set the effective level for which
log records we notify Rollbar about instead of which
records we save to the history.


def scale_image(self, in_fname, out_fname, max_width, max_height):

<EOL>try:<EOL><INDENT>from PIL import Image<EOL><DEDENT>except ImportError:<EOL><INDENT>import Image<EOL><DEDENT>img = Image.open(in_fname)<EOL>width_in, height_in = img.size<EOL>scale_w = max_width / float(width_in)<EOL>scale_h = max_height / float(height_in)<EOL>if height_in * scale_w <= max_height:<EOL><INDENT>scale = scale_w<EOL><DEDENT>else:<EOL><INDENT>scale = scale_h<EOL><DEDENT>if scale >= <NUM_LIT:1.0> and in_fname == out_fname:<EOL><INDENT>return<EOL><DEDENT>width_sc = int(round(scale * width_in))<EOL>height_sc = int(round(scale * height_in))<EOL>img.thumbnail((width_sc, height_sc), Image.ANTIALIAS)<EOL>thumb = Image.new('<STR_LIT>', (max_width, max_height), (<NUM_LIT:255>, <NUM_LIT:255>, <NUM_LIT:255>))<EOL>pos_insert = (<EOL>(max_width - width_sc) // <NUM_LIT:2>, (max_height - height_sc) // <NUM_LIT:2>)<EOL>thumb.paste(img, pos_insert)<EOL>thumb.save(out_fname)<EOL>

Scales an image with the same aspect ratio centered in an
           image with a given max_width and max_height
           if in_fname == out_fname the image can only be scaled down


code_example = self.code_example<EOL>if code_example is None:<EOL><INDENT>return None<EOL><DEDENT>return self.CODE_TEMPLATE.format(<EOL>snippet=self.get_description()[<NUM_LIT:1>], code=code_example,<EOL>ref_name=self.reference)<EOL>

The string for creating a code example for the gallery


ret = None<EOL>if self._thumbnail_figure is not None:<EOL><INDENT>if not isstring(self._thumbnail_figure):<EOL><INDENT>ret = self._thumbnail_figure<EOL><DEDENT>else:<EOL><INDENT>ret = osp.join(osp.dirname(self.outfile),<EOL>osp.basename(self._thumbnail_figure))<EOL>copyfile(self._thumbnail_figure, ret)<EOL>return ret<EOL><DEDENT><DEDENT>elif hasattr(self.nb.metadata, '<STR_LIT>'):<EOL><INDENT>if not isstring(self.nb.metadata.thumbnail_figure):<EOL><INDENT>ret = self.nb.metadata.thumbnail_figure<EOL><DEDENT>else:<EOL><INDENT>ret = osp.join(osp.dirname(self.outfile), '<STR_LIT>',<EOL>osp.basename(self.nb.metadata.thumbnail_figure))<EOL>copyfile(osp.join(osp.dirname(self.infile),<EOL>self.nb.metadata.thumbnail_figure),<EOL>ret)<EOL><DEDENT><DEDENT>return ret<EOL>


def __init__(self, infile, outfile, disable_warnings=True,<EOL>preprocess=True, clear=True, code_example=None,<EOL>supplementary_files=None, other_supplementary_files=None,<EOL>thumbnail_figure=None, url=None, insert_bokeh=False,<EOL>insert_bokeh_widgets=False, tag_options={}):

self.infile = infile<EOL>self.outfile = outfile<EOL>self.preprocess = preprocess<EOL>self.clear = clear<EOL>self._code_example = code_example<EOL>self._supplementary_files = supplementary_files<EOL>self._other_supplementary_files = other_supplementary_files<EOL>self._thumbnail_figure = thumbnail_figure<EOL>self._url = url<EOL>self.insert_bokeh = insert_bokeh<EOL>self.insert_bokeh_widgets = insert_bokeh_widgets<EOL>self.tag_options = tag_options<EOL>self.process_notebook(disable_warnings)<EOL>self.create_thumb()<EOL>

Parameters
----------
infile: str
    path to the existing notebook
outfile: str
    path to the new notebook
disable_warnings: bool
    Boolean to control whether warnings shall be included in the rst
    file or not
preprocess: bool
    If True, the notebook is processed when generating the rst file
clear: bool
    If True, the output in the download notebook is cleared
code_example: str
    A python code sample that shall be used instead of a thumbnail
    figure in the gallery. Note that you can also include a
    ``'code_example'`` key in the metadata of the notebook
supplementary_files: list of str
    Supplementary data files that shall be copied to the output
    directory and inserted in the rst file for download
other_supplementary_files: list of str
    Other supplementary data files that shall be copied but not
    inserted for download
thumbnail_figure: int
    The number of the figure that shall be used for download or a path
    to a file
url: str
    The url where to download the notebook
insert_bokeh: False or str
    The version string for bokeh to use for the style sheet
insert_bokeh_widgets: bool or str
    The version string for bokeh to use for the widgets style sheet
tag_options: dict
    A dictionary with traitlets for the
    :class:`nbconvert.preprocessors.TagRemovePreprocessor`


raw_rst, resources = nbconvert.export_by_name('<STR_LIT>', nb)<EOL>rst_content = '<STR_LIT>'<EOL>i0 = <NUM_LIT:0><EOL>m = None<EOL>bokeh_str = '<STR_LIT>'<EOL>if '<STR_LIT>' in raw_rst and self.insert_bokeh:<EOL><INDENT>bokeh_str += self.BOKEH_TEMPLATE.format(<EOL>version=self.insert_bokeh)<EOL><DEDENT>if '<STR_LIT>' in raw_rst and self.insert_bokeh_widgets:<EOL><INDENT>bokeh_str += self.BOKEH_WIDGETS_TEMPLATE.format(<EOL>version=self.insert_bokeh_widgets)<EOL><DEDENT>for m in code_blocks.finditer(raw_rst):<EOL><INDENT>lines = m.group().splitlines(True)<EOL>header, content = lines[<NUM_LIT:0>], '<STR_LIT>'.join(lines[<NUM_LIT:1>:])<EOL>no_magics = magic_patt.sub('<STR_LIT>', content)<EOL>if no_magics.strip():<EOL><INDENT>rst_content += (<EOL>raw_rst[i0:m.start()] + bokeh_str + header + no_magics)<EOL>bokeh_str = '<STR_LIT>'<EOL>i0 = m.end()<EOL><DEDENT>else:<EOL><INDENT>rst_content += raw_rst[i0:m.start()]<EOL>i0 = m.end()<EOL><DEDENT><DEDENT>if m is not None:<EOL><INDENT>rst_content += bokeh_str + raw_rst[m.end():]<EOL><DEDENT>else:<EOL><INDENT>rst_content = raw_rst<EOL><DEDENT>rst_content = '<STR_LIT>' % self.reference +rst_content<EOL>url = self.url<EOL>if url is not None:<EOL><INDENT>rst_content += self.CODE_DOWNLOAD_NBVIEWER.format(<EOL>pyfile=os.path.basename(self.py_file),<EOL>nbfile=os.path.basename(self.outfile),<EOL>url=url)<EOL><DEDENT>else:<EOL><INDENT>rst_content += self.CODE_DOWNLOAD.format(<EOL>pyfile=os.path.basename(self.py_file),<EOL>nbfile=os.path.basename(self.outfile))<EOL><DEDENT>supplementary_files = self.supplementary_files<EOL>other_supplementary_files = self.other_supplementary_files<EOL>if supplementary_files or other_supplementary_files:<EOL><INDENT>for f in (supplementary_files or []) + (<EOL>other_supplementary_files or []):<EOL><INDENT>if not os.path.exists(os.path.join(odir, f)):<EOL><INDENT>copyfile(os.path.join(in_dir, f), os.path.join(odir, f))<EOL><DEDENT><DEDENT><DEDENT>if supplementary_files:<EOL><INDENT>rst_content += self.data_download(supplementary_files)<EOL><DEDENT>rst_file = self.get_out_file()<EOL>outputs = sorted(resources['<STR_LIT>'], key=rst_content.find)<EOL>base = os.path.join('<STR_LIT>', os.path.splitext(<EOL>os.path.basename(self.infile))[<NUM_LIT:0>] + '<STR_LIT>')<EOL>out_map = {os.path.basename(original): base % i<EOL>for i, original in enumerate(outputs)}<EOL>for original, final in six.iteritems(out_map):<EOL><INDENT>rst_content = rst_content.replace(original, final)<EOL><DEDENT>with open(rst_file, '<STR_LIT:w>')as f:<EOL><INDENT>f.write(rst_content.rstrip() + '<STR_LIT:\n>')<EOL><DEDENT>pictures = []<EOL>for original in outputs:<EOL><INDENT>fname = os.path.join(odir, out_map[os.path.basename(original)])<EOL>pictures.append(fname)<EOL>if six.PY3:<EOL><INDENT>f = open(fname, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>f = open(fname, '<STR_LIT:w>')<EOL><DEDENT>f.write(resources['<STR_LIT>'][original])<EOL>f.close()<EOL><DEDENT>self.pictures = pictures<EOL>

Create the rst file from the notebook node


if six.PY2:<EOL><INDENT>from urlparse import urlparse as urlsplit<EOL><DEDENT>else:<EOL><INDENT>from urllib.parse import urlsplit<EOL><DEDENT>info = urlsplit(url)<EOL>domain = info.netloc<EOL>url_type = '<STR_LIT>' if domain == '<STR_LIT>' else '<STR_LIT:url>'<EOL>return '<STR_LIT>' % (url_type, info.path)<EOL>

Return the link to the Jupyter nbviewer for the given notebook url


if self._code_example is not None:<EOL><INDENT>return self._code_example<EOL><DEDENT>return getattr(self.nb.metadata, '<STR_LIT>', None)<EOL>

The code example out of the notebook metadata


if self._url is not None:<EOL><INDENT>url = self._url<EOL><DEDENT>else:<EOL><INDENT>url = getattr(self.nb.metadata, '<STR_LIT:url>', None)<EOL><DEDENT>if url is not None:<EOL><INDENT>return nbviewer_link(url)<EOL><DEDENT>

The url on jupyter nbviewer for this notebook or None if unknown


def split_header(s, get_header=True):<EOL><INDENT>s = s.lstrip().rstrip()<EOL>parts = s.splitlines()<EOL>if parts[<NUM_LIT:0>].startswith('<STR_LIT:#>'):<EOL><INDENT>if get_header:<EOL><INDENT>header = re.sub('<STR_LIT>', '<STR_LIT>', parts.pop(<NUM_LIT:0>))<EOL>if not parts:<EOL><INDENT>return header, '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>header = '<STR_LIT>'<EOL><DEDENT>rest = '<STR_LIT:\n>'.join(parts).lstrip().split('<STR_LIT>')<EOL>desc = rest[<NUM_LIT:0>].replace('<STR_LIT:\n>', '<STR_LIT:U+0020>')<EOL>return header, desc<EOL><DEDENT>else:<EOL><INDENT>if get_header:<EOL><INDENT>if parts[<NUM_LIT:0>].startswith(('<STR_LIT:=>', '<STR_LIT:->')):<EOL><INDENT>parts = parts[<NUM_LIT:1>:]<EOL><DEDENT>header = parts.pop(<NUM_LIT:0>)<EOL>if parts and parts[<NUM_LIT:0>].startswith(('<STR_LIT:=>', '<STR_LIT:->')):<EOL><INDENT>parts.pop(<NUM_LIT:0>)<EOL><DEDENT>if not parts:<EOL><INDENT>return header, '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>header = '<STR_LIT>'<EOL><DEDENT>rest = '<STR_LIT:\n>'.join(parts).lstrip().split('<STR_LIT>')<EOL>desc = rest[<NUM_LIT:0>].replace('<STR_LIT:\n>', '<STR_LIT:U+0020>')<EOL>return header, desc<EOL><DEDENT><DEDENT>first_cell = self.nb['<STR_LIT>'][<NUM_LIT:0>]<EOL>if not first_cell['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>', '<STR_LIT>'<EOL><DEDENT>header, desc = split_header(first_cell['<STR_LIT:source>'])<EOL>if not desc and len(self.nb['<STR_LIT>']) > <NUM_LIT:1>:<EOL><INDENT>second_cell = self.nb['<STR_LIT>'][<NUM_LIT:1>]<EOL>if second_cell['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>_, desc = split_header(second_cell['<STR_LIT:source>'], False)<EOL><DEDENT><DEDENT>return header, desc<EOL>

Get summary and description of this notebook


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters = ['<STR_LIT>']<EOL>output_names = ['<STR_LIT>', '<STR_LIT>']<EOL>if r2_id:<EOL><INDENT>job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', '<STR_LIT:2>', '<STR_LIT>'])<EOL>output_names.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir, parameters=parameters)<EOL>output_files = [os.path.join(work_dir, x) for x in output_names]<EOL>tarball_files(tar_name='<STR_LIT>', file_paths=output_files, output_dir=work_dir)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Run Fastqc on the input reads

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str r1_id: FileStoreID of fastq read 1
:param str r2_id: FileStoreID of fastq read 2
:return: FileStoreID of fastQC output (tarball)
:rtype: str


def run_gatk_preprocessing(job, bam, bai, ref, ref_dict, fai, g1k, mills, dbsnp, realign=False, unsafe=False):

<EOL>mdups_disk = PromisedRequirement(lambda bam_, bai_: <NUM_LIT:2> * (bam_.size + bai_.size), bam, bai)<EOL>mdups = job.wrapJobFn(picard_mark_duplicates,<EOL>bam,<EOL>bai,<EOL>cores=job.cores,<EOL>disk=mdups_disk,<EOL>memory=job.memory)<EOL>bqsr_input_bam = mdups.rv(<NUM_LIT:0>)<EOL>bqsr_input_bai = mdups.rv(<NUM_LIT:1>)<EOL>genome_ref_size = ref.size + ref_dict.size + fai.size<EOL>if realign:<EOL><INDENT>indel_ref_size = mills.size + g1k.size + genome_ref_size<EOL>realigner_target_disk = PromisedRequirement(lambda bam_, bai_, ref_size:<EOL>bam_.size + bai_.size + <NUM_LIT:2> * ref_size,<EOL>mdups.rv(<NUM_LIT:0>),<EOL>mdups.rv(<NUM_LIT:1>),<EOL>indel_ref_size)<EOL>realigner_target = job.wrapJobFn(run_realigner_target_creator,<EOL>mdups.rv(<NUM_LIT:0>),<EOL>mdups.rv(<NUM_LIT:1>),<EOL>ref, ref_dict, fai,<EOL>g1k, mills,<EOL>unsafe=unsafe,<EOL>cores=<NUM_LIT:1>,  <EOL>disk=realigner_target_disk,<EOL>memory=job.memory)<EOL>indel_realign_disk = PromisedRequirement(lambda bam_, bai_, intervals, ref_size:<EOL><NUM_LIT:2> * (bam_.size + bai_.size) + intervals.size + ref_size,<EOL>mdups.rv(<NUM_LIT:0>),<EOL>mdups.rv(<NUM_LIT:1>),<EOL>realigner_target.rv(),<EOL>indel_ref_size)<EOL>indel_realign = job.wrapJobFn(run_indel_realignment,<EOL>realigner_target.rv(),<EOL>mdups.rv(<NUM_LIT:0>),<EOL>mdups.rv(<NUM_LIT:1>),<EOL>ref, ref_dict, fai,<EOL>g1k, mills,<EOL>unsafe=unsafe,<EOL>cores=<NUM_LIT:1>,  <EOL>disk=indel_realign_disk,<EOL>memory=job.memory)<EOL>mdups.addChild(realigner_target)<EOL>realigner_target.addChild(indel_realign)<EOL>bqsr_input_bam = indel_realign.rv(<NUM_LIT:0>)<EOL>bqsr_input_bai = indel_realign.rv(<NUM_LIT:1>)<EOL><DEDENT>bqsr_ref_size = dbsnp.size + mills.size + genome_ref_size<EOL>base_recal_disk = PromisedRequirement(lambda bam_, bai_, ref_size:<EOL>bam_.size + bai_.size + <NUM_LIT:2> * ref_size,<EOL>bqsr_input_bam,<EOL>bqsr_input_bai,<EOL>bqsr_ref_size)<EOL>base_recal = job.wrapJobFn(run_base_recalibration,<EOL>bqsr_input_bam,<EOL>bqsr_input_bai,<EOL>ref, ref_dict, fai,<EOL>dbsnp, mills,<EOL>unsafe=unsafe,<EOL>cores=job.cores,<EOL>disk=base_recal_disk,<EOL>memory=job.memory)<EOL>recalibrate_reads_disk = PromisedRequirement(lambda bam_, bai_, recal, ref_size:<EOL><NUM_LIT:2> * (bam_.size + bai_.size) + recal.size + ref_size,<EOL>bqsr_input_bam,<EOL>bqsr_input_bai,<EOL>base_recal.rv(),<EOL>genome_ref_size)<EOL>recalibrate_reads = job.wrapJobFn(apply_bqsr_recalibration,<EOL>base_recal.rv(),<EOL>bqsr_input_bam,<EOL>bqsr_input_bai,<EOL>ref, ref_dict, fai,<EOL>unsafe=unsafe,<EOL>cores=job.cores,<EOL>disk=recalibrate_reads_disk,<EOL>memory=job.memory)<EOL>job.addChild(mdups)<EOL>mdups.addFollowOn(base_recal)<EOL>base_recal.addChild(recalibrate_reads)<EOL>return recalibrate_reads.rv(<NUM_LIT:0>), recalibrate_reads.rv(<NUM_LIT:1>)<EOL>

GATK Preprocessing Pipeline
0: Mark duplicates
1: Create INDEL realignment intervals
2: Realign INDELs
3: Recalibrate base quality scores
4: Apply base score recalibration

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str ref: FileStoreID for reference genome fasta file
:param str ref_dict: FileStoreID for reference sequence dictionary file
:param str fai: FileStoreID for reference fasta index file
:param str g1k: FileStoreID for 1000 Genomes VCF file
:param str mills: FileStoreID for Mills VCF file
:param str dbsnp: FileStoreID for dbSNP VCF file
:param bool realign: If True, then runs GATK INDEL realignment"
:param bool unsafe: If True, runs GATK tools in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
:return: FileStoreIDs for BAM and BAI files
:rtype: tuple(str, str)


def run_cutadapt(job, r1_id, r2_id, fwd_3pr_adapter, rev_3pr_adapter):

work_dir = job.fileStore.getLocalTempDir()<EOL>if r2_id:<EOL><INDENT>require(rev_3pr_adapter, "<STR_LIT>")<EOL><DEDENT>parameters = ['<STR_LIT>', fwd_3pr_adapter,<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>if r1_id and r2_id:<EOL><INDENT>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', rev_3pr_adapter,<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir, parameters=parameters)<EOL>if r1_id and r2_id:<EOL><INDENT>r1_cut_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>r2_cut_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>r1_cut_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>r2_cut_id = None<EOL><DEDENT>return r1_cut_id, r2_cut_id<EOL>

Adapter trimming for RNA-seq data

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str r1_id: FileStoreID of fastq read 1
:param str r2_id: FileStoreID of fastq read 2 (if paired data)
:param str fwd_3pr_adapter: Adapter sequence for the forward 3' adapter
:param str rev_3pr_adapter: Adapter sequence for the reverse 3' adapter (second fastq pair)
:return: R1 and R2 FileStoreIDs
:rtype: tuple


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(sam, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']<EOL>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>')<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Marks reads as PCR duplicates using SAMBLASTER

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str sam: FileStoreID for SAM file
:return: FileStoreID for deduped SAM file
:rtype: str


def run_indel_realignment(job, intervals, bam, bai, ref, ref_dict, fai, g1k, mills, unsafe=False):

inputs = {'<STR_LIT>': ref,<EOL>'<STR_LIT>': fai,<EOL>'<STR_LIT>': ref_dict,<EOL>'<STR_LIT>': bam,<EOL>'<STR_LIT>': bai,<EOL>'<STR_LIT>': intervals,<EOL>'<STR_LIT>': g1k,<EOL>'<STR_LIT>': mills}<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(<NUM_LIT>),  <EOL>'<STR_LIT>', str(<NUM_LIT>),  <EOL>'<STR_LIT>', '<STR_LIT>']<EOL>if unsafe:<EOL><INDENT>parameters.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>docker_parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>'.format(work_dir)]<EOL>start_time = time.time()<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir,<EOL>parameters=parameters,<EOL>dockerParameters=docker_parameters)<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>indel_bam = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>indel_bai = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>return indel_bam, indel_bai<EOL>

Realigns BAM file at realignment target intervals

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str intervals: FileStoreID for INDEL realignment intervals file
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str ref: FileStoreID for reference genome fasta file
:param str ref_dict: FileStoreID for reference sequence dictionary file
:param str fai: FileStoreID for reference fasta index file
:param str g1k: FileStoreID for 1000 Genomes VCF file
:param str mills: FileStoreID for Mills VCF file
:param bool unsafe: If True, runs GATK in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
:return: FileStoreIDs for realigned BAM and BAI files
:rtype: tuple(str, str)


def picard_mark_duplicates(job, bam, bai, validation_stringency='<STR_LIT>'):

work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>job.fileStore.readGlobalFile(bai, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % validation_stringency.upper()]<EOL>docker_parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>'.format(work_dir)]<EOL>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>',<EOL>dockerParameters=docker_parameters)<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>bam = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>bai = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>return bam, bai<EOL>

Runs Picard MarkDuplicates on a BAM file. Requires that the BAM file be coordinate sorted.

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str validation_stringency: BAM file validation stringency, default is LENIENT
:return: FileStoreIDs for BAM and BAI files
:rtype: tuple


def run_rsem_postprocess(job, rsem_gene_id, rsem_isoform_id):

work_dir = job.fileStore.getLocalTempDir()<EOL>genes = job.fileStore.readGlobalFile(rsem_gene_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>iso = job.fileStore.readGlobalFile(rsem_isoform_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>parameters=command, workDir=work_dir)<EOL>hugo_files = [os.path.join(work_dir, x) for x in ['<STR_LIT>', '<STR_LIT>']]<EOL>tarball_files('<STR_LIT>', file_paths=[os.path.join(work_dir, x) for x in [genes, iso]], output_dir=work_dir)<EOL>tarball_files('<STR_LIT>', file_paths=[os.path.join(work_dir, x) for x in hugo_files], output_dir=work_dir)<EOL>rsem_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>hugo_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>return rsem_id, hugo_id<EOL>

Parses RSEMs output to produce the separate .tab files (TPM, FPKM, counts) for both gene and isoform.
These are two-column files: Genes and Quantifications.
HUGO files are also provided that have been mapped from Gencode/ENSEMBLE names.

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str rsem_gene_id: FileStoreID of rsem_gene_ids
:param str rsem_isoform_id: FileStoreID of rsem_isoform_ids
:return: FileStoreID from RSEM post process tarball
:rytpe: str


def run_rsem(job, bam_id, rsem_ref_url, paired=True):

work_dir = job.fileStore.getLocalTempDir()<EOL>download_url(job, url=rsem_ref_url, name='<STR_LIT>', work_dir=work_dir)<EOL>subprocess.check_call(['<STR_LIT>', '<STR_LIT>', os.path.join(work_dir, '<STR_LIT>'), '<STR_LIT>', work_dir])<EOL>os.remove(os.path.join(work_dir, '<STR_LIT>'))<EOL>rsem_files = []<EOL>for root, directories, files in os.walk(work_dir):<EOL><INDENT>rsem_files.extend([os.path.join(root, x) for x in files])<EOL><DEDENT>ref_prefix = [os.path.basename(os.path.splitext(x)[<NUM_LIT:0>]) for x in rsem_files if '<STR_LIT>' in x][<NUM_LIT:0>]<EOL>ref_folder = os.path.join('<STR_LIT>', os.listdir(work_dir)[<NUM_LIT:0>]) if len(os.listdir(work_dir)) == <NUM_LIT:1> else '<STR_LIT>'<EOL>job.fileStore.readGlobalFile(bam_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>output_prefix = '<STR_LIT>'<EOL>parameters = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', str(job.cores),<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>os.path.join(ref_folder, ref_prefix),<EOL>output_prefix]<EOL>if paired:<EOL><INDENT>parameters = ['<STR_LIT>'] + parameters<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>parameters=parameters, workDir=work_dir)<EOL>gene_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, output_prefix + '<STR_LIT>'))<EOL>isoform_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, output_prefix + '<STR_LIT>'))<EOL>return gene_id, isoform_id<EOL>

RNA quantification with RSEM

:param JobFunctionWrappingJob job: Passed automatically by Toil
:param str bam_id: FileStoreID of transcriptome bam for quantification
:param str rsem_ref_url: URL of RSEM reference (tarball)
:param bool paired: If True, uses parameters for paired end data
:return: FileStoreIDs for RSEM's gene and isoform output
:rtype: str


def run_mutect(job, normal_bam, normal_bai, tumor_bam, tumor_bai, ref, ref_dict, fai, cosmic, dbsnp):

work_dir = job.fileStore.getLocalTempDir()<EOL>file_ids = [normal_bam, normal_bai, tumor_bam, tumor_bai, ref, fai, ref_dict, cosmic, dbsnp]<EOL>file_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for file_store_id, name in zip(file_ids, file_names):<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(<NUM_LIT:10>),  <EOL>'<STR_LIT>', str(<NUM_LIT>),  <EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>dockerCall(job=job, workDir=work_dir, parameters=parameters,<EOL>tool='<STR_LIT>')<EOL>output_file_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>output_file_paths = [os.path.join(work_dir, x) for x in output_file_names]<EOL>tarball_files('<STR_LIT>', file_paths=output_file_paths, output_dir=work_dir)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Calls MuTect to perform variant analysis

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str normal_bam: Normal BAM FileStoreID
:param str normal_bai: Normal BAM index FileStoreID
:param str tumor_bam: Tumor BAM FileStoreID
:param str tumor_bai: Tumor BAM Index FileStoreID
:param str ref: Reference genome FileStoreID
:param str ref_dict: Reference dictionary FileStoreID
:param str fai: Reference index FileStoreID
:param str cosmic: Cosmic VCF FileStoreID
:param str dbsnp: DBSNP VCF FileStoreID
:return: MuTect output (tarball) FileStoreID
:rtype: str


def gatk_combine_variants(job, vcfs, ref_fasta, ref_fai, ref_dict, merge_option='<STR_LIT>'):

job.fileStore.logToMaster('<STR_LIT>')<EOL>inputs = {'<STR_LIT>': ref_fasta,<EOL>'<STR_LIT>': ref_fai,<EOL>'<STR_LIT>': ref_dict}<EOL>inputs.update(vcfs)<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>command = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', merge_option]<EOL>for uuid, vcf_id in vcfs.iteritems():<EOL><INDENT>command.extend(['<STR_LIT>', os.path.join('<STR_LIT>', uuid)])<EOL><DEDENT>docker_parameters = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory)]<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>',<EOL>dockerParameters=docker_parameters)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Merges VCF files using GATK CombineVariants

:param JobFunctionWrappingJob job: Toil Job instance
:param dict vcfs: Dictionary of VCF FileStoreIDs {sample identifier: FileStoreID}
:param str ref_fasta: FileStoreID for reference genome fasta
:param str ref_fai: FileStoreID for reference genome index file
:param str ref_dict: FileStoreID for reference genome sequence dictionary file
:param str merge_option: Value for --genotypemergeoption flag (Default: 'UNIQUIFY')
                        'UNIQUIFY': Multiple variants at a single site are merged into a
                                    single variant record.
                        'UNSORTED': Used to merge VCFs from the same sample
:return: FileStoreID for merged VCF file
:rtype: str


def run_oncotator(job, vcf_id, oncotator_db):

job.fileStore.logToMaster('<STR_LIT>')<EOL>inputs = {'<STR_LIT>': vcf_id,<EOL>'<STR_LIT>': oncotator_db}<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>inputs[name] = job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>if tarfile.is_tarfile(inputs['<STR_LIT>']):<EOL><INDENT>tar = tarfile.open(inputs['<STR_LIT>'])<EOL>tar.extractall(path=work_dir)<EOL>inputs['<STR_LIT>'] = tar.getmembers()[<NUM_LIT:0>].name<EOL>tar.close()<EOL><DEDENT>command = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', inputs['<STR_LIT>'],<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']  <EOL>docker_parameters = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory)]<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>',<EOL>dockerParameters=docker_parameters)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Uses Oncotator to add cancer relevant variant annotations to a VCF file. Oncotator can accept
other genome builds, but the output VCF is based on hg19.

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str vcf_id: FileStoreID for VCF file
:param str oncotator_db: FileStoreID for Oncotator database
:return: Annotated VCF FileStoreID
:rtype: str


__forall_files(file_paths, output_dir, shutil.copy)<EOL>

Moves files from the working directory to the output directory.

:param str output_dir: Output directory
:param list[str] file_paths: Absolute file paths to move


Launches the Hadoop datanode.

:param job: The underlying job.


result = []<EOL>for el in x:<EOL><INDENT>if hasattr(el, "<STR_LIT>") and not isinstance(el, basestring):<EOL><INDENT>result.extend(flatten(el))<EOL><DEDENT>else:<EOL><INDENT>result.append(el)<EOL><DEDENT><DEDENT>return result<EOL>

Flattens a nested array into a single list

:param list x: The nested list/tuple to be flattened.


class RequiredLength(argparse.Action):<EOL><INDENT>def __call__(self, parser, args, values, option_string=None):<EOL><INDENT>if not nmin <= len(values) <= nmax:<EOL><INDENT>msg = '<STR_LIT>'.format(<EOL>f=self.dest, nmin=nmin, nmax=nmax)<EOL>raise argparse.ArgumentTypeError(msg)<EOL><DEDENT>setattr(args, self.dest, values)<EOL><DEDENT><DEDENT>return RequiredLength<EOL>

For use with argparse's action argument. Allows setting a range for nargs.
Example: nargs='+', action=required_length(2, 3)

:param int nmin: Minimum number of arguments
:param int nmax: Maximum number of arguments
:return: RequiredLength object


for i in xrange(<NUM_LIT:0>, len(l), partition_size):<EOL><INDENT>yield l[i:i + partition_size]<EOL><DEDENT>


def download_url(job, url, work_dir='<STR_LIT:.>', name=None, s3_key_path=None, cghub_key_path=None):

file_path = os.path.join(work_dir, name) if name else os.path.join(work_dir, os.path.basename(url))<EOL>if cghub_key_path:<EOL><INDENT>_download_with_genetorrent(job, url, file_path, cghub_key_path)<EOL><DEDENT>elif urlparse(url).scheme == '<STR_LIT>':<EOL><INDENT>_s3am_with_retry(job, num_cores=<NUM_LIT:1>, file_path=file_path, s3_url=url, mode='<STR_LIT>', s3_key_path=s3_key_path)<EOL><DEDENT>elif urlparse(url).scheme == '<STR_LIT:file>':<EOL><INDENT>shutil.copy(urlparse(url).path, file_path)<EOL><DEDENT>else:<EOL><INDENT>subprocess.check_call(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:5>', '<STR_LIT>', url, '<STR_LIT>', file_path])<EOL><DEDENT>assert os.path.exists(file_path)<EOL>return file_path<EOL>

Downloads URL, can pass in file://, http://, s3://, or ftp://, gnos://cghub/analysisID, or gnos:///analysisID
If downloading S3 URLs, the S3AM binary must be on the PATH

:param toil.job.Job job: Toil job that is calling this function
:param str url: URL to download from
:param str work_dir: Directory to download file to
:param str name: Name of output file, if None, basename of URL is used
:param str s3_key_path: Path to 32-byte encryption key if url points to S3 file that uses SSE-C
:param str cghub_key_path: Path to cghub key used to download from CGHub.
:return: Path to the downloaded file
:rtype: str


return '<STR_LIT>' % (os.getcwd(), self._name)<EOL>

Returns the path of a pipeline config file, without regard for its existence.


def _create_pipeline_command(self, args, workdir_path, config_path):

return ([self._name, '<STR_LIT>', os.path.join(workdir_path, '<STR_LIT>'),<EOL>'<STR_LIT>', config_path,<EOL>'<STR_LIT>', workdir_path, '<STR_LIT>', '<STR_LIT:1>']<EOL>+ (['<STR_LIT>'] if args.restart else []))<EOL>

Creates and returns a list that represents a command for running the pipeline.


def _add_option(self, arg_parser, name, *args, **kwargs):

arg_parser.add_argument('<STR_LIT>' + name, *args, **kwargs)<EOL>

Add an argument to the given arg_parser with the given name.

:param argparse.ArgumentParser arg_parser:
:param str name: The name of the option.


return cls.__run(cls, ctx)<EOL>


def __init__(self, x=<NUM_LIT:0>, y=<NUM_LIT:0>, z=<NUM_LIT:0>):

self.x = x<EOL>self.y = y<EOL>self.z = z<EOL>

Create a new Vertex representing the position (x, y, z).


<EOL>axes = [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>]<EOL>if self.v0.x == self.v1.x == self.v2.x:<EOL><INDENT>axes[<NUM_LIT:0>] = <NUM_LIT:0><EOL><DEDENT>if self.v0.y == self.v1.y == self.v2.y:<EOL><INDENT>axes[<NUM_LIT:1>] = <NUM_LIT:0><EOL><DEDENT>if self.v0.z == self.v1.z == self.v2.z:<EOL><INDENT>axes[<NUM_LIT:2>] = <NUM_LIT:0><EOL><DEDENT>u = [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]<EOL>for i in range(<NUM_LIT:3>):<EOL><INDENT>if axes[i] == <NUM_LIT:1>:<EOL><INDENT>u[i] = <NUM_LIT:1><EOL>axes[i] = <NUM_LIT:0><EOL>break<EOL><DEDENT><DEDENT>v = [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]<EOL>for i in range(<NUM_LIT:3>):<EOL><INDENT>if axes[i] == <NUM_LIT:1>:<EOL><INDENT>v[i] = -<NUM_LIT:1><EOL>break<EOL><DEDENT><DEDENT>uaxis = Axis(u[<NUM_LIT:0>], u[<NUM_LIT:1>], u[<NUM_LIT:2>])<EOL>vaxis = Axis(v[<NUM_LIT:0>], v[<NUM_LIT:1>], v[<NUM_LIT:2>])<EOL>return (uaxis, vaxis)<EOL>

Returns a sensible uaxis and vaxis for this plane.


return self.brush.children[<NUM_LIT:0>]<EOL>


return Template(template).render(RequestContext({}, kwargs)).strip()<EOL>

Return the rendering of a given template


if arg == u'<STR_LIT>':<EOL><INDENT>directions = (u'<STR_LIT>', u'<STR_LIT>')<EOL><DEDENT>elif arg == u'<STR_LIT>':<EOL><INDENT>directions = (u'<STR_LIT>', u'<STR_LIT>')<EOL><DEDENT>elif arg == u'<STR_LIT>':<EOL><INDENT>directions = (u'<STR_LIT>', u'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise template.TemplateSyntaxError('<STR_LIT>')<EOL><DEDENT>parts = value.rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>if not len(parts):<EOL><INDENT>return value<EOL><DEDENT>elif len(parts) == <NUM_LIT:1>:<EOL><INDENT>return value + directions[translation.get_language_bidi()]<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:.>'.join((parts[<NUM_LIT:0>]+directions[translation.get_language_bidi()],parts[<NUM_LIT:1>]))<EOL><DEDENT>

Adds direction to the element

    :arguments:
        arg
            * rtl_only: Add the direction only in case of a
              right-to-left language (default)
            * both: add the direction in both case
            * ltr_only: Add the direction only in case of a
              left-to-right language

    {{image_name|add_direction}} when image_name is 'start_arrow.png'
    results in 'start_arrow_rtl.png' in case of RTL language, and
    'start_arrow.png' or 'start_arrow_ltr.png' depends on `arg` value.


def accuracy(output, target, topk=(<NUM_LIT:1>,)):

with torch.no_grad():<EOL><INDENT>maxk = max(topk)<EOL>batch_size = target.size(<NUM_LIT:0>)<EOL>_, pred = output.topk(maxk, <NUM_LIT:1>, True, True)<EOL>pred = pred.t()<EOL>correct = pred.eq(target[None])<EOL>res = []<EOL>for k in topk:<EOL><INDENT>correct_k = correct[:k].flatten().sum(dtype=torch.float32)<EOL>res.append(correct_k * (<NUM_LIT> / batch_size))<EOL><DEDENT>return res<EOL><DEDENT>

Computes the accuracy over the k top predictions for the specified values of k


i, j, h, w = self.get_params(img, self.scale, self.ratio)<EOL>return F.resized_crop(img, i, j, h, w, self.size, self.interpolation)<EOL>

Args:
    img (PIL Image): Image to be cropped and resized.

Returns:
    PIL Image: Randomly cropped and resized image.


angle = random.uniform(degrees[<NUM_LIT:0>], degrees[<NUM_LIT:1>])<EOL>return angle<EOL>

Get parameters for ``rotate`` for a random rotation.

        Returns:
            sequence: params to be passed to ``rotate`` for random rotation.


return F.to_grayscale(img, num_output_channels=self.num_output_channels)<EOL>

Args:
    img (PIL Image): Image to be converted to grayscale.

Returns:
    PIL Image: Randomly grayscaled image.


return F.to_pil_image(pic, self.mode)<EOL>

Args:
    pic (Tensor or numpy.ndarray): Image to be converted to PIL Image.

Returns:
    PIL Image: Image converted to PIL Image.


if random.random() < self.p:<EOL><INDENT>return F.hflip(img)<EOL><DEDENT>return img<EOL>

Args:
    img (PIL Image): Image to be flipped.

Returns:
    PIL Image: Randomly flipped image.


@staticmethod<EOL><INDENT>def get_params(width, height, distortion_scale):<DEDENT>

half_height = int(height / <NUM_LIT:2>)<EOL>half_width = int(width / <NUM_LIT:2>)<EOL>topleft = (random.randint(<NUM_LIT:0>, int(distortion_scale * half_width)),<EOL>random.randint(<NUM_LIT:0>, int(distortion_scale * half_height)))<EOL>topright = (random.randint(width - int(distortion_scale * half_width) - <NUM_LIT:1>, width - <NUM_LIT:1>),<EOL>random.randint(<NUM_LIT:0>, int(distortion_scale * half_height)))<EOL>botright = (random.randint(width - int(distortion_scale * half_width) - <NUM_LIT:1>, width - <NUM_LIT:1>),<EOL>random.randint(height - int(distortion_scale * half_height) - <NUM_LIT:1>, height - <NUM_LIT:1>))<EOL>botleft = (random.randint(<NUM_LIT:0>, int(distortion_scale * half_width)),<EOL>random.randint(height - int(distortion_scale * half_height) - <NUM_LIT:1>, height - <NUM_LIT:1>))<EOL>startpoints = [(<NUM_LIT:0>, <NUM_LIT:0>), (width - <NUM_LIT:1>, <NUM_LIT:0>), (width - <NUM_LIT:1>, height - <NUM_LIT:1>), (<NUM_LIT:0>, height - <NUM_LIT:1>)]<EOL>endpoints = [topleft, topright, botright, botleft]<EOL>return startpoints, endpoints<EOL>

Get parameters for ``perspective`` for a random perspective transform.

        Args:
            width : width of the image.
            height : height of the image.

        Returns:
            List containing [top-left, top-right, bottom-right, bottom-left] of the orignal image,
            List containing [top-left, top-right, bottom-right, bottom-left] of the transformed image.


@staticmethod<EOL><INDENT>def get_params(brightness, contrast, saturation, hue):<DEDENT>

transforms = []<EOL>if brightness is not None:<EOL><INDENT>brightness_factor = random.uniform(brightness[<NUM_LIT:0>], brightness[<NUM_LIT:1>])<EOL>transforms.append(Lambda(lambda img: F.adjust_brightness(img, brightness_factor)))<EOL><DEDENT>if contrast is not None:<EOL><INDENT>contrast_factor = random.uniform(contrast[<NUM_LIT:0>], contrast[<NUM_LIT:1>])<EOL>transforms.append(Lambda(lambda img: F.adjust_contrast(img, contrast_factor)))<EOL><DEDENT>if saturation is not None:<EOL><INDENT>saturation_factor = random.uniform(saturation[<NUM_LIT:0>], saturation[<NUM_LIT:1>])<EOL>transforms.append(Lambda(lambda img: F.adjust_saturation(img, saturation_factor)))<EOL><DEDENT>if hue is not None:<EOL><INDENT>hue_factor = random.uniform(hue[<NUM_LIT:0>], hue[<NUM_LIT:1>])<EOL>transforms.append(Lambda(lambda img: F.adjust_hue(img, hue_factor)))<EOL><DEDENT>random.shuffle(transforms)<EOL>transform = Compose(transforms)<EOL>return transform<EOL>

Get a randomized transform to be applied on image.

        Arguments are same as that of __init__.

        Returns:
            Transform which randomly adjusts brightness, contrast and
            saturation in a random order.


def resize(img, size, interpolation=Image.BILINEAR):

if not _is_pil_image(img):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(img)))<EOL><DEDENT>if not (isinstance(size, int) or (isinstance(size, Iterable) and len(size) == <NUM_LIT:2>)):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(size))<EOL><DEDENT>if isinstance(size, int):<EOL><INDENT>w, h = img.size<EOL>if (w <= h and w == size) or (h <= w and h == size):<EOL><INDENT>return img<EOL><DEDENT>if w < h:<EOL><INDENT>ow = size<EOL>oh = int(size * h / w)<EOL>return img.resize((ow, oh), interpolation)<EOL><DEDENT>else:<EOL><INDENT>oh = size<EOL>ow = int(size * w / h)<EOL>return img.resize((ow, oh), interpolation)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return img.resize(size[::-<NUM_LIT:1>], interpolation)<EOL><DEDENT>

r"""Resize the input PIL Image to the given size.

    Args:
        img (PIL Image): Image to be resized.
        size (sequence or int): Desired output size. If size is a sequence like
            (h, w), the output size will be matched to this. If size is an int,
            the smaller edge of the image will be matched to this number maintaing
            the aspect ratio. i.e, if height > width, then image will be rescaled to
            :math:`\left(\text{size} \times \frac{\text{height}}{\text{width}}, \text{size}\right)`
        interpolation (int, optional): Desired interpolation. Default is
            ``PIL.Image.BILINEAR``

    Returns:
        PIL Image: Resized image.


def pad(img, padding, fill=<NUM_LIT:0>, padding_mode='<STR_LIT>'):

if not _is_pil_image(img):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(img)))<EOL><DEDENT>if not isinstance(padding, (numbers.Number, tuple)):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>if not isinstance(fill, (numbers.Number, str, tuple)):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>if not isinstance(padding_mode, str):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>if isinstance(padding, Sequence) and len(padding) not in [<NUM_LIT:2>, <NUM_LIT:4>]:<EOL><INDENT>raise ValueError("<STR_LIT>" +<EOL>"<STR_LIT>".format(len(padding)))<EOL><DEDENT>assert padding_mode in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'],'<STR_LIT>'<EOL>if padding_mode == '<STR_LIT>':<EOL><INDENT>if img.mode == '<STR_LIT:P>':<EOL><INDENT>palette = img.getpalette()<EOL>image = ImageOps.expand(img, border=padding, fill=fill)<EOL>image.putpalette(palette)<EOL>return image<EOL><DEDENT>return ImageOps.expand(img, border=padding, fill=fill)<EOL><DEDENT>else:<EOL><INDENT>if isinstance(padding, int):<EOL><INDENT>pad_left = pad_right = pad_top = pad_bottom = padding<EOL><DEDENT>if isinstance(padding, Sequence) and len(padding) == <NUM_LIT:2>:<EOL><INDENT>pad_left = pad_right = padding[<NUM_LIT:0>]<EOL>pad_top = pad_bottom = padding[<NUM_LIT:1>]<EOL><DEDENT>if isinstance(padding, Sequence) and len(padding) == <NUM_LIT:4>:<EOL><INDENT>pad_left = padding[<NUM_LIT:0>]<EOL>pad_top = padding[<NUM_LIT:1>]<EOL>pad_right = padding[<NUM_LIT:2>]<EOL>pad_bottom = padding[<NUM_LIT:3>]<EOL><DEDENT>if img.mode == '<STR_LIT:P>':<EOL><INDENT>palette = img.getpalette()<EOL>img = np.asarray(img)<EOL>img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right)), padding_mode)<EOL>img = Image.fromarray(img)<EOL>img.putpalette(palette)<EOL>return img<EOL><DEDENT>img = np.asarray(img)<EOL>if len(img.shape) == <NUM_LIT:3>:<EOL><INDENT>img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right), (<NUM_LIT:0>, <NUM_LIT:0>)), padding_mode)<EOL><DEDENT>if len(img.shape) == <NUM_LIT:2>:<EOL><INDENT>img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right)), padding_mode)<EOL><DEDENT>return Image.fromarray(img)<EOL><DEDENT>

r"""Pad the given PIL Image on all sides with specified padding mode and fill value.

    Args:
        img (PIL Image): Image to be padded.
        padding (int or tuple): Padding on each border. If a single int is provided this
            is used to pad all borders. If tuple of length 2 is provided this is the padding
            on left/right and top/bottom respectively. If a tuple of length 4 is provided
            this is the padding for the left, top, right and bottom borders
            respectively.
        fill: Pixel fill value for constant fill. Default is 0. If a tuple of
            length 3, it is used to fill R, G, B channels respectively.
            This value is only used when the padding_mode is constant
        padding_mode: Type of padding. Should be: constant, edge, reflect or symmetric. Default is constant.

            - constant: pads with a constant value, this value is specified with fill

            - edge: pads with the last value on the edge of the image

            - reflect: pads with reflection of image (without repeating the last value on the edge)

                       padding [1, 2, 3, 4] with 2 elements on both sides in reflect mode
                       will result in [3, 2, 1, 2, 3, 4, 3, 2]

            - symmetric: pads with reflection of image (repeating the last value on the edge)

                         padding [1, 2, 3, 4] with 2 elements on both sides in symmetric mode
                         will result in [2, 1, 1, 2, 3, 4, 4, 3]

    Returns:
        PIL Image: Padded image.


if isinstance(size, numbers.Number):<EOL><INDENT>size = (int(size), int(size))<EOL><DEDENT>else:<EOL><INDENT>assert len(size) == <NUM_LIT:2>, "<STR_LIT>"<EOL><DEDENT>w, h = img.size<EOL>crop_h, crop_w = size<EOL>if crop_w > w or crop_h > h:<EOL><INDENT>raise ValueError("<STR_LIT>".format(size,<EOL>(h, w)))<EOL><DEDENT>tl = img.crop((<NUM_LIT:0>, <NUM_LIT:0>, crop_w, crop_h))<EOL>tr = img.crop((w - crop_w, <NUM_LIT:0>, w, crop_h))<EOL>bl = img.crop((<NUM_LIT:0>, h - crop_h, crop_w, h))<EOL>br = img.crop((w - crop_w, h - crop_h, w, h))<EOL>center = center_crop(img, (crop_h, crop_w))<EOL>return (tl, tr, bl, br, center)<EOL>

Crop the given PIL Image into four corners and the central crop.

    .. Note::
        This transform returns a tuple of images and there may be a
        mismatch in the number of inputs and targets your ``Dataset`` returns.

    Args:
       size (sequence or int): Desired output size of the crop. If size is an
           int instead of sequence like (h, w), a square crop (size, size) is
           made.

    Returns:
       tuple: tuple (tl, tr, bl, br, center)
                Corresponding top left, top right, bottom left, bottom right and center crop.


def resized_crop(img, i, j, h, w, size, interpolation=Image.BILINEAR):

assert _is_pil_image(img), '<STR_LIT>'<EOL>img = crop(img, i, j, h, w)<EOL>img = resize(img, size, interpolation)<EOL>return img<EOL>

Crop the given PIL Image and resize it to desired size.

    Notably used in :class:`~torchvision.transforms.RandomResizedCrop`.

    Args:
        img (PIL Image): Image to be cropped.
        i (int): i in (i,j) i.e coordinates of the upper left corner
        j (int): j in (i,j) i.e coordinates of the upper left corner
        h (int): Height of the cropped image.
        w (int): Width of the cropped image.
        size (sequence or int): Desired output size. Same semantics as ``resize``.
        interpolation (int, optional): Desired interpolation. Default is
            ``PIL.Image.BILINEAR``.
    Returns:
        PIL Image: Cropped image.


if not _is_pil_image(img):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(img)))<EOL><DEDENT>enhancer = ImageEnhance.Color(img)<EOL>img = enhancer.enhance(saturation_factor)<EOL>return img<EOL>

Adjust color saturation of an image.

    Args:
        img (PIL Image): PIL Image to be adjusted.
        saturation_factor (float):  How much to adjust the saturation. 0 will
            give a black and white image, 1 will give the original image while
            2 will enhance the saturation by a factor of 2.

    Returns:
        PIL Image: Saturation adjusted image.


if not(isinstance(pic, torch.Tensor) or isinstance(pic, np.ndarray)):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(pic)))<EOL><DEDENT>elif isinstance(pic, torch.Tensor):<EOL><INDENT>if pic.ndimension() not in {<NUM_LIT:2>, <NUM_LIT:3>}:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(pic.ndimension()))<EOL><DEDENT>elif pic.ndimension() == <NUM_LIT:2>:<EOL><INDENT>pic = pic.unsqueeze(<NUM_LIT:0>)<EOL><DEDENT><DEDENT>elif isinstance(pic, np.ndarray):<EOL><INDENT>if pic.ndim not in {<NUM_LIT:2>, <NUM_LIT:3>}:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(pic.ndim))<EOL><DEDENT>elif pic.ndim == <NUM_LIT:2>:<EOL><INDENT>pic = np.expand_dims(pic, <NUM_LIT:2>)<EOL><DEDENT><DEDENT>npimg = pic<EOL>if isinstance(pic, torch.FloatTensor):<EOL><INDENT>pic = pic.mul(<NUM_LIT:255>).byte()<EOL><DEDENT>if isinstance(pic, torch.Tensor):<EOL><INDENT>npimg = np.transpose(pic.numpy(), (<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:0>))<EOL><DEDENT>if not isinstance(npimg, np.ndarray):<EOL><INDENT>raise TypeError('<STR_LIT>' +<EOL>'<STR_LIT>'.format(type(npimg)))<EOL><DEDENT>if npimg.shape[<NUM_LIT:2>] == <NUM_LIT:1>:<EOL><INDENT>expected_mode = None<EOL>npimg = npimg[:, :, <NUM_LIT:0>]<EOL>if npimg.dtype == np.uint8:<EOL><INDENT>expected_mode = '<STR_LIT:L>'<EOL><DEDENT>elif npimg.dtype == np.int16:<EOL><INDENT>expected_mode = '<STR_LIT>'<EOL><DEDENT>elif npimg.dtype == np.int32:<EOL><INDENT>expected_mode = '<STR_LIT:I>'<EOL><DEDENT>elif npimg.dtype == np.float32:<EOL><INDENT>expected_mode = '<STR_LIT:F>'<EOL><DEDENT>if mode is not None and mode != expected_mode:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>.format(mode, np.dtype, expected_mode))<EOL><DEDENT>mode = expected_mode<EOL><DEDENT>elif npimg.shape[<NUM_LIT:2>] == <NUM_LIT:2>:<EOL><INDENT>permitted_2_channel_modes = ['<STR_LIT>']<EOL>if mode is not None and mode not in permitted_2_channel_modes:<EOL><INDENT>raise ValueError("<STR_LIT>".format(permitted_2_channel_modes))<EOL><DEDENT>if mode is None and npimg.dtype == np.uint8:<EOL><INDENT>mode = '<STR_LIT>'<EOL><DEDENT><DEDENT>elif npimg.shape[<NUM_LIT:2>] == <NUM_LIT:4>:<EOL><INDENT>permitted_4_channel_modes = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if mode is not None and mode not in permitted_4_channel_modes:<EOL><INDENT>raise ValueError("<STR_LIT>".format(permitted_4_channel_modes))<EOL><DEDENT>if mode is None and npimg.dtype == np.uint8:<EOL><INDENT>mode = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>permitted_3_channel_modes = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if mode is not None and mode not in permitted_3_channel_modes:<EOL><INDENT>raise ValueError("<STR_LIT>".format(permitted_3_channel_modes))<EOL><DEDENT>if mode is None and npimg.dtype == np.uint8:<EOL><INDENT>mode = '<STR_LIT>'<EOL><DEDENT><DEDENT>if mode is None:<EOL><INDENT>raise TypeError('<STR_LIT>'.format(npimg.dtype))<EOL><DEDENT>return Image.fromarray(npimg, mode=mode)<EOL>

Convert a tensor or an ndarray to PIL Image.

    See :class:`~torchvision.transforms.ToPILImage` for more details.

    Args:
        pic (Tensor or numpy.ndarray): Image to be converted to PIL Image.
        mode (`PIL.Image mode`_): color space and pixel depth of input data (optional).

    .. _PIL.Image mode: https://pillow.readthedocs.io/en/latest/handbook/concepts.html#concept-modes

    Returns:
        PIL Image: Image converted to PIL Image.


if not _is_pil_image(img):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(img)))<EOL><DEDENT>if num_output_channels == <NUM_LIT:1>:<EOL><INDENT>img = img.convert('<STR_LIT:L>')<EOL><DEDENT>elif num_output_channels == <NUM_LIT:3>:<EOL><INDENT>img = img.convert('<STR_LIT:L>')<EOL>np_img = np.array(img, dtype=np.uint8)<EOL>np_img = np.dstack([np_img, np_img, np_img])<EOL>img = Image.fromarray(np_img, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>return img<EOL>

Convert image to grayscale version of image.

    Args:
        img (PIL Image): Image to be converted to grayscale.

    Returns:
        PIL Image: Grayscale version of the image.
            if num_output_channels = 1 : returned image is single channel

            if num_output_channels = 3 : returned image is 3 channel with r = g = b


def conv3x3(in_planes, out_planes, stride=<NUM_LIT:1>, groups=<NUM_LIT:1>, dilation=<NUM_LIT:1>):

return nn.Conv2d(in_planes, out_planes, kernel_size=<NUM_LIT:3>, stride=stride,<EOL>padding=dilation, groups=groups, bias=False, dilation=dilation)<EOL>


model = AlexNet(**kwargs)<EOL>if pretrained:<EOL><INDENT>model.load_state_dict(model_zoo.load_url(model_urls['<STR_LIT>']))<EOL><DEDENT>return model<EOL>

r"""AlexNet model architecture from the
    `"One weird trick..." <https://arxiv.org/abs/1404.5997>`_ paper.

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet


if pretrained:<EOL><INDENT>kwargs['<STR_LIT>'] = False<EOL><DEDENT>model = VGG(make_layers(cfg['<STR_LIT:D>']), **kwargs)<EOL>if pretrained:<EOL><INDENT>model.load_state_dict(model_zoo.load_url(model_urls['<STR_LIT>']))<EOL><DEDENT>return model<EOL>

VGG 16-layer model (configuration "D")

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet


if pretrained:<EOL><INDENT>kwargs['<STR_LIT>'] = False<EOL><DEDENT>model = VGG(make_layers(cfg['<STR_LIT:B>'], batch_norm=True), **kwargs)<EOL>if pretrained:<EOL><INDENT>model.load_state_dict(model_zoo.load_url(model_urls['<STR_LIT>']))<EOL><DEDENT>return model<EOL>

VGG 13-layer model (configuration "B") with batch normalization

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet


if pretrained:<EOL><INDENT>kwargs['<STR_LIT>'] = False<EOL><DEDENT>model = VGG(make_layers(cfg['<STR_LIT:B>']), **kwargs)<EOL>if pretrained:<EOL><INDENT>model.load_state_dict(model_zoo.load_url(model_urls['<STR_LIT>']))<EOL><DEDENT>return model<EOL>

VGG 13-layer model (configuration "B")

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet


model = DenseNet(num_init_features=<NUM_LIT>, growth_rate=<NUM_LIT>, block_config=(<NUM_LIT:6>, <NUM_LIT:12>, <NUM_LIT>, <NUM_LIT>),<EOL>**kwargs)<EOL>if pretrained:<EOL><INDENT>_load_state_dict(model, model_urls['<STR_LIT>'])<EOL><DEDENT>return model<EOL>

r"""Densenet-161 model from
    `"Densely Connected Convolutional Networks" <https://arxiv.org/pdf/1608.06993.pdf>`_

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet


matches = []<EOL>with open(os.path.join(data_dir, matches_file), '<STR_LIT:r>') as f:<EOL><INDENT>for line in f:<EOL><INDENT>line_split = line.split()<EOL>matches.append([int(line_split[<NUM_LIT:0>]), int(line_split[<NUM_LIT:3>]),<EOL>int(line_split[<NUM_LIT:1>] == line_split[<NUM_LIT:4>])])<EOL><DEDENT><DEDENT>return torch.LongTensor(matches)<EOL>

Return a Tensor containing the ground truth matches
       Read the file and keep only 3D point ID.
       Matches are represented with a 1, non matches with a 0.


if self.train:<EOL><INDENT>data = self.data[index]<EOL>if self.transform is not None:<EOL><INDENT>data = self.transform(data)<EOL><DEDENT>return data<EOL><DEDENT>m = self.matches[index]<EOL>data1, data2 = self.data[m[<NUM_LIT:0>]], self.data[m[<NUM_LIT:1>]]<EOL>if self.transform is not None:<EOL><INDENT>data1 = self.transform(data1)<EOL>data2 = self.transform(data2)<EOL><DEDENT>return data1, data2, m[<NUM_LIT:2>]<EOL>

Args:
    index (int): Index

Returns:
    tuple: (data1, data2, matches)


labels = []<EOL>with open(os.path.join(data_dir, info_file), '<STR_LIT:r>') as f:<EOL><INDENT>labels = [int(line.split()[<NUM_LIT:0>]) for line in f]<EOL><DEDENT>return torch.LongTensor(labels)<EOL>

Return a Tensor containing the list of labels
       Read the file and keep only the ID of the 3D point.


if self.labels is not None:<EOL><INDENT>img, target = self.data[index], int(self.labels[index])<EOL><DEDENT>else:<EOL><INDENT>img, target = self.data[index], None<EOL><DEDENT>img = Image.fromarray(np.transpose(img, (<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:0>)))<EOL>if self.transform is not None:<EOL><INDENT>img = self.transform(img)<EOL><DEDENT>if self.target_transform is not None:<EOL><INDENT>target = self.target_transform(target)<EOL><DEDENT>return img, target<EOL>

Args:
    index (int): Index

Returns:
    tuple: (image, target) where target is index of the target class.


root = self.root<EOL>fpath = os.path.join(root, self.filename)<EOL>if not check_integrity(fpath, self.md5_checksum):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Check the md5 checksum of the downloaded tarball.


return len(self.photos)<EOL>


import tarfile<EOL>if self._check_integrity():<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>download_url(self.url, self.root, self.filename, self.md5_checksum)<EOL>with tarfile.open(os.path.join(self.root, self.filename), '<STR_LIT>') as tar:<EOL><INDENT>tar.extractall(path=self.root)<EOL><DEDENT>with open(os.path.join(self.root, '<STR_LIT>', '<STR_LIT>')) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>url = line.rstrip()<EOL>try:<EOL><INDENT>download_url(url, os.path.join(self.root, '<STR_LIT>'))<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>

Download and extract the tarball, and download each individual photo.


def list_files(root, suffix, prefix=False):

root = os.path.expanduser(root)<EOL>files = list(<EOL>filter(<EOL>lambda p: os.path.isfile(os.path.join(root, p)) and p.endswith(suffix),<EOL>os.listdir(root)<EOL>)<EOL>)<EOL>if prefix is True:<EOL><INDENT>files = [os.path.join(root, d) for d in files]<EOL><DEDENT>return files<EOL>

List all files ending with a suffix at a given root

    Args:
        root (str): Path to directory whose folders need to be listed
        suffix (str or tuple): Suffix of the files to match, e.g. '.png' or ('.jpg', '.png').
            It uses the Python "str.endswith" method and is passed directly
        prefix (bool, optional): If true, prepends the path to each result, otherwise
            only returns the name of the files found


root = os.path.expanduser(root)<EOL>directories = list(<EOL>filter(<EOL>lambda p: os.path.isdir(os.path.join(root, p)),<EOL>os.listdir(root)<EOL>)<EOL>)<EOL>if prefix is True:<EOL><INDENT>directories = [os.path.join(root, d) for d in directories]<EOL><DEDENT>return directories<EOL>

List all directories at a given root

    Args:
        root (str): Path to directory whose folders need to be listed
        prefix (bool, optional): If true, prepends the path to each result, otherwise
            only returns the name of the directories found


def download_file_from_google_drive(file_id, root, filename=None, md5=None):

<EOL>import requests<EOL>url = "<STR_LIT>"<EOL>root = os.path.expanduser(root)<EOL>if not filename:<EOL><INDENT>filename = file_id<EOL><DEDENT>fpath = os.path.join(root, filename)<EOL>makedir_exist_ok(root)<EOL>if os.path.isfile(fpath) and check_integrity(fpath, md5):<EOL><INDENT>print('<STR_LIT>' + fpath)<EOL><DEDENT>else:<EOL><INDENT>session = requests.Session()<EOL>response = session.get(url, params={'<STR_LIT:id>': file_id}, stream=True)<EOL>token = _get_confirm_token(response)<EOL>if token:<EOL><INDENT>params = {'<STR_LIT:id>': file_id, '<STR_LIT>': token}<EOL>response = session.get(url, params=params, stream=True)<EOL><DEDENT>_save_response_content(response, fpath)<EOL><DEDENT>

Download a Google Drive file from  and place it in root.

    Args:
        file_id (str): id of file to be downloaded
        root (str): Directory to place downloaded file in
        filename (str, optional): Name to save the file under. If None, use the id of the file.
        md5 (str, optional): MD5 checksum of the download. If None, do not check


def download_url(url, root, filename=None, md5=None):

from six.moves import urllib<EOL>root = os.path.expanduser(root)<EOL>if not filename:<EOL><INDENT>filename = os.path.basename(url)<EOL><DEDENT>fpath = os.path.join(root, filename)<EOL>makedir_exist_ok(root)<EOL>if os.path.isfile(fpath) and check_integrity(fpath, md5):<EOL><INDENT>print('<STR_LIT>' + fpath)<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>print('<STR_LIT>' + url + '<STR_LIT>' + fpath)<EOL>urllib.request.urlretrieve(<EOL>url, fpath,<EOL>reporthook=gen_bar_updater()<EOL>)<EOL><DEDENT>except OSError:<EOL><INDENT>if url[:<NUM_LIT:5>] == '<STR_LIT>':<EOL><INDENT>url = url.replace('<STR_LIT>', '<STR_LIT>')<EOL>print('<STR_LIT>'<EOL>'<STR_LIT>' + url + '<STR_LIT>' + fpath)<EOL>urllib.request.urlretrieve(<EOL>url, fpath,<EOL>reporthook=gen_bar_updater()<EOL>)<EOL><DEDENT><DEDENT><DEDENT>

Download a file from a url and place it in root.

    Args:
        url (str): URL to download file from
        root (str): Directory to place downloaded file in
        filename (str, optional): Name to save the file under. If None, use the basename of the URL
        md5 (str, optional): MD5 checksum of the download. If None, do not check


path, target = self.samples[index]<EOL>sample = self.loader(path)<EOL>if self.transform is not None:<EOL><INDENT>sample = self.transform(sample)<EOL><DEDENT>if self.target_transform is not None:<EOL><INDENT>target = self.target_transform(target)<EOL><DEDENT>return sample, target<EOL>

Args:
    index (int): Index

Returns:
    tuple: (sample, target) where target is class_index of the target class.


@classmethod<EOL><INDENT>def from_str(cls, human_readable_str, decimal=False, bits=False):<DEDENT>

divisor = <NUM_LIT:1000> if decimal else <NUM_LIT><EOL>num = []<EOL>c = "<STR_LIT>"<EOL>for c in human_readable_str:<EOL><INDENT>if c not in cls.digits:<EOL><INDENT>break<EOL><DEDENT>num.append(c)<EOL><DEDENT>num = "<STR_LIT>".join(num)<EOL>try:<EOL><INDENT>num = int(num)<EOL><DEDENT>except ValueError:<EOL><INDENT>num = float(num)<EOL><DEDENT>if bits:<EOL><INDENT>num /= <NUM_LIT:8><EOL><DEDENT>return cls(round(num * divisor ** cls.key[c.lower()]))<EOL>

attempt to parse a size in bytes from a human-readable string.


def deepupdate(<EOL>mapping: abc.MutableMapping, other: abc.Mapping, listextend=False<EOL>):

def inner(other, previouskeys):<EOL><INDENT>"""<STR_LIT>"""<EOL>for key, value in other.items():<EOL><INDENT>if isinstance(value, abc.Mapping):<EOL><INDENT>inner(value, (*previouskeys, key))<EOL><DEDENT>else:<EOL><INDENT>node = mapping<EOL>for previouskey in previouskeys:<EOL><INDENT>node = node.setdefault(previouskey, {})<EOL><DEDENT>target = node.get(key)<EOL>if (<EOL>listextend<EOL>and isinstance(target, abc.MutableSequence)<EOL>and isinstance(value, abc.Sequence)<EOL>):<EOL><INDENT>target.extend(value)<EOL><DEDENT>else:<EOL><INDENT>node[key] = value<EOL><DEDENT><DEDENT><DEDENT><DEDENT>inner(other, ())<EOL>

update one dictionary from another recursively. Only individual
    values will be overwritten--not entire branches of nested
    dictionaries.


url = home_url()<EOL>if url:<EOL><INDENT>return url<EOL><DEDENT>

A template tag to return the project's home URL.

    PROJECT_HOME_NAMESPACE must be defined in settings.
    For example:
        PROJECT_HOME_NAMESPACE = 'project_name:index_view'

    Usage Example:
        {% load project_home_tags %}

        Home


resource = "<STR_LIT>"<EOL>fields = dict(grant_type="<STR_LIT>", refresh_token=self.refresh_token,<EOL>client_id=self.auth.client_id, client_secret=self.auth.client_secret,<EOL>format="<STR_LIT>")<EOL>status, data = self._handle_response("<STR_LIT:POST>", resource, fields=fields, <EOL>refresh_access_token=False)<EOL>if "<STR_LIT>" in data:<EOL><INDENT>self.access_token = data["<STR_LIT>"]<EOL>if callable(self.access_token_refreshed_callback):<EOL><INDENT>self.access_token_refreshed_callback(self.access_token)<EOL><DEDENT>return True<EOL><DEDENT>return False<EOL>

If the client application has a refresh token, it can use it to send a request for a new access token.

To ask for a new access token, the client application should send a POST request to https://login.instance_name/services/oauth2/token with the following query parameters:

    grant_type:     Value must be refresh_token for this flow.
    refresh_token:  The refresh token the client application already received.
    client_id:      Consumer key from the remote access application definition.
    client_secret:  Consumer secret from the remote access application definition.
    format:         Expected return format. This parameter is optional. The default is json. Values are:

        * urlencoded
        * json
        * xml

e.g.

    $ curl -i --form grant_type=refresh_token \
        --form refresh_token=<refresh_token> \
        --form client_id=<client_id> \
        --form client_secret=<client_secret> \
        --form format=json \
        https://na1.salesforce.com/services/oauth2/token


self.session._add_object()<EOL>self.session._out('<STR_LIT>')<EOL>self.session._out('<STR_LIT>' + self._text_to_string(<EOL>'<STR_LIT>'))<EOL>if self.title:<EOL><INDENT>self.session._out('<STR_LIT>' + self._text_to_string(self.title))<EOL><DEDENT>if self.subject:<EOL><INDENT>self.session._out('<STR_LIT>' + self._text_to_string(self.subject))<EOL><DEDENT>if self.author:<EOL><INDENT>self.session._out('<STR_LIT>' + self._text_to_string(self.author))<EOL><DEDENT>if self.keywords:<EOL><INDENT>self.session._out('<STR_LIT>' +<EOL>self._text_to_string(self.keywords))<EOL><DEDENT>if self.creator:<EOL><INDENT>self.session._out('<STR_LIT>' + self._text_to_string(self.creator))<EOL><DEDENT>self.session._out('<STR_LIT>' + self._text_to_string(<EOL>'<STR_LIT>' + datetime.now().strftime('<STR_LIT>')))<EOL>self.session._out('<STR_LIT>')<EOL>self.session._out('<STR_LIT>')<EOL>


startxref = len(self.session.buffer)<EOL>self._put_cross_reference()<EOL>md5 = hashlib.md5()<EOL>md5.update(datetime.now().strftime('<STR_LIT>'))<EOL>try:<EOL><INDENT>md5.update(self.filepath)<EOL><DEDENT>except TypeError:<EOL><INDENT>pass<EOL><DEDENT>if self.title:<EOL><INDENT>md5.update(self.title)<EOL><DEDENT>if self.subject:<EOL><INDENT>md5.update(self.subject)<EOL><DEDENT>if self.author:<EOL><INDENT>md5.update(self.author)<EOL><DEDENT>if self.keywords:<EOL><INDENT>md5.update(self.keywords)<EOL><DEDENT>if self.creator:<EOL><INDENT>md5.update(self.creator)<EOL><DEDENT>objnum = len(self.session.objects)<EOL>self.session._out('<STR_LIT>')<EOL>self.session._out('<STR_LIT>')<EOL>self.session._out('<STR_LIT>' % objnum)<EOL>self.session._out('<STR_LIT>' % (objnum - <NUM_LIT:1>))<EOL>self.session._out('<STR_LIT>' % (objnum - <NUM_LIT:2>))<EOL>self.session._out('<STR_LIT>' % (md5.hexdigest(),md5.hexdigest()))<EOL>self.session._out('<STR_LIT>')<EOL>self.session._out('<STR_LIT>')<EOL>self.session._out(startxref)<EOL>self.session._out('<STR_LIT>')<EOL>

Final Trailer calculations, and end-of-file
            reference.


self.session._add_object(<NUM_LIT:2>)<EOL>self.session._out('<STR_LIT>')<EOL>self.session._out('<STR_LIT>')<EOL>self.session._out('<STR_LIT>')<EOL>for font in self.document.fonts:<EOL><INDENT>self.session._out('<STR_LIT>' % (font.index, font.number))<EOL><DEDENT>self.session._out('<STR_LIT>')<EOL>if self.document.images:<EOL><INDENT>self.session._out('<STR_LIT>')<EOL>for image in self.document.images:<EOL><INDENT>self.session._out('<STR_LIT>' % (image.index, image.number))<EOL><DEDENT>self.session._out('<STR_LIT>')<EOL><DEDENT>self.session._out('<STR_LIT>')<EOL>self.session._out('<STR_LIT>')<EOL>

Creates PDF reference to resource objects.