self.stopwords = stopwords<EOL>

Redefine a list of stopwords.

        Args:
            stopwords: a list of stopwords

        Returns:
            None


def __call__(self, batch: List[Union[str, List[str]]]) -> List[Union[List[str], str]]:

if isinstance(batch[<NUM_LIT:0>], str):<EOL><INDENT>return [self.tokenizer.tokenize(line, escape=self.escape) for line in batch]<EOL><DEDENT>else:<EOL><INDENT>return [self.detokenizer.detokenize(line, return_str=True, unescape=self.escape)<EOL>for line in batch]<EOL><DEDENT>

Tokenize given batch of strings or detokenize given batch of lists of tokens

        Args:
            batch: list of text samples or list of lists of tokens

        Returns:
            list of lists of tokens or list of text samples


def __call__(self, batch: List[str]) -> List[List[str]]:

return [self.tokenizer(sent) for sent in batch]<EOL>

Tokenize given batch

        Args:
            batch: list of text samples

        Returns:
            list of lists of tokens


text = '<STR_LIT:U+0020>'.join(tokens)<EOL>step0 = text.replace('<STR_LIT>', '<STR_LIT>')<EOL>step1 = step0.replace("<STR_LIT>", '<STR_LIT:">').replace("<STR_LIT>", '<STR_LIT:">')<EOL>step2 = step1.replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>")<EOL>step3 = re.sub(r'<STR_LIT>', r"<STR_LIT>", step2)<EOL>step4 = re.sub(r'<STR_LIT>', r"<STR_LIT>", step3)<EOL>step5 = step4.replace("<STR_LIT>", "<STR_LIT:'>").replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>")<EOL>step6 = step5.replace("<STR_LIT>", "<STR_LIT>")<EOL>return step6.strip()<EOL>

Detokenizing a text undoes the tokenizing operation, restores
punctuation and spaces to the places that people expect them to be.
Ideally, `detokenize(tokenize(text))` should be identical to `text`,
except for line breaks.


def ngramize(items: List[str], ngram_range=(<NUM_LIT:1>, <NUM_LIT:1>)) -> Generator[List[str], Any, None]:

ngrams = []<EOL>ranges = [(<NUM_LIT:0>, i) for i in range(ngram_range[<NUM_LIT:0>], ngram_range[<NUM_LIT:1>] + <NUM_LIT:1>)]<EOL>for r in ranges:<EOL><INDENT>ngrams += list(zip(*[items[j:] for j in range(*r)]))<EOL><DEDENT>formatted_ngrams = ['<STR_LIT:U+0020>'.join(item) for item in ngrams]<EOL>yield formatted_ngrams<EOL>

Make ngrams from a list of tokens/lemmas
:param items: list of tokens, lemmas or other strings to form ngrams
:param ngram_range: range for producing ngrams, ex. for unigrams + bigrams should be set to
(1, 2), for bigrams only should be set to (2, 2)
:return: ngrams (as strings) generator


def __call__(self, *args: List[Any], **kwargs: Dict[str, Any]):

data = kwargs or dict(zip(self.param_names, args))<EOL>if self.debatchify:<EOL><INDENT>batch_size = <NUM_LIT:0><EOL>for v in data.values():<EOL><INDENT>batch_size = len(v)<EOL>break<EOL><DEDENT>assert batch_size > <NUM_LIT:0><EOL>async def collect():<EOL><INDENT>return [j async for j in self.get_async_response(data, batch_size)]<EOL><DEDENT>loop = asyncio.get_event_loop()<EOL>response = loop.run_until_complete(collect())<EOL><DEDENT>else:<EOL><INDENT>response = requests.post(self.url, json=data).json()<EOL><DEDENT>if self.out_count > <NUM_LIT:1>:<EOL><INDENT>response = list(zip(*response))<EOL><DEDENT>return response<EOL>

Args:
    *args: list of parameters sent to the API endpoint. Parameter names are taken from self.param_names.
    **kwargs: named parameters to send to the API endpoint. If not empty, args are ignored

Returns:
    result of the API request(s)


async def get_async_response(self, data: dict, batch_size: int) -> AsyncIterable:

loop = asyncio.get_event_loop()<EOL>futures = [<EOL>loop.run_in_executor(<EOL>None,<EOL>requests.post,<EOL>self.url,<EOL>None,<EOL>{k: v[i] for k, v in data.items()}<EOL>)<EOL>for i in range(batch_size)<EOL>]<EOL>for r in await asyncio.gather(*futures):<EOL><INDENT>yield r.json()<EOL><DEDENT>

Helper function for sending requests asynchronously if the API endpoint does not support batching

        Args:
            data: data to be passed to the API endpoint
            batch_size: requests count

        Yields:
            requests results parsed as json


with ProcessPoolExecutor(self.n_workers) as executor:<EOL><INDENT>futures = [executor.submit(api_requester, *args) for api_requester<EOL>in<EOL>self.api_requesters]<EOL>concurrent.futures.wait(futures)<EOL>results = []<EOL>for future, api_requester in zip(futures, self.api_requesters):<EOL><INDENT>result = future.result()<EOL>if api_requester.out_count > <NUM_LIT:1>:<EOL><INDENT>results += result<EOL><DEDENT>else:<EOL><INDENT>results.append(result)<EOL><DEDENT><DEDENT><DEDENT>return results<EOL>

Args:
    *args: list of arguments to forward to the API requesters

Returns:
    results of the requests


def _call(self, utterances_batch: list, utterances_ids: Optional[list]=None) -> list:

batch_size = len(utterances_batch)<EOL>ids = utterances_ids or list(range(batch_size))<EOL>batch_history = [self.history[utt_id] for utt_id in ids]<EOL>responses = []<EOL>filtered = self.skills_filter(utterances_batch, batch_history)<EOL>for skill_i, (filtered_utterances, skill) in enumerate(zip(filtered, self.wrapped_skills)):<EOL><INDENT>skill_i_utt_indexes = [utt_index for utt_index, utt_filter in enumerate(filtered_utterances) if utt_filter]<EOL>if skill_i_utt_indexes:<EOL><INDENT>skill_i_utt_batch = [utterances_batch[i] for i in skill_i_utt_indexes]<EOL>skill_i_utt_ids = [ids[i] for i in skill_i_utt_indexes]<EOL>res = [(None, <NUM_LIT:0.>)] * batch_size<EOL>predicted, confidence = skill(skill_i_utt_batch, skill_i_utt_ids)<EOL>for i, predicted, confidence in zip(skill_i_utt_indexes, predicted, confidence):<EOL><INDENT>res[i] = (predicted, confidence)<EOL><DEDENT>responses.append(res)<EOL><DEDENT><DEDENT>responses = self.skills_processor(utterances_batch, batch_history, *responses)<EOL>return responses<EOL>

Processes batch of utterances and returns corresponding responses batch.

Each call of Agent passes incoming utterances batch through skills filter,
agent skills, skills processor. Batch of dialog IDs can be provided, in
other case utterances indexes in incoming batch are used as dialog IDs.

Args:
    utterances_batch: Batch of incoming utterances.
    utterances_ids: Batch of dialog IDs corresponding to incoming utterances.

Returns:
    responses: A batch of responses corresponding to the
        utterance batch received by agent.


def __call__(self, utterances_batch: list, history_batch: list) -> list:

return [[True] * len(utterances_batch)] * self.size<EOL>

Returns skills-utterances application matrix.

        Generates skills-utterances application matrix with all True
        elements.

        Args:
            utterances_batch: A batch of utterances of any type.
            history_batch: Not used.

        Returns:
            response: Skills-utterances application matrix with all True
                elements.


def show_details(item_data: Dict[Any, Any]) -> str:

txt = "<STR_LIT>"<EOL>for key, value in item_data.items():<EOL><INDENT>txt += "<STR_LIT>" + str(key) + "<STR_LIT>" + '<STR_LIT>' + str(value) + "<STR_LIT>"<EOL><DEDENT>return txt<EOL>

Format catalog item output

    Parameters:
        item_data: item's attributes values

    Returns:
        [rich_message]: list of formatted rich message


config_path = find_config('<STR_LIT>')<EOL>skill = build_model(config_path)<EOL>agent = EcommerceAgent(skills=[skill])<EOL>return agent<EOL>

Make an agent

    Returns:
        agent: created Ecommerce agent


args = parser.parse_args()<EOL>run_ms_bot_framework_server(agent_generator=make_agent,<EOL>app_id=args.ms_id,<EOL>app_secret=args.ms_secret,<EOL>stateful=True)<EOL>

Parse parameters and run ms bot framework


def _call(self, utterances_batch: List[str], utterances_ids: List[int] = None) -> List[RichMessage]:

rich_message = RichMessage()<EOL>for utt_id, utt in enumerate(utterances_batch):<EOL><INDENT>if utterances_ids:<EOL><INDENT>id_ = utterances_ids[utt_id]<EOL><DEDENT>log.debug(f'<STR_LIT>')<EOL>if utt == "<STR_LIT>":<EOL><INDENT>welcome = "<STR_LIT>"<EOL>rich_message.add_control(PlainText(welcome))<EOL>continue<EOL><DEDENT>if utt[<NUM_LIT:0>] == "<STR_LIT:@>":<EOL><INDENT>command, *parts = utt.split("<STR_LIT::>")<EOL>log.debug(f'<STR_LIT>')<EOL>if command == "<STR_LIT>":<EOL><INDENT>batch_index = int(parts[<NUM_LIT:0>])  <EOL>item_index = int(parts[<NUM_LIT:1>])  <EOL>rich_message.add_control(PlainText(show_details(<EOL>self.history[id_][batch_index][item_index])))<EOL>continue<EOL><DEDENT>if command == "<STR_LIT>":<EOL><INDENT>state = self.history[id_][int(parts[<NUM_LIT:0>])]<EOL>state[parts[<NUM_LIT:1>]] = parts[<NUM_LIT:2>]<EOL>state["<STR_LIT:start>"] = <NUM_LIT:0><EOL>state["<STR_LIT>"] = <NUM_LIT:5><EOL>utt = state['<STR_LIT>']<EOL>self.states[id_] = state<EOL><DEDENT>if command == "<STR_LIT>":<EOL><INDENT>state = self.history[id_][int(parts[<NUM_LIT:0>])]<EOL>state['<STR_LIT:start>'] = state['<STR_LIT>']<EOL>state['<STR_LIT>'] = state['<STR_LIT>'] + <NUM_LIT:5><EOL>utt = state['<STR_LIT>']<EOL>self.states[id_] = state<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if id_ not in self.states:<EOL><INDENT>self.states[id_] = {}<EOL><DEDENT>self.states[id_]["<STR_LIT:start>"] = <NUM_LIT:0><EOL>self.states[id_]["<STR_LIT>"] = <NUM_LIT:5><EOL><DEDENT>responses_batch, confidences_batch, state_batch = self.skills[<NUM_LIT:0>](<EOL>[utt], self.history[id_], [self.states[id_]])<EOL>self.states[id_] = state_batch[<NUM_LIT:0>]<EOL>self.states[id_]["<STR_LIT>"] = utt<EOL>items_batch, entropy_batch = responses_batch<EOL>for batch_idx, items in enumerate(items_batch):<EOL><INDENT>self.history[id_].append(items)<EOL>self.history[id_].append(self.states[id_])<EOL>for idx, item in enumerate(items):<EOL><INDENT>rich_message.add_control(_draw_item(item, idx, self.history[id_]))<EOL><DEDENT>if len(items) == self.states[id_]['<STR_LIT>'] - self.states[id_]['<STR_LIT:start>']:<EOL><INDENT>buttons_frame = _draw_tail(entropy_batch[batch_idx], self.history[id_])<EOL>rich_message.add_control(buttons_frame)<EOL><DEDENT><DEDENT><DEDENT>return [rich_message]<EOL>

Processes batch of utterances and returns corresponding responses batch.

        Args:
            utterances_batch: Batch of incoming utterances.
            utterances_ids: Batch of dialog IDs corresponding to incoming utterances.

        Returns:
            responses: A batch of responses corresponding to the
                utterance batch received by agent.


def __call__(self, utterances: list, batch_history: list, *responses: list) -> list:

responses, confidences = zip(*[zip(*r) for r in responses])<EOL>indexes = [c.index(max(c)) for c in zip(*confidences)]<EOL>result = [responses[i] for i, *responses in zip(indexes, *responses)]<EOL>return result<EOL>

Selects for each utterance response with highest confidence.

        Args:
            utterances_batch: Not used.
            history_batch: Not used.
            responses: Each response positional argument corresponds to
                response of one of Agent skills and is represented by
                batch (list) of (response, confidence) tuple structures.

        Returns:
            responses: A batch of responses corresponding to the
                utterance batch received by agent.


result = [random.choice([t for t, sc in r if t]) for r in zip(*responses)]<EOL>return result<EOL>

Selects result of a random skill for each utterance.

        Args:
            utterances_batch: Not used.
            history_batch: Not used.
            responses: Each response positional argument corresponds to
                response of one of Agent skills and is represented by
                batch (list) of (response, confidence) tuple structures.

        Returns:
            result: A batch of responses corresponding to the utterance
                batch received by agent.


responses, confidences = zip(*[zip(*r) for r in responses])<EOL>indexes = [c.index(max(c)) for c in zip(*confidences)]<EOL>result = []<EOL>for i, *responses in zip(indexes, *responses):<EOL><INDENT>rich_message = RichMessage()<EOL>plain_text = PlainText(str(responses[i]))<EOL>rich_message.add_control(plain_text)<EOL>result.append(rich_message)<EOL><DEDENT>return result<EOL>

Selects for each utterance response with highest confidence and wraps them to RichControl objects.

        Args:
            utterances_batch: Not used.
            history_batch: Not used.
            responses: Each response positional argument corresponds to
                response of one of Agent skills and is represented by
                batch (list) of (response, confidence) tuple structures.

        Returns:
            result: A batch of responses corresponding to the utterance
                batch received by agent.


self.control_json['<STR_LIT:content>'] = self.content<EOL>return self.control_json<EOL>

Returns json compatible state of the PlainText instance.

        Returns:
            control_json: Json representation of PlainText state.


out_activity = {}<EOL>out_activity['<STR_LIT:type>'] = '<STR_LIT:message>'<EOL>out_activity['<STR_LIT:text>'] = self.content<EOL>return out_activity<EOL>

Returns MS Bot Framework compatible state of the PlainText instance.

        Creating MS Bot Framework activity blank with "text" field populated.

        Returns:
            out_activity: MS Bot Framework representation of PlainText state.


response = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>'<STR_LIT:text>': self.content},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>'<STR_LIT:content>': self.content<EOL>}<EOL>}<EOL>}<EOL>return response<EOL>

Returns Amazon Alexa compatible state of the PlainText instance.

        Creating Amazon Alexa response blank with populated "outputSpeech" and
        "card sections.

        Returns:
            response: Amazon Alexa representation of PlainText state.


content = {}<EOL>content['<STR_LIT:name>'] = self.name<EOL>content['<STR_LIT>'] = self.callback<EOL>self.control_json['<STR_LIT:content>'] = content<EOL>return self.control_json<EOL>

Returns json compatible state of the Button instance.

        Returns:
            control_json: Json representation of Button state.


card_action = {}<EOL>card_action['<STR_LIT:type>'] = '<STR_LIT>'<EOL>card_action['<STR_LIT:title>'] = self.name<EOL>card_action['<STR_LIT:value>'] = self.callback = self.callback<EOL>return card_action<EOL>

Returns MS Bot Framework compatible state of the Button instance.

        Creates MS Bot Framework CardAction (button) with postBack value return.

        Returns:
            control_json: MS Bot Framework representation of Button state.


self.content.append(button)<EOL>

Adds Button instance to RichMessage.

        Args:
            button: Button instance.


content = {}<EOL>if self.text:<EOL><INDENT>content['<STR_LIT:text>'] = self.text<EOL><DEDENT>content['<STR_LIT>'] = [control.json() for control in self.content]<EOL>self.control_json['<STR_LIT:content>'] = content<EOL>return self.control_json<EOL>

Returns json compatible state of the ButtonsFrame instance.

        Returns json compatible state of the ButtonsFrame instance including
        all nested buttons.

        Returns:
            control_json: Json representation of ButtonsFrame state.


rich_card = {}<EOL>buttons = [button.ms_bot_framework() for button in self.content]<EOL>rich_card['<STR_LIT>'] = buttons<EOL>if self.text:<EOL><INDENT>rich_card['<STR_LIT:title>'] = self.text<EOL><DEDENT>attachments = [<EOL>{<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:content>": rich_card<EOL>}<EOL>]<EOL>out_activity = {}<EOL>out_activity['<STR_LIT:type>'] = '<STR_LIT:message>'<EOL>out_activity['<STR_LIT>'] = attachments<EOL>return out_activity<EOL>

Returns MS Bot Framework compatible state of the ButtonsFrame instance.

        Creating MS Bot Framework activity blank with RichCard in "attachments". RichCard
        is populated with CardActions corresponding buttons embedded in ButtonsFrame.

        Returns:
            control_json: MS Bot Framework representation of ButtonsFrame state.


def make_hello_bot_agent() -> DefaultAgent:

skill_hello = PatternMatchingSkill(['<STR_LIT>'], patterns=['<STR_LIT>', '<STR_LIT:hello>', '<STR_LIT>'])<EOL>skill_bye = PatternMatchingSkill(['<STR_LIT>', '<STR_LIT>'], patterns=['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>skill_fallback = PatternMatchingSkill(['<STR_LIT>', '<STR_LIT>'])<EOL>agent = DefaultAgent([skill_hello, skill_bye, skill_fallback], skills_processor=HighestConfidenceSelector())<EOL>return agent<EOL>

Builds agent based on PatternMatchingSkill and HighestConfidenceSelector.

    This is agent building tutorial. You can use this .py file to check how hello-bot agent works.

    Returns:
        agent: Agent capable of handling several simple greetings.


@abstractmethod<EOL><INDENT>def __call__(self, utterances_batch: list, history_batch: list,<EOL>states_batch: Optional[list]=None) -> Union[Tuple[list, list], Tuple[list, list, Optional[list]]]:<DEDENT>

Returns skill inference result.

        Returns batches of skill inference results, estimated confidence
        levels and up to date states corresponding to incoming utterance
        batch.

        Args:
            utterances_batch: A batch of utterances of any type.
            history_batch: A batch of list typed histories for each utterance.
            states_batch: Optional. A batch of arbitrary typed states for
                each utterance.

        Returns:
            response: A batch of arbitrary typed skill inference results.
            confidence: A batch of float typed confidence levels for each of
                skill inference result.
            states: Optional. A batch of arbitrary typed states for each
                response.


def train_on_batches(self, iterator: DataLearningIterator) -> None:

self.start_time = time.time()<EOL>if self.validate_first:<EOL><INDENT>self._validate(iterator)<EOL><DEDENT>while True:<EOL><INDENT>impatient = False<EOL>self._send_event(event_name='<STR_LIT>')<EOL>for x, y_true in iterator.gen_batches(self.batch_size, data_type='<STR_LIT:train>'):<EOL><INDENT>self.last_result = self._chainer.train_on_batch(x, y_true)<EOL>if self.last_result is None:<EOL><INDENT>self.last_result = {}<EOL><DEDENT>elif not isinstance(self.last_result, dict):<EOL><INDENT>self.last_result = {'<STR_LIT>': self.last_result}<EOL><DEDENT>if '<STR_LIT>' in self.last_result:<EOL><INDENT>self.losses.append(self.last_result.pop('<STR_LIT>'))<EOL><DEDENT>self.train_batches_seen += <NUM_LIT:1><EOL>self.examples += len(x)<EOL>if self.log_every_n_batches > <NUM_LIT:0> and self.train_batches_seen % self.log_every_n_batches == <NUM_LIT:0>:<EOL><INDENT>self._log(iterator, tensorboard_tag='<STR_LIT>', tensorboard_index=self.train_batches_seen)<EOL><DEDENT>if self.val_every_n_batches > <NUM_LIT:0> and self.train_batches_seen % self.val_every_n_batches == <NUM_LIT:0>:<EOL><INDENT>self._validate(iterator,<EOL>tensorboard_tag='<STR_LIT>', tensorboard_index=self.train_batches_seen)<EOL><DEDENT>self._send_event(event_name='<STR_LIT>')<EOL>if <NUM_LIT:0> < self.max_batches <= self.train_batches_seen:<EOL><INDENT>impatient = True<EOL>break<EOL><DEDENT>if <NUM_LIT:0> < self.validation_patience <= self.patience:<EOL><INDENT>log.info('<STR_LIT>')<EOL>impatient = True<EOL>break<EOL><DEDENT><DEDENT>if impatient:<EOL><INDENT>break<EOL><DEDENT>self.epoch += <NUM_LIT:1><EOL>if self.log_every_n_epochs > <NUM_LIT:0> and self.epoch % self.log_every_n_epochs == <NUM_LIT:0>:<EOL><INDENT>self._log(iterator, tensorboard_tag='<STR_LIT>', tensorboard_index=self.epoch)<EOL><DEDENT>if self.val_every_n_epochs > <NUM_LIT:0> and self.epoch % self.val_every_n_epochs == <NUM_LIT:0>:<EOL><INDENT>self._validate(iterator, tensorboard_tag='<STR_LIT>', tensorboard_index=self.epoch)<EOL><DEDENT>self._send_event(event_name='<STR_LIT>')<EOL>if <NUM_LIT:0> < self.max_epochs <= self.epoch:<EOL><INDENT>break<EOL><DEDENT>if <NUM_LIT:0> < self.validation_patience <= self.patience:<EOL><INDENT>log.info('<STR_LIT>')<EOL>break<EOL><DEDENT><DEDENT>

Train pipeline on batches using provided data iterator and initialization parameters


def train(self, iterator: DataLearningIterator) -> None:

self.fit_chainer(iterator)<EOL>if callable(getattr(self._chainer, '<STR_LIT>', None)):<EOL><INDENT>try:<EOL><INDENT>self.train_on_batches(iterator)<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>log.info('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.warn(f'<STR_LIT>')<EOL><DEDENT>if not self._saved:<EOL><INDENT>self.save()<EOL><DEDENT>

Call :meth:`~fit_chainer` and then :meth:`~train_on_batches` with provided data iterator as an argument


def fit_chainer(self, iterator: Union[DataFittingIterator, DataLearningIterator]) -> None:

if self._built:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>for component_index, component_config in enumerate(self.chainer_config['<STR_LIT>'], <NUM_LIT:1>):<EOL><INDENT>component = from_params(component_config, mode='<STR_LIT:train>')<EOL>if '<STR_LIT>' in component_config:<EOL><INDENT>component: Estimator<EOL>targets = component_config['<STR_LIT>']<EOL>if isinstance(targets, str):<EOL><INDENT>targets = [targets]<EOL><DEDENT>if self.batch_size > <NUM_LIT:0> and callable(getattr(component, '<STR_LIT>', None)):<EOL><INDENT>writer = None<EOL>for i, (x, y) in enumerate(iterator.gen_batches(self.batch_size, shuffle=False)):<EOL><INDENT>preprocessed = self._chainer.compute(x, y, targets=targets)<EOL>result = component.partial_fit(*preprocessed)<EOL>if result is not None and self.tensorboard_log_dir is not None:<EOL><INDENT>if writer is None:<EOL><INDENT>writer = self._tf.summary.FileWriter(str(self.tensorboard_log_dir /<EOL>f'<STR_LIT>'))<EOL><DEDENT>for name, score in result.items():<EOL><INDENT>summary = self._tf.Summary()<EOL>summary.value.add(tag='<STR_LIT>' + name, simple_value=score)<EOL>writer.add_summary(summary, i)<EOL><DEDENT>writer.flush()<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>preprocessed = self._chainer.compute(*iterator.get_instances(), targets=targets)<EOL>if len(targets) == <NUM_LIT:1>:<EOL><INDENT>preprocessed = [preprocessed]<EOL><DEDENT>result: Optional[Dict[str, Iterable[float]]] = component.fit(*preprocessed)<EOL>if result is not None and self.tensorboard_log_dir is not None:<EOL><INDENT>writer = self._tf.summary.FileWriter(str(self.tensorboard_log_dir /<EOL>f'<STR_LIT>'))<EOL>for name, scores in result.items():<EOL><INDENT>for i, score in enumerate(scores):<EOL><INDENT>summary = self._tf.Summary()<EOL>summary.value.add(tag='<STR_LIT>' + name, simple_value=score)<EOL>writer.add_summary(summary, i)<EOL><DEDENT><DEDENT>writer.flush()<EOL><DEDENT><DEDENT>component.save()<EOL><DEDENT>if '<STR_LIT>' in component_config:<EOL><INDENT>c_in = component_config['<STR_LIT>']<EOL>c_out = component_config['<STR_LIT>']<EOL>in_y = component_config.get('<STR_LIT>', None)<EOL>main = component_config.get('<STR_LIT>', False)<EOL>self._chainer.append(component, c_in, c_out, in_y, main)<EOL><DEDENT><DEDENT>self._built = True<EOL>

Build the pipeline :class:`~deeppavlov.core.common.chainer.Chainer` and successively fit
:class:`Estimator <deeppavlov.core.models.estimator.Estimator>` components using a provided data iterator


self._load()<EOL>return self._chainer<EOL>

Return a :class:`~deeppavlov.core.common.chainer.Chainer` built from ``self.chainer_config`` for inference


def train(self, iterator: Union[DataFittingIterator, DataLearningIterator]) -> None:

self.fit_chainer(iterator)<EOL>self._saved = True<EOL>

Call :meth:`~fit_chainer` with provided data iterator as an argument


def evaluate(self, iterator: DataLearningIterator, evaluation_targets: Optional[Iterable[str]] = None, *,<EOL>print_reports: bool = True) -> Dict[str, dict]:

self._load()<EOL>if evaluation_targets is None:<EOL><INDENT>evaluation_targets = self.evaluation_targets<EOL><DEDENT>res = {}<EOL>for data_type in evaluation_targets:<EOL><INDENT>data_gen = iterator.gen_batches(self.batch_size, data_type=data_type, shuffle=False)<EOL>report = self.test(data_gen)<EOL>res[data_type] = report<EOL>if print_reports:<EOL><INDENT>print(json.dumps({data_type: report}, ensure_ascii=False))<EOL><DEDENT><DEDENT>return res<EOL>

Run :meth:`test` on multiple data types using provided data iterator

Args:
    iterator: :class:`~deeppavlov.core.data.data_learning_iterator.DataLearningIterator` used for evaluation
    evaluation_targets: iterable of data types to evaluate on
    print_reports: a flag used to print evaluation reports as json lines

Returns:
    a dictionary with data types as keys and evaluation reports as values


def prettify_metrics(metrics: List[Tuple[str, float]], precision: int = <NUM_LIT:4>) -> OrderedDict:

prettified_metrics = OrderedDict()<EOL>for key, value in metrics:<EOL><INDENT>value = round(value, precision)<EOL>prettified_metrics[key] = value<EOL><DEDENT>return prettified_metrics<EOL>


@abstractmethod<EOL><INDENT>def json(self) -> Union[list, dict]:<DEDENT>

pass<EOL>

Returns json compatible state of the control instance including
        its nested controls.

        Returns:
            control: Json representation of control state.


return None<EOL>

Returns MS Bot Framework compatible state of the control instance
        including its nested controls.

        Returns:
            control: MS Bot Framework representation of control state.


return None<EOL>

Returns Telegram compatible state of the control instance
        including its nested controls.

        Returns:
            control: Telegram representation of control state.


return None<EOL>

Returns Amazon Alexa compatible state of the control instance
        including its nested controls.

        Returns:
            control: Amazon Alexa representation of control state.


def add_control(self, control: RichControl):

self.controls.append(control)<EOL>

Adds RichControl instance to RichMessage.

        Args:
            control: RichControl instance.


json_controls = [control.json() for control in self.controls]<EOL>return json_controls<EOL>

Returns list of json compatible states of the RichMessage instance
        nested controls.

        Returns:
            json_controls: Json representation of RichMessage instance
                nested controls.


ms_bf_controls = [control.ms_bot_framework() for control in self.controls]<EOL>return ms_bf_controls<EOL>

Returns list of MS Bot Framework compatible states of the
        RichMessage instance nested controls.

        Returns:
            ms_bf_controls: MS Bot Framework representation of RichMessage instance
                nested controls.


telegram_controls = [control.telegram() for control in self.controls]<EOL>return telegram_controls<EOL>

Returns list of Telegram compatible states of the RichMessage
        instance nested controls.

        Returns:
            telegram_controls: Telegram representation of RichMessage instance nested
                controls.


alexa_controls = [control.alexa() for control in self.controls]<EOL>return alexa_controls<EOL>

Returns list of Amazon Alexa compatible states of the RichMessage
        instance nested controls.

        Returns:
            alexa_controls: Amazon Alexa representation of RichMessage instance nested
                controls.


@staticmethod<EOL><INDENT>def _get_timestamp_utc_str() -> str:<DEDENT>

utc_timestamp_str = datetime.strftime(datetime.utcnow(), LOG_TIMESTAMP_FORMAT)<EOL>return utc_timestamp_str<EOL>

Returns str converted current UTC timestamp.

        Returns:
            utc_timestamp_str: str converted current UTC timestamp.


log_dir: Path = Path(self.config['<STR_LIT>']).expanduser().resolve() / self.agent_name<EOL>log_dir.mkdir(parents=True, exist_ok=True)<EOL>log_file_path = Path(log_dir, f'<STR_LIT>')<EOL>log_file = open(log_file_path, '<STR_LIT:a>', buffering=<NUM_LIT:1>, encoding='<STR_LIT:utf8>')<EOL>return log_file<EOL>

Returns opened file object for writing dialog logs.

        Returns:
            log_file: opened Python file object.


def _log(self, utterance: Any, direction: str, dialog_id: Optional[Hashable]=None):

if isinstance(utterance, str):<EOL><INDENT>pass<EOL><DEDENT>elif isinstance(utterance, RichMessage):<EOL><INDENT>utterance = utterance.json()<EOL><DEDENT>elif isinstance(utterance, (list, dict)):<EOL><INDENT>utterance = jsonify_data(utterance)<EOL><DEDENT>else:<EOL><INDENT>utterance = str(utterance)<EOL><DEDENT>dialog_id = str(dialog_id) if not isinstance(dialog_id, str) else dialog_id<EOL>if self.log_file.tell() >= self.log_max_size * <NUM_LIT>:<EOL><INDENT>self.log_file.close()<EOL>self.log_file = self._get_log_file()<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>log_msg = {}<EOL>log_msg['<STR_LIT>'] = self._get_timestamp_utc_str()<EOL>log_msg['<STR_LIT>'] = dialog_id<EOL>log_msg['<STR_LIT>'] = direction<EOL>log_msg['<STR_LIT:message>'] = utterance<EOL>log_str = json.dumps(log_msg, ensure_ascii=self.config['<STR_LIT>'])<EOL>self.log_file.write(f'<STR_LIT>')<EOL><DEDENT>except IOError:<EOL><INDENT>log.error('<STR_LIT>')<EOL><DEDENT><DEDENT>

Logs single dialog utterance to current dialog log file.

        Args:
            utterance: Dialog utterance.
            direction: 'in' or 'out' utterance direction.
            dialog_id: Dialog ID.


def log_in(self, utterance: Any, dialog_id: Optional[Hashable] = None) -> None:

if self.enabled:<EOL><INDENT>self._log(utterance, '<STR_LIT>', dialog_id)<EOL><DEDENT>

Wraps _log method for all input utterances.
        Args:
            utterance: Dialog utterance.
            dialog_id: Dialog ID.


def log_out(self, utterance: Any, dialog_id: Optional[Hashable] = None) -> None:

Wraps _log method for all output utterances.
        Args:
            utterance: Dialog utterance.
            dialog_id: Dialog ID.


def __call__(self, utterances_batch: list, utterances_ids: Optional[list] = None) -> list:

responses_batch = self._call(utterances_batch, utterances_ids)<EOL>batch_size = len(utterances_batch)<EOL>ids = utterances_ids or list(range(batch_size))<EOL>for utt_batch_idx, utt_id in enumerate(ids):<EOL><INDENT>self.history[utt_id].append(str(utterances_batch[utt_batch_idx]))<EOL>self.dialog_logger.log_in(utterances_batch[utt_batch_idx], utt_id)<EOL>self.history[utt_id].append(str(responses_batch[utt_batch_idx]))<EOL>self.dialog_logger.log_out(responses_batch[utt_batch_idx], utt_id)<EOL><DEDENT>return responses_batch<EOL>

Wraps _call method and updates utterances history.

        Args:
            utterances_batch: Batch of incoming utterances.
            utterances_ids: Batch of dialog IDs corresponding to incoming utterances.

        Returns:
            responses: A batch of responses corresponding to the
                utterance batch received by agent.


@abstractmethod<EOL><INDENT>def _call(self, utterances_batch: list, utterances_ids: Optional[list] = None) -> list:<DEDENT>

pass<EOL>

Processes batch of utterances and returns corresponding responses batch.

        Each call of Agent processes incoming utterances and returns response
        for each utterance Batch of dialog IDs can be provided, in other case
        utterances indexes in incoming batch are used as dialog IDs.

        Args:
            utterances_batch: Batch of incoming utterances.
            utterances_ids: Batch of dialog IDs corresponding to incoming utterances.

        Returns:
            responses: A batch of responses corresponding to the
                utterance batch received by agent.


def __call__(self, utterances_batch: list, utterances_ids: Optional[list] = None) -> Tuple[list, list]:

history_batch = [self.agent.history[utt_id] for utt_id in utterances_ids]<EOL>states_batch = [self.agent.states[utt_id][self.skill_id] for utt_id in utterances_ids]<EOL>predicted, confidence, *states = self.skill(utterances_batch, history_batch, states_batch)<EOL>states = states[<NUM_LIT:0>] if states else [None] * len(predicted)<EOL>for utt_id, state in zip(utterances_ids, states):<EOL><INDENT>self.agent.states[utt_id][self.skill_id] = state<EOL><DEDENT>return predicted, confidence<EOL>

Wraps __call__ method of Skill instance.

            Provides skill __call__ with signature of Agent __call__ and handles
            automatic state management for skill.

        Args:
            utterances_batch: Batch of incoming utterances.
            utterances_ids: Batch of dialog IDs corresponding to incoming utterances.

        Returns:
            response: A batch of arbitrary typed skill inference results.
            confidence: A batch of float typed confidence levels for each of
                skill inference result.
            states: Optional. A batch of arbitrary typed states for each
                response.


@abstractmethod<EOL><INDENT>def __call__(self, utterances_batch: list, history_batch: list) -> list:<DEDENT>

pass<EOL>

Returns skills-utterances application matrix.

        Returns skills-utterances application matrix which contains
        information about Agent skills to be applied to each utterance
        from incoming batch.

        Args:
            utterances_batch: A batch of utterances of any type.
            history_batch: A batch of list typed histories
                for each utterance.

        Returns:
            response: Skills-utterances application matrix,
            for example:
            [[True, False, True, True],
             [False, True, True, True]]
            Where each inner dict corresponds to one of the Agent
            skills and each value in the inner dict contains information
            about whether the skill will be applied to the utterance
            with the same position in the utterances_batch.


@abstractmethod<EOL><INDENT>def __call__(self, utterances_batch: list, history_batch: list, *responses: list) -> list:<DEDENT>

pass<EOL>

Returns final response for each incoming utterance.

        Processes Agent skills and generates one final response for each
        utterance in incoming batch.

        Args:
            utterances_batch: A batch of utterances of any type
            history_batch: A batch of list typed histories
                for each utterance
            responses: Each response positional argument corresponds to
                response of one of Agent skills and is represented by
                batch (list) of (response, confidence) tuple structures.

        Returns:
            responses: A batch of responses corresponding to the
                utterance batch received by agent.


@wraps(func)<EOL>def _wrapped(*args, **kwargs):<EOL><INDENT>with graph.as_default():<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT><DEDENT>return _wrapped<EOL>

Constructs function encapsulated in the graph.


import keras.backend as K<EOL>@wraps(func)<EOL>def _wrapped(*args, **kwargs):<EOL><INDENT>with graph.as_default():<EOL><INDENT>K.set_session(session)<EOL>return func(*args, **kwargs)<EOL><DEDENT><DEDENT>return _wrapped<EOL>

Constructs function encapsulated in the graph and the session.


def load(self, exclude_scopes: tuple = ('<STR_LIT>',)) -> None:

if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>'.format(self.__class__.__name__))<EOL><DEDENT>path = str(self.load_path.resolve())<EOL>if tf.train.checkpoint_exists(path):<EOL><INDENT>log.info('<STR_LIT>'.format(path))<EOL>var_list = self._get_saveable_variables(exclude_scopes)<EOL>saver = tf.train.Saver(var_list)<EOL>saver.restore(self.sess, path)<EOL><DEDENT>

Load model parameters from self.load_path


def save(self, exclude_scopes: tuple = ('<STR_LIT>',)) -> None:

if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>'.format(self.__class__.__name__))<EOL><DEDENT>path = str(self.save_path.resolve())<EOL>log.info('<STR_LIT>'.format(path))<EOL>var_list = self._get_saveable_variables(exclude_scopes)<EOL>saver = tf.train.Saver(var_list)<EOL>saver.save(self.sess, path)<EOL>


if optimizer_scope_name is None:<EOL><INDENT>opt_scope = tf.variable_scope('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>opt_scope = tf.variable_scope(optimizer_scope_name)<EOL><DEDENT>with opt_scope:<EOL><INDENT>if learnable_scopes is None:<EOL><INDENT>variables_to_train = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)<EOL><DEDENT>else:<EOL><INDENT>variables_to_train = []<EOL>for scope_name in learnable_scopes:<EOL><INDENT>variables_to_train.extend(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope_name))<EOL><DEDENT><DEDENT>if optimizer is None:<EOL><INDENT>optimizer = tf.train.AdamOptimizer<EOL><DEDENT>extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)<EOL>with tf.control_dependencies(extra_update_ops):<EOL><INDENT>def clip_if_not_none(grad):<EOL><INDENT>if grad is not None:<EOL><INDENT>return tf.clip_by_norm(grad, clip_norm)<EOL><DEDENT><DEDENT>opt = optimizer(learning_rate, **kwargs)<EOL>grads_and_vars = opt.compute_gradients(loss, var_list=variables_to_train)<EOL>if clip_norm is not None:<EOL><INDENT>grads_and_vars = [(clip_if_not_none(grad), var)<EOL>for grad, var in grads_and_vars]<EOL><DEDENT>train_op = opt.apply_gradients(grads_and_vars)<EOL><DEDENT><DEDENT>return train_op<EOL>

Get train operation for given loss

Args:
    loss: loss, tf tensor or scalar
    learning_rate: scalar or placeholder.
    clip_norm: clip gradients norm by clip_norm.
    learnable_scopes: which scopes are trainable (None for all).
    optimizer: instance of tf.train.Optimizer, default Adam.
    **kwargs: parameters passed to tf.train.Optimizer object
       (scalars or placeholders).

Returns:
    train_op


log.info('<STR_LIT>')<EOL>variables = tf.trainable_variables()<EOL>blocks = defaultdict(int)<EOL>for var in variables:<EOL><INDENT>block_name = var.name.split('<STR_LIT:/>')[<NUM_LIT:0>]<EOL>number_of_parameters = np.prod(var.get_shape().as_list())<EOL>blocks[block_name] += number_of_parameters<EOL><DEDENT>for block_name, cnt in blocks.items():<EOL><INDENT>log.info("<STR_LIT>".format(block_name, cnt))<EOL><DEDENT>total_num_parameters = np.sum(list(blocks.values()))<EOL>log.info('<STR_LIT>'.format(total_num_parameters))<EOL>

Print number of *trainable* parameters in the network


self.epochs_done = <NUM_LIT:0><EOL>self.batches_seen = <NUM_LIT:0><EOL>self.train_examples_seen = <NUM_LIT:0><EOL>super().__init__(save_path=kwargs.get("<STR_LIT>"),<EOL>load_path=kwargs.get("<STR_LIT>"),<EOL>mode=kwargs.get("<STR_LIT>"))<EOL>

Initialize model using keyword parameters

Args:
    kwargs (dict): Dictionary with model parameters


config = tf.ConfigProto()<EOL>config.gpu_options.allow_growth = True<EOL>config.gpu_options.visible_device_list = '<STR_LIT:0>'<EOL>return tf.Session(config=config)<EOL>

Configure session for particular device

Returns:
    tensorflow.Session


def process_event(self, event_name: str, data: dict) -> None:

if event_name == "<STR_LIT>":<EOL><INDENT>self.epochs_done = data["<STR_LIT>"]<EOL>self.batches_seen = data["<STR_LIT>"]<EOL>self.train_examples_seen = data["<STR_LIT>"]<EOL><DEDENT>return<EOL>

Process event after epoch
Args:
    event_name: whether event is send after epoch or batch.
            Set of values: ``"after_epoch", "after_batch"``
    data: event data (dictionary)

Returns:
    None


<EOL>if self.load_path.exists():<EOL><INDENT>path = str(self.load_path.resolve())<EOL>log.info('<STR_LIT>'.format(path))<EOL>self._net.load(path)<EOL><DEDENT>

Checks existence of the model file, loads the model if the file exists


path = str(self.save_path.absolute())<EOL>log.info('<STR_LIT>'.format(path))<EOL>self._net.save(path)<EOL>

Saves model to the save_path, provided in config. The directory is
        already created by super().__init__, which is called in __init__ of this class


def train_on_batch(self, *args) -> None:

*data, labels = args<EOL>self._net.train_on_batch(data, labels)<EOL>

Trains the model on a single batch.

        Args:
            *args: the list of network inputs.
            Last element of `args` is the batch of targets,
            all previous elements are training data batches


def __call__(self, *x_batch, **kwargs) -> Union[List, np.ndarray]:

with self.graph.as_default():<EOL><INDENT>K.set_session(self.sess)<EOL>return self._net.predict_on_batch(x_batch, **kwargs)<EOL><DEDENT>

Predicts answers on batch elements.

Args:
    instance: a batch to predict answers on


if isinstance(kwargs.get("<STR_LIT>"), float) and isinstance(kwargs.get("<STR_LIT>"), float):<EOL><INDENT>KerasModel.__init__(self, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>KerasModel.__init__(self, **kwargs)<EOL>LRScheduledModel.__init__(self, **kwargs)<EOL><DEDENT>

Initialize model with given parameters

Args:
    **kwargs: dictionary of parameters


pass<EOL>

Return instance of keras optimizer

Args:
    None


return None<EOL>

Initialize learning rate

Returns:
    None


return None<EOL>


return self.get_optimizer().lr<EOL>

Extract value of learning rate from optimizer

Returns:
    learning rate value


optimizer = self.get_optimizer()<EOL>if hasattr(optimizer, '<STR_LIT>'):<EOL><INDENT>return optimizer.rho<EOL><DEDENT>elif hasattr(optimizer, '<STR_LIT>'):<EOL><INDENT>return optimizer.beta_1<EOL><DEDENT>return None<EOL>

Extract values of momentum variables from optimizer

Returns:
    optimizer's `rho` or `beta_1`


@overrides<EOL><INDENT>def _update_graph_variables(self, learning_rate: float = None, momentum: float = None):<DEDENT>

if learning_rate is not None:<EOL><INDENT>K.set_value(self.get_learning_rate_variable(), learning_rate)<EOL><DEDENT>if momentum is not None:<EOL><INDENT>K.set_value(self.get_momentum_variable(), momentum)<EOL><DEDENT>

Update graph variables setting giving `learning_rate` and `momentum`

Args:
    learning_rate: learning rate value to be set in graph (set if not None)
    momentum: momentum value to be set in graph (set if not None)

Returns:
    None


def process_event(self, event_name: str, data: dict):

if (isinstance(self.opt.get("<STR_LIT>", None), float) and<EOL>isinstance(self.opt.get("<STR_LIT>", None), float)):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>if event_name == '<STR_LIT>':<EOL><INDENT>if (self.get_learning_rate_variable() is not None) and ('<STR_LIT>' not in data):<EOL><INDENT>data['<STR_LIT>'] = float(K.get_value(self.get_learning_rate_variable()))<EOL><DEDENT>if (self.get_momentum_variable() is not None) and ('<STR_LIT>' not in data):<EOL><INDENT>data['<STR_LIT>'] = float(K.get_value(self.get_momentum_variable()))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>super().process_event(event_name, data)<EOL><DEDENT><DEDENT>


@classmethod<EOL><INDENT>def from_str(cls, label: str) -> int:<DEDENT>

label_norm = label.replace('<STR_LIT:1>', '<STR_LIT>').upper()<EOL>if label_norm in cls.__members__:<EOL><INDENT>return DecayType[label_norm]<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError<EOL><DEDENT>

Convert given string label of decay type to special index

Args:
    label: name of decay type.
        Set of values: `"linear"`, `"cosine"`, `"exponential"`,
         `"onecycle"`, `"trapezoid"`, `["polynomial", K]`, where K is a polynomial power

Returns:
    index of decay type


@abstractmethod<EOL><INDENT>def _update_graph_variables(self, learning_rate=None, momentum=None):<DEDENT>

if learning_rate is not None:<EOL><INDENT>pass<EOL><DEDENT>if momentum is not None:<EOL><INDENT>pass<EOL><DEDENT>

Update learning rate in graph if `learning_rate` is not None,
Update momentum in graph if `momentum` is not None


def __init__(self,<EOL>learning_rate: Union[None, float, Tuple[float, float]] = None,<EOL>learning_rate_decay: Union[DType, Tuple[DType, float]] = DecayType.NO,<EOL>learning_rate_decay_epochs: int = <NUM_LIT:0>,<EOL>learning_rate_decay_batches: int = <NUM_LIT:0>,<EOL>learning_rate_drop_div: float = <NUM_LIT>,<EOL>learning_rate_drop_patience: Optional[int] = None,<EOL>momentum: Union[None, float, Tuple[float, float]] = None,<EOL>momentum_decay: Union[DType, Tuple[DType, float]] = DecayType.NO,<EOL>momentum_decay_epochs: int = <NUM_LIT:0>,<EOL>momentum_decay_batches: int = <NUM_LIT:0>,<EOL>fit_batch_size: Union[None, int, str] = None,<EOL>fit_learning_rate: Tuple[float, float] = (<NUM_LIT>, <NUM_LIT:100>),<EOL>fit_learning_rate_div: float = <NUM_LIT>,<EOL>fit_beta: float = <NUM_LIT>,<EOL>fit_min_batches: int = <NUM_LIT:10>,<EOL>fit_max_batches: Optional[int] = None,<EOL>*args, **kwargs) -> None:

if learning_rate_decay_epochs and learning_rate_decay_batches:<EOL><INDENT>raise ConfigError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if momentum_decay_epochs and momentum_decay_batches:<EOL><INDENT>raise ConfigError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>start_val, end_val = learning_rate, None<EOL>if isinstance(learning_rate, (tuple, list)):<EOL><INDENT>start_val, end_val = learning_rate<EOL><DEDENT>dec_type, extra = learning_rate_decay, None<EOL>if isinstance(learning_rate_decay, (tuple, list)):<EOL><INDENT>dec_type, extra = learning_rate_decay<EOL><DEDENT>self._lr = start_val<EOL>num_it, self._lr_update_on_batch = learning_rate_decay_epochs, False<EOL>if learning_rate_decay_batches > <NUM_LIT:0>:<EOL><INDENT>num_it, self._lr_update_on_batch = learning_rate_decay_batches, True<EOL><DEDENT>self._lr_schedule = DecayScheduler(start_val=start_val, end_val=end_val,<EOL>num_it=num_it, dec_type=dec_type, extra=extra)<EOL>self._lr_var = self._init_learning_rate_variable()<EOL>start_val, end_val = momentum, None<EOL>if isinstance(momentum, (tuple, list)):<EOL><INDENT>start_val, end_val = momentum<EOL><DEDENT>dec_type, extra = momentum_decay, None<EOL>if isinstance(momentum_decay, (tuple, list)):<EOL><INDENT>dec_type, extra = momentum_decay<EOL><DEDENT>self._mom = start_val<EOL>num_it, self._mom_update_on_batch = momentum_decay_epochs, False<EOL>self._mom_update_on_batch = momentum_decay_batches > <NUM_LIT:0><EOL>num_it = momentum_decay_epochs if self._mom_update_on_batch else momentum_decay_batches<EOL>self._mom_schedule = DecayScheduler(start_val=start_val, end_val=end_val,<EOL>num_it=num_it, dec_type=dec_type,<EOL>extra=extra)<EOL>self._mom_var = self._init_momentum_variable()<EOL>self._learning_rate_drop_patience = learning_rate_drop_patience<EOL>self._learning_rate_drop_div = learning_rate_drop_div<EOL>self._learning_rate_cur_impatience = <NUM_LIT:0.><EOL>self._learning_rate_last_impatience = <NUM_LIT:0.><EOL>self._learning_rate_cur_div = <NUM_LIT:1.><EOL>self._fit_batch_size = fit_batch_size<EOL>self._fit_learning_rate = fit_learning_rate<EOL>self._fit_learning_rate_div = fit_learning_rate_div<EOL>self._fit_beta = fit_beta<EOL>self._fit_min_batches = fit_min_batches<EOL>self._fit_max_batches = fit_max_batches<EOL>


if self._lr is None:<EOL><INDENT>raise ConfigError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>return self._lr<EOL>

Return current learning rate value

Returns:
    learning rate


return self._lr_var<EOL>

Return current learning rate variable

Returns:
    learning rate variable


return self._mom<EOL>

Return current momentum value

Returns:
    momentum


return self._mom_var<EOL>

Return current momentum variable

Returns:
    momentum variable


data = list(zip(*args))<EOL>self.save()<EOL>if self._fit_batch_size is None:<EOL><INDENT>raise ConfigError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>bs = int(self._fit_batch_size)<EOL>data_len = len(data)<EOL>num_batches = self._fit_max_batches or ((data_len - <NUM_LIT:1>) // bs + <NUM_LIT:1>)<EOL>avg_loss = <NUM_LIT:0.><EOL>best_loss = float('<STR_LIT>')<EOL>lrs, losses = [], []<EOL>_lr_find_schedule = DecayScheduler(start_val=self._fit_learning_rate[<NUM_LIT:0>],<EOL>end_val=self._fit_learning_rate[<NUM_LIT:1>],<EOL>dec_type="<STR_LIT>",<EOL>num_it=num_batches)<EOL>self._lr = _lr_find_schedule.start_val<EOL>self._mom = <NUM_LIT:0.><EOL>self._update_graph_variables(learning_rate=self._lr, momentum=self._mom)<EOL>best_lr = _lr_find_schedule.start_val<EOL>for i in range(num_batches):<EOL><INDENT>batch_start = (i * bs) % data_len<EOL>batch_end = batch_start + bs<EOL>report = self.train_on_batch(*zip(*data[batch_start:batch_end]))<EOL>if not isinstance(report, dict):<EOL><INDENT>report = {'<STR_LIT>': report}<EOL><DEDENT>avg_loss = self._fit_beta*avg_loss + (<NUM_LIT:1> - self._fit_beta)*report['<STR_LIT>']<EOL>smoothed_loss = avg_loss / (<NUM_LIT:1> - self._fit_beta**(i + <NUM_LIT:1>))<EOL>lrs.append(self._lr)<EOL>losses.append(smoothed_loss)<EOL>log.info(f"<STR_LIT>"<EOL>f"<STR_LIT>")<EOL>if math.isnan(smoothed_loss) or (smoothed_loss > <NUM_LIT:4> * best_loss):<EOL><INDENT>break<EOL><DEDENT>if (smoothed_loss < best_loss) and (i >= self._fit_min_batches):<EOL><INDENT>best_loss = smoothed_loss<EOL>best_lr = self._lr<EOL><DEDENT>self._lr = _lr_find_schedule.next_val()<EOL>self._update_graph_variables(learning_rate=self._lr)<EOL>if i >= num_batches:<EOL><INDENT>break<EOL><DEDENT><DEDENT>end_val = self._get_best(lrs, losses)<EOL>start_val = end_val<EOL>if self._lr_schedule.dec_type in (DecayType.ONECYCLE, DecayType.TRAPEZOID):<EOL><INDENT>start_val = end_val / self._fit_learning_rate_div<EOL><DEDENT>elif self._lr_schedule.dec_type in (DecayType.POLYNOMIAL, DecayType.EXPONENTIAL,<EOL>DecayType.LINEAR, DecayType.COSINE):<EOL><INDENT>start_val = end_val<EOL>end_val = end_val / self._fit_learning_rate_div<EOL><DEDENT>self._lr_schedule = DecayScheduler(start_val=start_val,<EOL>end_val=end_val,<EOL>num_it=self._lr_schedule.nb,<EOL>dec_type=self._lr_schedule.dec_type,<EOL>extra=self._lr_schedule.extra)<EOL>log.info(f"<STR_LIT>"<EOL>f"<STR_LIT>")<EOL>self.load()<EOL>self._lr = self._lr_schedule.start_val<EOL>self._mom = self._mom_schedule.start_val<EOL>self._update_graph_variables(learning_rate=self._lr, momentum=self._mom)<EOL>return {'<STR_LIT>': losses, '<STR_LIT>': lrs}<EOL>

Find the best learning rate schedule, and set obtained values of learning rate
and momentum for further model training. Best learning rate will be divided
by `fit_learning_rate_div` for further training model.

Args:
    *args: arguments

Returns:


@staticmethod<EOL><INDENT>def _get_best(values: List[float], losses: List[float],<EOL>max_loss_div: float = <NUM_LIT>, min_val_div: float = <NUM_LIT>) -> float:<DEDENT>

assert len(values) == len(losses), "<STR_LIT>"<EOL>min_ind = np.argmin(losses)<EOL>for i in range(min_ind - <NUM_LIT:1>, <NUM_LIT:0>, -<NUM_LIT:1>):<EOL><INDENT>if (losses[i] * max_loss_div > losses[min_ind]) or(values[i] * min_val_div < values[min_ind]):<EOL><INDENT>return values[i + <NUM_LIT:1>]<EOL><DEDENT><DEDENT>return values[min_ind] / min_val_div<EOL>

Find the best value according to given losses

Args:
    values: list of considered values
    losses: list of obtained loss values corresponding to `values`
    max_loss_div: maximal divergence of loss to be considered significant
    min_val_div: minimum divergence of loss to be considered significant

Returns:
    best value divided by `min_val_div`


if event_name == "<STR_LIT>":<EOL><INDENT>if data['<STR_LIT>'] > self._learning_rate_last_impatience:<EOL><INDENT>self._learning_rate_cur_impatience += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>self._learning_rate_cur_impatience = <NUM_LIT:0><EOL><DEDENT>self._learning_rate_last_impatience = data['<STR_LIT>']<EOL>if (self._learning_rate_drop_patience is not None) and(self._learning_rate_cur_impatience >=<EOL>self._learning_rate_drop_patience):<EOL><INDENT>self._learning_rate_cur_impatience = <NUM_LIT:0><EOL>self._learning_rate_cur_div *= self._learning_rate_drop_div<EOL>self._lr /= self._learning_rate_drop_div<EOL>self._update_graph_variables(learning_rate=self._lr)<EOL>log.info(f"<STR_LIT>")<EOL><DEDENT><DEDENT>if event_name == '<STR_LIT>':<EOL><INDENT>if (self._lr is not None) and self._lr_update_on_batch:<EOL><INDENT>self._lr = self._lr_schedule.next_val() / self._learning_rate_cur_div<EOL>self._update_graph_variables(learning_rate=self._lr)<EOL><DEDENT>if (self._mom is not None) and self._mom_update_on_batch:<EOL><INDENT>self._mom = min(<NUM_LIT:1.>, max(<NUM_LIT:0.>, self._mom_schedule.next_val()))<EOL>self._update_graph_variables(momentum=self._mom)<EOL><DEDENT><DEDENT>if event_name == '<STR_LIT>':<EOL><INDENT>if (self._lr is not None) and not self._lr_update_on_batch:<EOL><INDENT>self._lr = self._lr_schedule.next_val() / self._learning_rate_cur_div<EOL>self._update_graph_variables(learning_rate=self._lr)<EOL><DEDENT>if (self._mom is not None) and not self._mom_update_on_batch:<EOL><INDENT>self._mom = min(<NUM_LIT:1.>, max(<NUM_LIT:0.>, self._mom_schedule.next_val()))<EOL>self._update_graph_variables(momentum=self._mom)<EOL><DEDENT><DEDENT>if event_name == '<STR_LIT>':<EOL><INDENT>if (self._lr is not None) and ('<STR_LIT>' not in data):<EOL><INDENT>data['<STR_LIT>'] = self._lr<EOL><DEDENT>if (self._mom is not None) and ('<STR_LIT>' not in data):<EOL><INDENT>data['<STR_LIT>'] = self._mom<EOL><DEDENT><DEDENT>

Update learning rate and momentum variables after event (given by `event_name`)

Args:
    event_name: name of event after which the method was called.
            Set of values: `"after_validation"`, `"after_batch"`, `"after_epoch"`, `"after_train_log"`
    data: dictionary with parameters values

Returns:
    None


l2_reg = tf.nn.l2_loss if add_l2_losses else None<EOL>for n_layer, n_hidden in enumerate(n_hidden_list):<EOL><INDENT>if use_dilation:<EOL><INDENT>dilation_rate = <NUM_LIT:2> ** n_layer<EOL><DEDENT>else:<EOL><INDENT>dilation_rate = <NUM_LIT:1><EOL><DEDENT>units = tf.layers.conv1d(units,<EOL>n_hidden,<EOL>filter_width,<EOL>padding='<STR_LIT>',<EOL>dilation_rate=dilation_rate,<EOL>kernel_initializer=INITIALIZER(),<EOL>kernel_regularizer=l2_reg)<EOL>if use_batch_norm:<EOL><INDENT>assert training_ph is not None<EOL>units = tf.layers.batch_normalization(units, training=training_ph)<EOL><DEDENT>units = tf.nn.relu(units)<EOL><DEDENT>return units<EOL>

Number of convolutional layers stacked on top of each other

    Args:
        units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
        n_hidden_list: list with number of hidden units at the ouput of each layer
        filter_width: width of the kernel in tokens
        use_batch_norm: whether to use batch normalization between layers
        use_dilation: use power of 2 dilation scheme [1, 2, 4, 8 .. ] for layers 1, 2, 3, 4 ...
        training_ph: boolean placeholder determining whether is training phase now or not.
            It is used only for batch normalization to determine whether to use
            current batch average (std) or memory stored average (std)
        add_l2_losses: whether to add l2 losses on network kernels to
                tf.GraphKeys.REGULARIZATION_LOSSES or not

    Returns:
        units: tensor at the output of the last convolutional layer


units_list = [units]<EOL>for n_layer, n_filters in enumerate(n_hidden_list):<EOL><INDENT>total_units = tf.concat(units_list, axis=-<NUM_LIT:1>)<EOL>if use_dilation:<EOL><INDENT>dilation_rate = <NUM_LIT:2> ** n_layer<EOL><DEDENT>else:<EOL><INDENT>dilation_rate = <NUM_LIT:1><EOL><DEDENT>units = tf.layers.conv1d(total_units,<EOL>n_filters,<EOL>filter_width,<EOL>dilation_rate=dilation_rate,<EOL>padding='<STR_LIT>',<EOL>kernel_initializer=INITIALIZER())<EOL>if use_batch_norm:<EOL><INDENT>units = tf.layers.batch_normalization(units, training=training_ph)<EOL><DEDENT>units = tf.nn.relu(units)<EOL>units_list.append(units)<EOL><DEDENT>return units<EOL>

Densely connected convolutional layers. Based on the paper:
        [Gao 17] https://arxiv.org/abs/1608.06993

        Args:
            units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
            n_hidden_list: list with number of hidden units at the ouput of each layer
            filter_width: width of the kernel in tokens
            use_batch_norm: whether to use batch normalization between layers
            use_dilation: use power of 2 dilation scheme [1, 2, 4, 8 .. ] for layers 1, 2, 3, 4 ...
            training_ph: boolean placeholder determining whether is training phase now or not.
                It is used only for batch normalization to determine whether to use
                current batch average (std) or memory stored average (std)
        Returns:
            units: tensor at the output of the last convolutional layer
                with dimensionality [None, n_tokens, n_hidden_list[-1]]


with tf.variable_scope(name + '<STR_LIT:_>' + cell_type.upper()):<EOL><INDENT>if cell_type == '<STR_LIT>':<EOL><INDENT>forward_cell = tf.nn.rnn_cell.GRUCell(n_hidden, kernel_initializer=INITIALIZER())<EOL>backward_cell = tf.nn.rnn_cell.GRUCell(n_hidden, kernel_initializer=INITIALIZER())<EOL>if trainable_initial_states:<EOL><INDENT>initial_state_fw = tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL>initial_state_bw = tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>initial_state_fw = initial_state_bw = None<EOL><DEDENT><DEDENT>elif cell_type == '<STR_LIT>':<EOL><INDENT>forward_cell = tf.nn.rnn_cell.LSTMCell(n_hidden, use_peepholes=use_peepholes, initializer=INITIALIZER())<EOL>backward_cell = tf.nn.rnn_cell.LSTMCell(n_hidden, use_peepholes=use_peepholes, initializer=INITIALIZER())<EOL>if trainable_initial_states:<EOL><INDENT>initial_state_fw = tf.nn.rnn_cell.LSTMStateTuple(<EOL>tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>)),<EOL>tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>)))<EOL>initial_state_bw = tf.nn.rnn_cell.LSTMStateTuple(<EOL>tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>)),<EOL>tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>)))<EOL><DEDENT>else:<EOL><INDENT>initial_state_fw = initial_state_bw = None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>(rnn_output_fw, rnn_output_bw), (fw, bw) =tf.nn.bidirectional_dynamic_rnn(forward_cell,<EOL>backward_cell,<EOL>units,<EOL>dtype=tf.float32,<EOL>sequence_length=seq_lengths,<EOL>initial_state_fw=initial_state_fw,<EOL>initial_state_bw=initial_state_bw)<EOL><DEDENT>kernels = [var for var in forward_cell.trainable_variables +<EOL>backward_cell.trainable_variables if '<STR_LIT>' in var.name]<EOL>for kernel in kernels:<EOL><INDENT>tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, tf.nn.l2_loss(kernel))<EOL><DEDENT>return (rnn_output_fw, rnn_output_bw), (fw, bw)<EOL>

Bi directional recurrent neural network. GRU or LSTM

        Args:
            units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
            n_hidden: list with number of hidden units at the ouput of each layer
            seq_lengths: length of sequences for different length sequences in batch
                can be None for maximum length as a length for every sample in the batch
            cell_type: 'lstm' or 'gru'
            trainable_initial_states: whether to create a special trainable variable
                to initialize the hidden states of the network or use just zeros
            use_peepholes: whether to use peephole connections (only 'lstm' case affected)
            name: what variable_scope to use for the network parameters

        Returns:
            units: tensor at the output of the last recurrent layer
                with dimensionality [None, n_tokens, n_hidden_list[-1]]
            last_units: tensor of last hidden states for GRU and tuple
                of last hidden stated and last cell states for LSTM
                dimensionality of cell states and hidden states are
                similar and equal to [B x 2 * H], where B - batch
                size and H is number of hidden units


for n, n_hidden in enumerate(n_hidden_list):<EOL><INDENT>with tf.variable_scope(name + '<STR_LIT:_>' + str(n)):<EOL><INDENT>if cell_type == '<STR_LIT>':<EOL><INDENT>forward_cell = tf.nn.rnn_cell.GRUCell(n_hidden)<EOL>backward_cell = tf.nn.rnn_cell.GRUCell(n_hidden)<EOL><DEDENT>elif cell_type == '<STR_LIT>':<EOL><INDENT>forward_cell = tf.nn.rnn_cell.LSTMCell(n_hidden, use_peepholes=use_peepholes)<EOL>backward_cell = tf.nn.rnn_cell.LSTMCell(n_hidden, use_peepholes=use_peepholes)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>(rnn_output_fw, rnn_output_bw), (fw, bw) =tf.nn.bidirectional_dynamic_rnn(forward_cell,<EOL>backward_cell,<EOL>units,<EOL>dtype=tf.float32,<EOL>sequence_length=seq_lengths)<EOL>units = tf.concat([rnn_output_fw, rnn_output_bw], axis=<NUM_LIT:2>)<EOL>if cell_type == '<STR_LIT>':<EOL><INDENT>last_units = tf.concat([fw, bw], axis=<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>(c_fw, h_fw), (c_bw, h_bw) = fw, bw<EOL>c = tf.concat([c_fw, c_bw], axis=<NUM_LIT:1>)<EOL>h = tf.concat([h_fw, h_bw], axis=<NUM_LIT:1>)<EOL>last_units = (h, c)<EOL><DEDENT><DEDENT><DEDENT>return units, last_units<EOL>

Stackted recurrent neural networks GRU or LSTM

        Args:
            units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
            n_hidden_list: list with number of hidden units at the ouput of each layer
            seq_lengths: length of sequences for different length sequences in batch
                can be None for maximum length as a length for every sample in the batch
            cell_type: 'lstm' or 'gru'
            use_peepholes: whether to use peephole connections (only 'lstm' case affected)
            name: what variable_scope to use for the network parameters
        Returns:
            units: tensor at the output of the last recurrent layer
                with dimensionality [None, n_tokens, n_hidden_list[-1]]
            last_units: tensor of last hidden states for GRU and tuple
                of last hidden stated and last cell states for LSTM
                dimensionality of cell states and hidden states are
                similar and equal to [B x 2 * H], where B - batch
                size and H is number of hidden units


<EOL>units_for_skip_conn = []<EOL>conv_net_params = {'<STR_LIT>': filter_width,<EOL>'<STR_LIT>': use_batch_norm,<EOL>'<STR_LIT>': training_ph}<EOL>for n_hidden in n_hidden_list:<EOL><INDENT>units = stacked_cnn(units, [n_hidden], **conv_net_params)<EOL>units_for_skip_conn.append(units)<EOL>units = tf.layers.max_pooling1d(units, pool_size=<NUM_LIT:2>, strides=<NUM_LIT:2>, padding='<STR_LIT>')<EOL><DEDENT>units = stacked_cnn(units, [n_hidden], **conv_net_params)<EOL>for down_step, n_hidden in enumerate(n_hidden_list[::-<NUM_LIT:1>]):<EOL><INDENT>units = tf.expand_dims(units, axis=<NUM_LIT:2>)<EOL>units = tf.layers.conv2d_transpose(units, n_hidden, filter_width, strides=(<NUM_LIT:2>, <NUM_LIT:1>), padding='<STR_LIT>')<EOL>units = tf.squeeze(units, axis=<NUM_LIT:2>)<EOL>skip_units = units_for_skip_conn[-(down_step + <NUM_LIT:1>)]<EOL>if skip_units.get_shape().as_list()[-<NUM_LIT:1>] != n_hidden:<EOL><INDENT>skip_units = tf.layers.dense(skip_units, n_hidden)<EOL><DEDENT>units = skip_units + units<EOL>units = stacked_cnn(units, [n_hidden], **conv_net_params)<EOL><DEDENT>return units<EOL>

Network architecture inspired by One Hundred layer Tiramisu.
        https://arxiv.org/abs/1611.09326. U-Net like.

        Args:
            units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
            n_hidden_list: list with number of hidden units at the ouput of each layer
            filter_width: width of the kernel in tokens
            use_batch_norm: whether to use batch normalization between layers
            training_ph: boolean placeholder determining whether is training phase now or not.
                It is used only for batch normalization to determine whether to use
                current batch average (std) or memory stored average (std)
        Returns:
            units: tensor at the output of the last convolutional layer
                    with dimensionality [None, n_tokens, n_hidden_list[-1]]


for n_layer, n_hidden in enumerate(n_hidden_list):<EOL><INDENT>input_units = units<EOL>if input_units.get_shape().as_list()[-<NUM_LIT:1>] != n_hidden:<EOL><INDENT>input_units = tf.layers.dense(input_units, n_hidden)<EOL><DEDENT>if use_dilation:<EOL><INDENT>dilation_rate = <NUM_LIT:2> ** n_layer<EOL><DEDENT>else:<EOL><INDENT>dilation_rate = <NUM_LIT:1><EOL><DEDENT>units = tf.layers.conv1d(units,<EOL>n_hidden,<EOL>filter_width,<EOL>padding='<STR_LIT>',<EOL>dilation_rate=dilation_rate,<EOL>kernel_initializer=INITIALIZER())<EOL>if use_batch_norm:<EOL><INDENT>units = tf.layers.batch_normalization(units, training=training_ph)<EOL><DEDENT>sigmoid_gate = tf.layers.dense(input_units, <NUM_LIT:1>, activation=tf.sigmoid, kernel_initializer=INITIALIZER())<EOL>input_units = sigmoid_gate * input_units + (<NUM_LIT:1> - sigmoid_gate) * units<EOL>input_units = tf.nn.relu(input_units)<EOL><DEDENT>units = input_units<EOL>return units<EOL>

Highway convolutional network. Skip connection with gating
        mechanism.

    Args:
        units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
        n_hidden_list: list with number of hidden units at the output of each layer
        filter_width: width of the kernel in tokens
        use_batch_norm: whether to use batch normalization between layers
        use_dilation: use power of 2 dilation scheme [1, 2, 4, 8 .. ] for layers 1, 2, 3, 4 ...
        training_ph: boolean placeholder determining whether is training phase now or not.
            It is used only for batch normalization to determine whether to use
            current batch average (std) or memory stored average (std)
    Returns:
        units: tensor at the output of the last convolutional layer
                with dimensionality [None, n_tokens, n_hidden_list[-1]]


if token_embedding_matrix is not None:<EOL><INDENT>tok_mat = token_embedding_matrix<EOL>if trainable:<EOL><INDENT>Warning('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>tok_mat = np.random.randn(n_tokens, token_embedding_dim).astype(np.float32) / np.sqrt(token_embedding_dim)<EOL><DEDENT>tok_emb_mat = tf.Variable(tok_mat, name=name, trainable=trainable)<EOL>embedded_tokens = tf.nn.embedding_lookup(tok_emb_mat, token_indices)<EOL>return embedded_tokens<EOL>

Token embedding layer. Create matrix of for token embeddings.
        Can be initialized with given matrix (for example pre-trained
        with word2ve algorithm

    Args:
        token_indices: token indices tensor of type tf.int32
        token_embedding_matrix: matrix of embeddings with dimensionality
            [n_tokens, embeddings_dimension]
        n_tokens: total number of unique tokens
        token_embedding_dim: dimensionality of embeddings, typical 100..300
        name: embedding matrix name (variable name)
        trainable: whether to set the matrix trainable or not

    Returns:
        embedded_tokens: tf tensor of size [B, T, E], where B - batch size
            T - number of tokens, E - token_embedding_dim


def character_embedding_network(char_placeholder: tf.Tensor,<EOL>n_characters: int = None,<EOL>emb_mat: np.array = None,<EOL>char_embedding_dim: int = None,<EOL>filter_widths=(<NUM_LIT:3>, <NUM_LIT:4>, <NUM_LIT:5>, <NUM_LIT:7>),<EOL>highway_on_top=False):

if emb_mat is None:<EOL><INDENT>emb_mat = np.random.randn(n_characters, char_embedding_dim).astype(np.float32) / np.sqrt(char_embedding_dim)<EOL><DEDENT>else:<EOL><INDENT>char_embedding_dim = emb_mat.shape[<NUM_LIT:1>]<EOL><DEDENT>char_emb_var = tf.Variable(emb_mat, trainable=True)<EOL>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>c_emb = tf.nn.embedding_lookup(char_emb_var, char_placeholder)<EOL>conv_results_list = []<EOL>for filter_width in filter_widths:<EOL><INDENT>conv_results_list.append(tf.layers.conv2d(c_emb,<EOL>char_embedding_dim,<EOL>(<NUM_LIT:1>, filter_width),<EOL>padding='<STR_LIT>',<EOL>kernel_initializer=INITIALIZER))<EOL><DEDENT>units = tf.concat(conv_results_list, axis=<NUM_LIT:3>)<EOL>units = tf.reduce_max(units, axis=<NUM_LIT:2>)<EOL>if highway_on_top:<EOL><INDENT>sigmoid_gate = tf.layers.dense(units,<EOL><NUM_LIT:1>,<EOL>activation=tf.sigmoid,<EOL>kernel_initializer=INITIALIZER,<EOL>kernel_regularizer=tf.nn.l2_loss)<EOL>deeper_units = tf.layers.dense(units,<EOL>tf.shape(units)[-<NUM_LIT:1>],<EOL>kernel_initializer=INITIALIZER,<EOL>kernel_regularizer=tf.nn.l2_loss)<EOL>units = sigmoid_gate * units + (<NUM_LIT:1> - sigmoid_gate) * deeper_units<EOL>units = tf.nn.relu(units)<EOL><DEDENT><DEDENT>return units<EOL>

Characters to vector. Every sequence of characters (token)
        is embedded to vector space with dimensionality char_embedding_dim
        Convolution plus max_pooling is used to obtain vector representations
        of words.

    Args:
        char_placeholder: placeholder of int32 type with dimensionality [B, T, C]
            B - batch size (can be None)
            T - Number of tokens (can be None)
            C - number of characters (can be None)
        n_characters: total number of unique characters
        emb_mat: if n_characters is not provided the emb_mat should be provided
            it is a numpy array with dimensions [V, E], where V - vocabulary size
            and E - embeddings dimension
        char_embedding_dim: dimensionality of characters embeddings
        filter_widths: array of width of kernel in convolutional embedding network
            used in parallel

    Returns:
        embeddings: tf.Tensor with dimensionality [B, T, F],
            where F is dimensionality of embeddings


assert axis in (<NUM_LIT:1>, <NUM_LIT:2>)<EOL>n_time_steps = tf.shape(units)[<NUM_LIT:1>]<EOL>repetitions = [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>]<EOL>repetitions[axis] = n_time_steps<EOL>return tf.tile(tf.expand_dims(units, axis), repetitions)<EOL>

Expand and tile tensor along given axis
    Args:
        units: tf tensor with dimensions [batch_size, time_steps, n_input_features]
        axis: axis along which expand and tile. Must be 1 or 2


def additive_self_attention(units, n_hidden=None, n_output_features=None, activation=None):

n_input_features = units.get_shape().as_list()[<NUM_LIT:2>]<EOL>if n_hidden is None:<EOL><INDENT>n_hidden = n_input_features<EOL><DEDENT>if n_output_features is None:<EOL><INDENT>n_output_features = n_input_features<EOL><DEDENT>units_pairs = tf.concat([expand_tile(units, <NUM_LIT:1>), expand_tile(units, <NUM_LIT:2>)], <NUM_LIT:3>)<EOL>query = tf.layers.dense(units_pairs, n_hidden, activation=tf.tanh, kernel_initializer=INITIALIZER())<EOL>attention = tf.nn.softmax(tf.layers.dense(query, <NUM_LIT:1>), dim=<NUM_LIT:2>)<EOL>attended_units = tf.reduce_sum(attention * expand_tile(units, <NUM_LIT:1>), axis=<NUM_LIT:2>)<EOL>output = tf.layers.dense(attended_units, n_output_features, activation, kernel_initializer=INITIALIZER())<EOL>return output<EOL>

Computes additive self attention for time series of vectors (with batch dimension)
        the formula: score(h_i, h_j) = <v, tanh(W_1 h_i + W_2 h_j)>
        v is a learnable vector of n_hidden dimensionality,
        W_1 and W_2 are learnable [n_hidden, n_input_features] matrices

    Args:
        units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
        n_hidden: number of units in hidden representation of similarity measure
        n_output_features: number of features in output dense layer
        activation: activation at the output

    Returns:
        output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]


def multiplicative_self_attention(units, n_hidden=None, n_output_features=None, activation=None):

n_input_features = units.get_shape().as_list()[<NUM_LIT:2>]<EOL>if n_hidden is None:<EOL><INDENT>n_hidden = n_input_features<EOL><DEDENT>if n_output_features is None:<EOL><INDENT>n_output_features = n_input_features<EOL><DEDENT>queries = tf.layers.dense(expand_tile(units, <NUM_LIT:1>), n_hidden, kernel_initializer=INITIALIZER())<EOL>keys = tf.layers.dense(expand_tile(units, <NUM_LIT:2>), n_hidden, kernel_initializer=INITIALIZER())<EOL>scores = tf.reduce_sum(queries * keys, axis=<NUM_LIT:3>, keep_dims=True)<EOL>attention = tf.nn.softmax(scores, dim=<NUM_LIT:2>)<EOL>attended_units = tf.reduce_sum(attention * expand_tile(units, <NUM_LIT:1>), axis=<NUM_LIT:2>)<EOL>output = tf.layers.dense(attended_units, n_output_features, activation, kernel_initializer=INITIALIZER())<EOL>return output<EOL>

Computes multiplicative self attention for time series of vectors (with batch dimension)
        the formula: score(h_i, h_j) = <W_1 h_i,  W_2 h_j>,  W_1 and W_2 are learnable matrices
        with dimensionality [n_hidden, n_input_features], where <a, b> stands for a and b
        dot product

    Args:
        units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
        n_hidden: number of units in hidden representation of similarity measure
        n_output_features: number of features in output dense layer
        activation: activation at the output

    Returns:
        output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]


def cudnn_gru(units, n_hidden, n_layers=<NUM_LIT:1>, trainable_initial_states=False,<EOL>seq_lengths=None, input_initial_h=None, name='<STR_LIT>', reuse=False):

with tf.variable_scope(name, reuse=reuse):<EOL><INDENT>gru = tf.contrib.cudnn_rnn.CudnnGRU(num_layers=n_layers,<EOL>num_units=n_hidden)<EOL>if trainable_initial_states:<EOL><INDENT>init_h = tf.get_variable('<STR_LIT>', [n_layers, <NUM_LIT:1>, n_hidden])<EOL>init_h = tf.tile(init_h, (<NUM_LIT:1>, tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>init_h = tf.zeros([n_layers, tf.shape(units)[<NUM_LIT:0>], n_hidden])<EOL><DEDENT>initial_h = input_initial_h or init_h<EOL>h, h_last = gru(tf.transpose(units, (<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>)), (initial_h, ))<EOL>h = tf.transpose(h, (<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>))<EOL>h_last = tf.squeeze(h_last, axis=<NUM_LIT:0>)[-<NUM_LIT:1>]  <EOL>if seq_lengths is not None:<EOL><INDENT>indices = tf.stack([tf.range(tf.shape(h)[<NUM_LIT:0>]), seq_lengths-<NUM_LIT:1>], axis=<NUM_LIT:1>)<EOL>h_last = tf.gather_nd(h, indices)<EOL><DEDENT>return h, h_last<EOL><DEDENT>

Fast CuDNN GRU implementation

    Args:
        units: tf.Tensor with dimensions [B x T x F], where
            B - batch size
            T - number of tokens
            F - features

        n_hidden: dimensionality of hidden state
        trainable_initial_states: whether to create a special trainable variable
            to initialize the hidden states of the network or use just zeros
        seq_lengths: tensor of sequence lengths with dimension [B]
        n_layers: number of layers
        input_initial_h: initial hidden state, tensor
        name: name of the variable scope to use
        reuse:whether to reuse already initialized variable

    Returns:
        h - all hidden states along T dimension,
            tf.Tensor with dimensionality [B x T x F]
        h_last - last hidden state, tf.Tensor with dimensionality [B x H]


def cudnn_compatible_gru(units, n_hidden, n_layers=<NUM_LIT:1>, trainable_initial_states=False,<EOL>seq_lengths=None, input_initial_h=None, name='<STR_LIT>', reuse=False):

with tf.variable_scope(name, reuse=reuse):<EOL><INDENT>if trainable_initial_states:<EOL><INDENT>init_h = tf.get_variable('<STR_LIT>', [n_layers, <NUM_LIT:1>, n_hidden])<EOL>init_h = tf.tile(init_h, (<NUM_LIT:1>, tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>init_h = tf.zeros([n_layers, tf.shape(units)[<NUM_LIT:0>], n_hidden])<EOL><DEDENT>initial_h = input_initial_h or init_h<EOL>with tf.variable_scope('<STR_LIT>', reuse=reuse):<EOL><INDENT>def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleGRUCell(n_hidden)<EOL>cell = tf.nn.rnn_cell.MultiRNNCell([single_cell() for _ in range(n_layers)])<EOL>units = tf.transpose(units, (<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>))<EOL>h, h_last = tf.nn.dynamic_rnn(cell=cell, inputs=units, time_major=True,<EOL>initial_state=tuple(tf.unstack(initial_h, axis=<NUM_LIT:0>)))<EOL>h = tf.transpose(h, (<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>))<EOL>h_last = h_last[-<NUM_LIT:1>]  <EOL>if seq_lengths is not None:<EOL><INDENT>indices = tf.stack([tf.range(tf.shape(h)[<NUM_LIT:0>]), seq_lengths-<NUM_LIT:1>], axis=<NUM_LIT:1>)<EOL>h_last = tf.gather_nd(h, indices)<EOL><DEDENT>return h, h_last<EOL><DEDENT><DEDENT>

CuDNN Compatible GRU implementation.
        It should be used to load models saved with CudnnGRUCell to run on CPU.

        Args:
            units: tf.Tensor with dimensions [B x T x F], where
                B - batch size
                T - number of tokens
                F - features

            n_hidden: dimensionality of hidden state
            trainable_initial_states: whether to create a special trainable variable
                to initialize the hidden states of the network or use just zeros
            seq_lengths: tensor of sequence lengths with dimension [B]
            n_layers: number of layers
            input_initial_h: initial hidden state, tensor
            name: name of the variable scope to use
            reuse:whether to reuse already initialized variable

        Returns:
            h - all hidden states along T dimension,
                tf.Tensor with dimensionality [B x T x F]
            h_last - last hidden state, tf.Tensor with dimensionality [B x H]


def cudnn_lstm(units, n_hidden, n_layers=<NUM_LIT:1>, trainable_initial_states=None, seq_lengths=None, initial_h=None,<EOL>initial_c=None, name='<STR_LIT>', reuse=False):

with tf.variable_scope(name, reuse=reuse):<EOL><INDENT>lstm = tf.contrib.cudnn_rnn.CudnnLSTM(num_layers=n_layers,<EOL>num_units=n_hidden)<EOL>if trainable_initial_states:<EOL><INDENT>init_h = tf.get_variable('<STR_LIT>', [n_layers, <NUM_LIT:1>, n_hidden])<EOL>init_h = tf.tile(init_h, (<NUM_LIT:1>, tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL>init_c = tf.get_variable('<STR_LIT>', [n_layers, <NUM_LIT:1>, n_hidden])<EOL>init_c = tf.tile(init_c, (<NUM_LIT:1>, tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>init_h = init_c = tf.zeros([n_layers, tf.shape(units)[<NUM_LIT:0>], n_hidden])<EOL><DEDENT>initial_h = initial_h or init_h<EOL>initial_c = initial_c or init_c<EOL>h, (h_last, c_last) = lstm(tf.transpose(units, (<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>)), (initial_h, initial_c))<EOL>h = tf.transpose(h, (<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>))<EOL>h_last = h_last[-<NUM_LIT:1>]<EOL>c_last = c_last[-<NUM_LIT:1>]<EOL>if seq_lengths is not None:<EOL><INDENT>indices = tf.stack([tf.range(tf.shape(h)[<NUM_LIT:0>]), seq_lengths-<NUM_LIT:1>], axis=<NUM_LIT:1>)<EOL>h_last = tf.gather_nd(h, indices)<EOL><DEDENT>return h, (h_last, c_last)<EOL><DEDENT>

Fast CuDNN LSTM implementation

        Args:
            units: tf.Tensor with dimensions [B x T x F], where
                B - batch size
                T - number of tokens
                F - features
            n_hidden: dimensionality of hidden state
            n_layers: number of layers
            trainable_initial_states: whether to create a special trainable variable
                to initialize the hidden states of the network or use just zeros
            seq_lengths: tensor of sequence lengths with dimension [B]
            initial_h: optional initial hidden state, masks trainable_initial_states
                if provided
            initial_c: optional initial cell state, masks trainable_initial_states
                if provided
            name: name of the variable scope to use
            reuse:whether to reuse already initialized variable

        Returns:
            h - all hidden states along T dimension,
                tf.Tensor with dimensionality [B x T x F]
            h_last - last hidden state, tf.Tensor with dimensionality [B x H]
                where H - number of hidden units
            c_last - last cell state, tf.Tensor with dimensionality [B x H]
                where H - number of hidden units


def cudnn_compatible_lstm(units, n_hidden, n_layers=<NUM_LIT:1>, trainable_initial_states=None, seq_lengths=None, initial_h=None,<EOL>initial_c=None, name='<STR_LIT>', reuse=False):

with tf.variable_scope(name, reuse=reuse):<EOL><INDENT>if trainable_initial_states:<EOL><INDENT>init_h = tf.get_variable('<STR_LIT>', [n_layers, <NUM_LIT:1>, n_hidden])<EOL>init_h = tf.tile(init_h, (<NUM_LIT:1>, tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL>init_c = tf.get_variable('<STR_LIT>', [n_layers, <NUM_LIT:1>, n_hidden])<EOL>init_c = tf.tile(init_c, (<NUM_LIT:1>, tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>init_h = init_c = tf.zeros([n_layers, tf.shape(units)[<NUM_LIT:0>], n_hidden])<EOL><DEDENT>initial_h = initial_h or init_h<EOL>initial_c = initial_c or init_c<EOL>with tf.variable_scope('<STR_LIT>', reuse=reuse):<EOL><INDENT>def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(n_hidden)<EOL>cell = tf.nn.rnn_cell.MultiRNNCell([single_cell() for _ in range(n_layers)])<EOL>units = tf.transpose(units, (<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>))<EOL>init = tuple([tf.nn.rnn_cell.LSTMStateTuple(ic, ih) for ih, ic in<EOL>zip(tf.unstack(initial_h, axis=<NUM_LIT:0>), tf.unstack(initial_c, axis=<NUM_LIT:0>))])<EOL>h, state = tf.nn.dynamic_rnn(cell=cell, inputs=units, time_major=True, initial_state=init)<EOL>h = tf.transpose(h, (<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>))<EOL>h_last = state[-<NUM_LIT:1>].h<EOL>c_last = state[-<NUM_LIT:1>].c<EOL>if seq_lengths is not None:<EOL><INDENT>indices = tf.stack([tf.range(tf.shape(h)[<NUM_LIT:0>]), seq_lengths-<NUM_LIT:1>], axis=<NUM_LIT:1>)<EOL>h_last = tf.gather_nd(h, indices)<EOL><DEDENT>return h, (h_last, c_last)<EOL><DEDENT><DEDENT>

CuDNN Compatible LSTM implementation.
        It should be used to load models saved with CudnnLSTMCell to run on CPU.

        Args:
            units: tf.Tensor with dimensions [B x T x F], where
                B - batch size
                T - number of tokens
                F - features
            n_hidden: dimensionality of hidden state
            n_layers: number of layers
            trainable_initial_states: whether to create a special trainable variable
                to initialize the hidden states of the network or use just zeros
            seq_lengths: tensor of sequence lengths with dimension [B]
            initial_h: optional initial hidden state, masks trainable_initial_states
                if provided
            initial_c: optional initial cell state, masks trainable_initial_states
                if provided
            name: name of the variable scope to use
            reuse:whether to reuse already initialized variable

        Returns:
            h - all hidden states along T dimension,
                tf.Tensor with dimensionality [B x T x F]
            h_last - last hidden state, tf.Tensor with dimensionality [B x H]
                where H - number of hidden units
            c_last - last cell state, tf.Tensor with dimensionality [B x H]
                where H - number of hidden units


with tf.variable_scope(name, reuse=reuse):<EOL><INDENT>if seq_lengths is None:<EOL><INDENT>seq_lengths = tf.ones([tf.shape(units)[<NUM_LIT:0>]], dtype=tf.int32) * tf.shape(units)[<NUM_LIT:1>]<EOL><DEDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>h_fw, h_last_fw = cudnn_gru_wrapper(units,<EOL>n_hidden,<EOL>n_layers=n_layers,<EOL>trainable_initial_states=trainable_initial_states,<EOL>seq_lengths=seq_lengths,<EOL>reuse=reuse)<EOL><DEDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>reversed_units = tf.reverse_sequence(units, seq_lengths=seq_lengths, seq_dim=<NUM_LIT:1>, batch_dim=<NUM_LIT:0>)<EOL>h_bw, h_last_bw = cudnn_gru_wrapper(reversed_units,<EOL>n_hidden,<EOL>n_layers=n_layers,<EOL>trainable_initial_states=trainable_initial_states,<EOL>seq_lengths=seq_lengths,<EOL>reuse=reuse)<EOL>h_bw = tf.reverse_sequence(h_bw, seq_lengths=seq_lengths, seq_dim=<NUM_LIT:1>, batch_dim=<NUM_LIT:0>)<EOL><DEDENT><DEDENT>return (h_fw, h_bw), (h_last_fw, h_last_bw)<EOL>

Fast CuDNN Bi-GRU implementation

    Args:
        units: tf.Tensor with dimensions [B x T x F], where
            B - batch size
            T - number of tokens
            F - features
        n_hidden: dimensionality of hidden state
        seq_lengths: number of tokens in each sample in the batch
        n_layers: number of layers
        trainable_initial_states: whether to create a special trainable variable
                to initialize the hidden states of the network or use just zeros
        name: name of the variable scope to use
        reuse:whether to reuse already initialized variable

    Returns:
        h - all hidden states along T dimension,
            tf.Tensor with dimensionality [B x T x F]
        h_last - last hidden state, tf.Tensor with dimensionality [B x H * 2]
            where H - number of hidden units


with tf.variable_scope(name, reuse=reuse):<EOL><INDENT>if seq_lengths is None:<EOL><INDENT>seq_lengths = tf.ones([tf.shape(units)[<NUM_LIT:0>]], dtype=tf.int32) * tf.shape(units)[<NUM_LIT:1>]<EOL><DEDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>h_fw, (h_fw_last, c_fw_last) = cudnn_lstm_wrapper(units,<EOL>n_hidden,<EOL>n_layers=n_layers,<EOL>trainable_initial_states=trainable_initial_states,<EOL>seq_lengths=seq_lengths)<EOL><DEDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>reversed_units = tf.reverse_sequence(units, seq_lengths=seq_lengths, seq_dim=<NUM_LIT:1>, batch_dim=<NUM_LIT:0>)<EOL>h_bw, (h_bw_last, c_bw_last) = cudnn_lstm_wrapper(reversed_units,<EOL>n_hidden,<EOL>n_layers=n_layers,<EOL>trainable_initial_states=trainable_initial_states,<EOL>seq_lengths=seq_lengths)<EOL>h_bw = tf.reverse_sequence(h_bw, seq_lengths=seq_lengths, seq_dim=<NUM_LIT:1>, batch_dim=<NUM_LIT:0>)<EOL><DEDENT>return (h_fw, h_bw), ((h_fw_last, c_fw_last), (h_bw_last, c_bw_last))<EOL><DEDENT>

Fast CuDNN Bi-LSTM implementation

    Args:
        units: tf.Tensor with dimensions [B x T x F], where
            B - batch size
            T - number of tokens
            F - features
        n_hidden: dimensionality of hidden state
        seq_lengths: number of tokens in each sample in the batch
        n_layers: number of layers
        trainable_initial_states: whether to create a special trainable variable
            to initialize the hidden states of the network or use just zeros
        name: name of the variable scope to use
        reuse:whether to reuse already initialized variable

    Returns:
        h - all hidden states along T dimension,
            tf.Tensor with dimensionality [B x T x F]
        h_last - last hidden state, tf.Tensor with dimensionality [B x H * 2]
            where H - number of hidden units
        c_last - last cell state, tf.Tensor with dimensionality [B x H * 2]
            where H - number of hidden units


if seq_lengths is None:<EOL><INDENT>seq_lengths = tf.ones([tf.shape(units)[<NUM_LIT:0>]], dtype=tf.int32) * tf.shape(units)[<NUM_LIT:1>]<EOL><DEDENT>outputs = [units]<EOL>with tf.variable_scope(name, reuse=reuse):<EOL><INDENT>for n in range(n_stacks):<EOL><INDENT>if n == <NUM_LIT:0>:<EOL><INDENT>inputs = outputs[-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>inputs = variational_dropout(outputs[-<NUM_LIT:1>], keep_prob=keep_prob)<EOL><DEDENT>(h_fw, h_bw), _ = cudnn_bi_gru(inputs, n_hidden, seq_lengths,<EOL>n_layers=<NUM_LIT:1>,<EOL>trainable_initial_states=trainable_initial_states,<EOL>name='<STR_LIT>'.format(n),<EOL>reuse=reuse)<EOL>outputs.append(tf.concat([h_fw, h_bw], axis=<NUM_LIT:2>))<EOL><DEDENT><DEDENT>if concat_stacked_outputs:<EOL><INDENT>return tf.concat(outputs[<NUM_LIT:1>:], axis=<NUM_LIT:2>)<EOL><DEDENT>return outputs[-<NUM_LIT:1>]<EOL>

Fast CuDNN Stacked Bi-GRU implementation

    Args:
        units: tf.Tensor with dimensions [B x T x F], where
            B - batch size
            T - number of tokens
            F - features
        n_hidden: dimensionality of hidden state
        seq_lengths: number of tokens in each sample in the batch
        n_stacks: number of stacked Bi-GRU
        keep_prob: dropout keep_prob between Bi-GRUs (intra-layer dropout)
        concat_stacked_outputs: return last Bi-GRU output or concat outputs from every Bi-GRU,
        trainable_initial_states: whether to create a special trainable variable
                to initialize the hidden states of the network or use just zeros
        name: name of the variable scope to use
        reuse: whether to reuse already initialized variable

    Returns:
        h - all hidden states along T dimension,
            tf.Tensor with dimensionality [B x T x ((n_hidden * 2) * n_stacks)]


def variational_dropout(units, keep_prob, fixed_mask_dims=(<NUM_LIT:1>,)):

units_shape = tf.shape(units)<EOL>noise_shape = [units_shape[n] for n in range(len(units.shape))]<EOL>for dim in fixed_mask_dims:<EOL><INDENT>noise_shape[dim] = <NUM_LIT:1><EOL><DEDENT>return tf.nn.dropout(units, keep_prob, noise_shape)<EOL>

Dropout with the same drop mask for all fixed_mask_dims

    Args:
        units: a tensor, usually with shapes [B x T x F], where
            B - batch size
            T - tokens dimension
            F - feature dimension
        keep_prob: keep probability
        fixed_mask_dims: in these dimensions the mask will be the same

    Returns:
        dropped units tensor


[ten, u] = input<EOL>shape_t = ten.shape<EOL>shape_u = u.shape<EOL>ten -= tf.reduce_mean(ten)<EOL>q = tf.exp(ten)<EOL>active = tf.ones_like(u, dtype=tf.int32)<EOL>mass = tf.constant(<NUM_LIT:0>, dtype=tf.float32)<EOL>found = tf.constant(True, dtype=tf.bool)<EOL>def loop(q_, mask, mass_, found_):<EOL><INDENT>q_list = tf.dynamic_partition(q_, mask, <NUM_LIT:2>)<EOL>condition_indices = tf.dynamic_partition(tf.range(tf.shape(q_)[<NUM_LIT:0>]), mask, <NUM_LIT:2>)  <EOL>p = q_list[<NUM_LIT:1>] * (<NUM_LIT:1.0> - mass_) / tf.reduce_sum(q_list[<NUM_LIT:1>])<EOL>p_new = tf.dynamic_stitch(condition_indices, [q_list[<NUM_LIT:0>], p])<EOL>less_mask = tf.cast(tf.less(u, p_new), tf.int32)  <EOL>condition_indices = tf.dynamic_partition(tf.range(tf.shape(p_new)[<NUM_LIT:0>]), less_mask,<EOL><NUM_LIT:2>)  <EOL>split_p_new = tf.dynamic_partition(p_new, less_mask, <NUM_LIT:2>)<EOL>split_u = tf.dynamic_partition(u, less_mask, <NUM_LIT:2>)<EOL>alpha = tf.dynamic_stitch(condition_indices, [split_p_new[<NUM_LIT:0>], split_u[<NUM_LIT:1>]])<EOL>mass_ += tf.reduce_sum(split_u[<NUM_LIT:1>])<EOL>mask = mask * (tf.ones_like(less_mask) - less_mask)<EOL>found_ = tf.cond(tf.equal(tf.reduce_sum(less_mask), <NUM_LIT:0>),<EOL>lambda: False,<EOL>lambda: True)<EOL>alpha = tf.reshape(alpha, q_.shape)<EOL>return alpha, mask, mass_, found_<EOL><DEDENT>(csoft, mask_, _, _) = tf.while_loop(cond=lambda _0, _1, _2, f: f,<EOL>body=loop,<EOL>loop_vars=(q, active, mass, found))<EOL>return [csoft, mask_]<EOL>

It is a implementation of the constrained softmax (csoftmax) for slice.
        Based on the paper:
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers" (page 4)
    Args:
        input: A list of [input tensor, cumulative attention].
    Returns:
        output: A list of [csoftmax results, masks]


shape_ten = tensor.shape<EOL>shape_cum = inv_cumulative_att.shape<EOL>merge_tensor = [tensor, inv_cumulative_att]<EOL>cs, _ = tf.map_fn(csoftmax_for_slice, merge_tensor, dtype=[tf.float32, tf.float32])  <EOL>return cs<EOL>

It is a implementation of the constrained softmax (csoftmax).
        Based on the paper:
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        tensor: A tensorflow tensor is score. This tensor have dimensionality [None, n_tokens]
        inv_cumulative_att: A inverse cumulative attention tensor with dimensionality [None, n_tokens]
    Returns:
        cs: Tensor at the output with dimensionality [None, n_tokens]


def attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketch, key, cum_att):

with tf.name_scope('<STR_LIT>'):<EOL><INDENT>sketch_dims = hidden_for_sketch.get_shape().as_list()<EOL>batch_size = sketch_dims[<NUM_LIT:0>]<EOL>num_tokens = sketch_dims[<NUM_LIT:1>]<EOL>hidden_size = sketch_dims[<NUM_LIT:2>]<EOL>attn_alignment_dims = hidden_for_attn_alignment.get_shape().as_list()<EOL>attn_alignment_hidden_size = attn_alignment_dims[<NUM_LIT:2>]<EOL>repeated_sketch = tf.tile(tf.reshape(sketch, [-<NUM_LIT:1>, <NUM_LIT:1>, hidden_size]), (<NUM_LIT:1>,num_tokens, <NUM_LIT:1>))<EOL>concat_mem = tf.concat([hidden_for_sketch, repeated_sketch],-<NUM_LIT:1>)<EOL>concat_mem = tf.reshape(concat_mem, [-<NUM_LIT:1>, num_tokens, <NUM_LIT:2>*hidden_size]) <EOL>reduce_mem = tf.layers.dense(concat_mem, hidden_size)<EOL>projected_key = tf.layers.dense(key, hidden_size)<EOL>t_key = tf.reshape(projected_key,[-<NUM_LIT:1>, hidden_size, <NUM_LIT:1>])<EOL>score = tf.reshape(tf.matmul(reduce_mem, t_key), [-<NUM_LIT:1>, num_tokens])<EOL>inv_cum_att = tf.reshape(tf.ones_like(cum_att) - cum_att, [-<NUM_LIT:1>, num_tokens])<EOL>att = csoftmax(score, inv_cum_att)<EOL>t_reduce_mem = tf.transpose(reduce_mem, [<NUM_LIT:0>,<NUM_LIT:2>,<NUM_LIT:1>])<EOL>t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [<NUM_LIT:0>,<NUM_LIT:2>,<NUM_LIT:1>])<EOL>r_att = tf.reshape(att, [-<NUM_LIT:1>, num_tokens, <NUM_LIT:1>])<EOL>next_sketch = tf.squeeze(tf.matmul(t_reduce_mem,r_att),-<NUM_LIT:1>)<EOL>aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment,r_att),-<NUM_LIT:1>)<EOL><DEDENT>return next_sketch, att, aligned_hidden_sketch<EOL>

It is a implementation one step of block of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        sketch: A previous step sketch tensor for a sketch computing. This tensor have dimensionality [None, sketch_hidden_size]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        cum_att: A cumulative attention tensor with dimensionality [None, max_num_tokens]
    Returns:
        next_sketch: Tensor of the current step sketch with dimensionality [None, sketch_hidden_size]
        att: Tensor of the current step attention with dimensionality [None, max_num_tokens]
        aligned_hidden_sketch: Tensor of aligned hidden state of current step with dimensionality [None, hidden_size_for_attn_alignment]


def attention_gen_block(hidden_for_sketch, hidden_for_attn_alignment, key, attention_depth):

with tf.name_scope('<STR_LIT>'):<EOL><INDENT>sketch_dims = tf.shape(hidden_for_sketch)<EOL>batch_size = sketch_dims[<NUM_LIT:0>]<EOL>num_tokens = sketch_dims[<NUM_LIT:1>]<EOL>hidden_size = sketch_dims[<NUM_LIT:2>]<EOL>attn_alignment_dims = tf.shape(hidden_for_attn_alignment)<EOL>attn_alignment_hidden_size = attn_alignment_dims[<NUM_LIT:2>]<EOL>sketches = [tf.zeros(shape=[batch_size, hidden_size], dtype=tf.float32)]<EOL>aligned_hiddens = []<EOL>cum_att = tf.zeros(shape=[batch_size, num_tokens])  <EOL>for i in range(attention_depth):<EOL><INDENT>sketch, cum_att_, aligned_hidden = attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketches[-<NUM_LIT:1>], key, cum_att)<EOL>sketches.append(sketch) <EOL>aligned_hiddens.append(aligned_hidden) <EOL>cum_att += cum_att_<EOL><DEDENT>final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>]),[<NUM_LIT:1>, attention_depth, attn_alignment_hidden_size])<EOL><DEDENT>return final_aligned_hiddens<EOL>

It is a implementation of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        attention_depth: Number of usage csoftmax
    Returns:
        final_aligned_hiddens: Tensor at the output with dimensionality [1, attention_depth, hidden_size_for_attn_alignment]