def mutation_of_param(self, param_path: list,<EOL>param_value: [int, float, str, list, dict, bool, np.ndarray]) -> Any:

if self.decision(self.p_mutation):<EOL><INDENT>param_name = param_path[-<NUM_LIT:1>]<EOL>basic_value = self.get_value_from_config(self.basic_config, param_path)<EOL>if isinstance(basic_value, dict):<EOL><INDENT>if basic_value.get('<STR_LIT>', False):<EOL><INDENT>val = round(param_value +<EOL>((<NUM_LIT:2> * np.random.random() - <NUM_LIT:1.>) * self.mutation_power<EOL>* self.sample_params(**{param_name: basic_value})[param_name]))<EOL>val = min(max(basic_value["<STR_LIT>"][<NUM_LIT:0>], val),<EOL>basic_value["<STR_LIT>"][<NUM_LIT:1>])<EOL>new_mutated_value = val<EOL><DEDENT>elif '<STR_LIT>' in basic_value.keys():<EOL><INDENT>val = param_value +((<NUM_LIT:2> * np.random.random() - <NUM_LIT:1.>) * self.mutation_power<EOL>* self.sample_params(**{param_name: basic_value})[param_name])<EOL>val = min(max(basic_value["<STR_LIT>"][<NUM_LIT:0>], val),<EOL>basic_value["<STR_LIT>"][<NUM_LIT:1>])<EOL>new_mutated_value = val<EOL><DEDENT>elif basic_value.get("<STR_LIT>"):<EOL><INDENT>new_mutated_value = self.sample_params(**{param_name: basic_value})[param_name]<EOL><DEDENT>elif basic_value.get("<STR_LIT>"):<EOL><INDENT>new_mutated_value = self.sample_params(**{param_name: basic_value})[param_name]<EOL><DEDENT>else:<EOL><INDENT>new_mutated_value = param_value<EOL><DEDENT><DEDENT>else:<EOL><INDENT>new_mutated_value = param_value<EOL><DEDENT><DEDENT>else:<EOL><INDENT>new_mutated_value = param_value<EOL><DEDENT>return new_mutated_value<EOL>

Mutate particular parameter separately

Args:
    param_path: path to parameter in basic config
    param_value: current parameter valuer

Returns:
    mutated parameter value


@staticmethod<EOL><INDENT>def decision(probability: float = <NUM_LIT:1.>) -> bool:<DEDENT>

r = np.random.random()<EOL>if r < probability:<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Make decision whether to do action or not with given probability

Args:
    probability: probability whether to do action or not

Returns:
    bool decision


def __init__(self, load_path: str, top_k_classes: int, linker: EntityLinker, classes_vocab_keys: Tuple,<EOL>debug: bool = False, relations_maping_filename: str = None, templates_filename: str = None,<EOL>return_confidences: bool = True, *args, **kwargs) -> None:

super().__init__(save_path=None, load_path=load_path)<EOL>self.top_k_classes = top_k_classes<EOL>self.classes = list(classes_vocab_keys)<EOL>self._debug = debug<EOL>self._relations_filename = relations_maping_filename<EOL>self._templates_filename = templates_filename<EOL>self._q_to_name: Optional[Dict[str, Dict[str, str]]] = None<EOL>self._relations_mapping: Optional[Dict[str, str]] = None<EOL>self.templates: Optional[Dict[str, str]] = None<EOL>self.return_confidences = return_confidences<EOL>self.linker = linker<EOL>self.load()<EOL>

Args:
    load_path: path to folder with wikidata files
    top_k_classes: number of relations with top k probabilities
    linker: component `deeppavlov.models.kbqa.entity_linking`
    classes_vocab_keys: list of relations predicted by `deeppavlov.models.ner.network` model
    debug: whether to print entities and relations extracted from the question
    relations_maping_filename: file with the dictionary of ids(keys) and titles(values) of relations
    from Wikidata
    templates_filename: file with the dictionary of question templates(keys) and relations for these templates
    (values)
    return_confidences: whether to return confidences of answers
    *args:
    **kwargs:


def __init__(self, load_path: str, wiki_filename: str, entities_filename: str, inverted_index_filename: str,<EOL>id_to_name_file: str, lemmatize: bool = True, debug: bool = False, rule_filter_entities: bool = True,<EOL>use_inverted_index: bool = True, language: str = '<STR_LIT>', *args, **kwargs) -> None:

super().__init__(save_path=None, load_path=load_path)<EOL>self.morph = pymorphy2.MorphAnalyzer()<EOL>self.lemmatize = lemmatize<EOL>self.debug = debug<EOL>self.rule_filter_entities = rule_filter_entities<EOL>self.use_inverted_index = use_inverted_index<EOL>self._language = language<EOL>if language not in self.LANGUAGES:<EOL><INDENT>log.warning(f'<STR_LIT>')<EOL><DEDENT>self._wiki_filename = wiki_filename<EOL>self._entities_filename = entities_filename<EOL>self.inverted_index_filename = inverted_index_filename<EOL>self.id_to_name_file = id_to_name_file<EOL>self.name_to_q: Optional[Dict[str, List[Tuple[str]]]] = None<EOL>self.wikidata: Optional[Dict[str, List[List[str]]]] = None<EOL>self.inverted_index: Optional[Dict[str, List[Tuple[str]]]] = None<EOL>self.id_to_name: Optional[Dict[str, Dict[List[str]]]] = None<EOL>self.load()<EOL>if self.use_inverted_index:<EOL><INDENT>alphabet = "<STR_LIT>"<EOL>dictionary_words = list(self.inverted_index.keys())<EOL>self.searcher = LevenshteinSearcher(alphabet, dictionary_words)<EOL><DEDENT>

Args:
    load_path: path to folder with wikidata files
    wiki_filename: file with Wikidata triplets
    entities_filename: file with dict of entity titles (keys) and entity ids (values)
    inverted_index_filename: file with dict of words (keys) and entities containing these words (values)
    id_to_name_file: file with dict of entity ids (keys) and entities names and aliases (values)
    lemmatize: whether to lemmatize tokens of extracted entity
    debug: whether to print entities extracted from Wikidata
    rule_filter_entities: whether to filter entities which do not fit the question
    use_inverted_index: whether to use inverted index for entity linking
    language - the language of the linker (used for filtration of some questions to improve overall performance)
    *args:
    **kwargs:


def __call__(self, batch: List[List[List[Tuple[float, str]]]]) -> List[List[str]]:

return [[max(sublist)[<NUM_LIT:1>] for sublist in candidates] for candidates in batch]<EOL>

Choose the best candidate for every token

        Args:
            batch: batch of probabilities and string values of candidates for every token in a sentence

        Returns:
            batch of corrected tokenized sentences


return [self._infer_instance(candidates) for candidates in batch]<EOL>


def __call__(self, data: Iterable[Iterable[str]], *args, **kwargs) -> List[List[List[Tuple[float, str]]]]:

data = list(data)<EOL>if len(data) > <NUM_LIT:1>:<EOL><INDENT>data = tqdm(data, desc='<STR_LIT>', leave=False)<EOL><DEDENT>return [self._infer_instance(instance) for instance in data]<EOL>

Propose candidates for tokens in sentences

        Args:
            data: batch of tokenized sentences

        Returns:
            batch of lists of probabilities and candidates for every token


def fit(self, x: List[str], y: List[str]):

changes = []<EOL>entries = []<EOL>data = list(zip(x, y))<EOL>window = <NUM_LIT:4><EOL>for error, correct in tqdm(data, desc='<STR_LIT>'):<EOL><INDENT>correct = '<STR_LIT>'.format('<STR_LIT:U+0020>'.join(correct))<EOL>error = '<STR_LIT>'.format('<STR_LIT:U+0020>'.join(error))<EOL>d, ops = self._distance_edits(correct, error)<EOL>if d <= <NUM_LIT:2>:<EOL><INDENT>w_ops = set()<EOL>for pos in range(len(ops)):<EOL><INDENT>left, right = list(zip(*ops))<EOL>for l in range(pos, max(<NUM_LIT:0>, pos - window) - <NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>for r in range(pos + <NUM_LIT:1>, min(len(ops), l + <NUM_LIT:2> + window)):<EOL><INDENT>w_ops.add((('<STR_LIT>'.join(left[l:r]), '<STR_LIT>'.join(right[l:r])), l, r))<EOL><DEDENT><DEDENT><DEDENT>ops = [x[<NUM_LIT:0>] for x in w_ops]<EOL>entries += [op[<NUM_LIT:0>] for op in ops]<EOL>changes += [op for op in ops]<EOL><DEDENT><DEDENT>e_count = Counter(entries)<EOL>c_count = Counter(changes)<EOL>incorrect_prior = <NUM_LIT:1><EOL>correct_prior = <NUM_LIT><EOL>for (w, s), c in c_count.items():<EOL><INDENT>c = c + (incorrect_prior if w != s else correct_prior)<EOL>e = e_count[w] + incorrect_prior + correct_prior<EOL>p = c / e<EOL>self.costs[(w, s)] = log(p)<EOL><DEDENT>

Calculate character sequences replacements probabilities

        Args:
            x: words with spelling errors
            y: words without spelling errors


logger.info("<STR_LIT>".format(self.save_path))<EOL>with open(self.save_path, '<STR_LIT:w>', newline='<STR_LIT>', encoding='<STR_LIT:utf8>') as tsv_file:<EOL><INDENT>writer = csv.writer(tsv_file, delimiter='<STR_LIT:\t>')<EOL>for (w, s), log_p in self.costs.items():<EOL><INDENT>writer.writerow([w, s, exp(log_p)])<EOL><DEDENT><DEDENT>

Save replacements probabilities to a file


if self.load_path:<EOL><INDENT>if self.load_path.is_file():<EOL><INDENT>logger.info("<STR_LIT>".format(self.load_path))<EOL>with open(self.load_path, '<STR_LIT:r>', newline='<STR_LIT>', encoding='<STR_LIT:utf8>') as tsv_file:<EOL><INDENT>reader = csv.reader(tsv_file, delimiter='<STR_LIT:\t>')<EOL>for w, s, p in reader:<EOL><INDENT>self.costs[(w, s)] = log(float(p))<EOL><DEDENT><DEDENT><DEDENT>elif not self.load_path.parent.is_dir():<EOL><INDENT>raise ConfigError("<STR_LIT>".format(<EOL>self.__class__.__name__))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logger.info('<STR_LIT>')<EOL><DEDENT>

Load replacements probabilities from a file


def _precompute_absense_costs(dictionary, removal_costs, insertion_costs, n,<EOL>allow_spaces=False):

answer = [dict() for node in dictionary.data]<EOL>if n == <NUM_LIT:0>:<EOL><INDENT>return answer<EOL><DEDENT>curr_alphabet = copy.copy(dictionary.alphabet)<EOL>if allow_spaces:<EOL><INDENT>curr_alphabet += ['<STR_LIT:U+0020>']<EOL><DEDENT>for l, (costs_in_node, node) in enumerate(zip(answer, dictionary.data)):<EOL><INDENT>curr_node_removal_costs = np.empty(dtype=np.float64, shape=(n,))<EOL>if len(node[<NUM_LIT:0>]) > <NUM_LIT:0>:<EOL><INDENT>curr_node_removal_costs[<NUM_LIT:0>] = min(removal_costs[symbol] for symbol in node[<NUM_LIT:0>])<EOL>for j, symbols in enumerate(node[<NUM_LIT:1>:], <NUM_LIT:1>):<EOL><INDENT>if len(symbols) == <NUM_LIT:0>:<EOL><INDENT>curr_node_removal_costs[j:] = curr_node_removal_costs[j-<NUM_LIT:1>]<EOL>break<EOL><DEDENT>curr_cost = min(removal_costs[symbol] for symbol in symbols)<EOL>curr_node_removal_costs[j] = min(curr_node_removal_costs[j-<NUM_LIT:1>], curr_cost)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>curr_node_removal_costs[:] = np.inf<EOL><DEDENT>for a in curr_alphabet:<EOL><INDENT>curr_symbol_costs = np.empty(dtype=np.float64, shape=(n,))<EOL>curr_symbol_costs.fill(insertion_costs[a])<EOL>for j, symbols in enumerate(node):<EOL><INDENT>if a in symbols:<EOL><INDENT>curr_symbol_costs[j:] = <NUM_LIT:0.0><EOL>break<EOL><DEDENT>curr_symbol_costs[j] = min(curr_symbol_costs[j], curr_node_removal_costs[j])<EOL><DEDENT>costs_in_node[a] = curr_symbol_costs<EOL><DEDENT><DEDENT>return answer<EOL>

Вычисляет минимальную стоимость появления нового символа в узлах словаря
в соответствии со штрафами из costs

Аргументы:
---------------
dictionary : Trie
    словарь, хранящийся в виде ациклического автомата

removal_costs : dict
    штрафы за удаление символов

insertion_costs : dict
    штрафы за вставку символов

n : int
    глубина ``заглядывания вперёд'' в словаре

Возвращает
---------------
answer : list of dicts, len(answer)=len(dictionary)
    answer[i][a][j] равно минимальному штрафу за появление символа a
    в j-ой позиции в вершине с номером i


def search(self, word, d, allow_spaces=True, return_cost=True):

if not all((c in self.alphabet<EOL>or (c == "<STR_LIT:U+0020>" and self.allow_spaces)) for c in word):<EOL><INDENT>return []<EOL><DEDENT>return self._trie_search(<EOL>word, d, allow_spaces=allow_spaces, return_cost=return_cost)<EOL>

Finds all dictionary words in d-window from word


def _trie_search(self, word, d, transducer=None,<EOL>allow_spaces=True, return_cost=True):

if transducer is None:<EOL><INDENT>transducer = self.transducer.inverse()<EOL><DEDENT>allow_spaces &= self.allow_spaces<EOL>trie = self.dictionary<EOL>used_agenda_keys = set()<EOL>agenda = SortedListWithKey(key=(lambda x:x[<NUM_LIT:1>]))<EOL>h = self.h_func(word, trie.root)<EOL>key, value = ("<STR_LIT>", <NUM_LIT:0>, trie.root), (<NUM_LIT:0.0>, <NUM_LIT:0.0>, h)<EOL>agenda.add((key, value))<EOL>answer = dict()<EOL>k = <NUM_LIT:0><EOL>while len(agenda) > <NUM_LIT:0>:<EOL><INDENT>key, value = agenda.pop(<NUM_LIT:0>)<EOL>if key in used_agenda_keys:<EOL><INDENT>continue<EOL><DEDENT>used_agenda_keys.add(key)<EOL>low, pos, index = key<EOL>cost, g, h = value<EOL>k += <NUM_LIT:1><EOL>max_upperside_length = min(len(word) - pos, transducer.max_up_length)<EOL>for upperside_length in range(max_upperside_length + <NUM_LIT:1>):<EOL><INDENT>new_pos = pos + upperside_length<EOL>curr_up = word[pos: new_pos]<EOL>if curr_up not in transducer.operation_costs:<EOL><INDENT>continue<EOL><DEDENT>for curr_low, curr_cost in transducer.operation_costs[curr_up].items():<EOL><INDENT>new_g = g + curr_cost<EOL>if new_g > d:  <EOL><INDENT>continue<EOL><DEDENT>if curr_low == "<STR_LIT:U+0020>":<EOL><INDENT>if allow_spaces and trie.is_final(index):<EOL><INDENT>new_index = trie.root<EOL><DEDENT>else:<EOL><INDENT>new_index = Trie.NO_NODE<EOL><DEDENT><DEDENT>else:<EOL><INDENT>new_index = trie.descend(index, curr_low)<EOL><DEDENT>if new_index is Trie.NO_NODE:<EOL><INDENT>continue<EOL><DEDENT>new_low = low + curr_low<EOL>new_h = self.h_func(word[new_pos: ], new_index)<EOL>new_cost = new_g + new_h<EOL>if new_cost > d:<EOL><INDENT>continue<EOL><DEDENT>new_key = (new_low, new_pos, new_index)<EOL>new_value = (new_cost, new_g, new_h)<EOL>if new_pos == len(word) and trie.is_final(new_index):<EOL><INDENT>old_g = answer.get(new_low, None)<EOL>if old_g is None or new_g < old_g:<EOL><INDENT>answer[new_low] = new_g<EOL><DEDENT><DEDENT>agenda.add((new_key, new_value))<EOL><DEDENT><DEDENT><DEDENT>answer = sorted(answer.items(), key=(lambda x: x[<NUM_LIT:1>]))<EOL>if return_cost:<EOL><INDENT>return answer<EOL><DEDENT>else:<EOL><INDENT>return [elem[<NUM_LIT:0>] for elem in answer]<EOL><DEDENT>

Находит все слова в префиксном боре, расстояние до которых
в соответствии с заданным преобразователем не превышает d


if self.euristics is None:<EOL><INDENT>return<EOL><DEDENT>removal_costs = {a : np.inf for a in self.alphabet}<EOL>insertion_costs = {a : np.inf for a in self.alphabet}<EOL>if self.allow_spaces:<EOL><INDENT>removal_costs['<STR_LIT:U+0020>'] = np.inf<EOL>insertion_costs['<STR_LIT:U+0020>'] = np.inf<EOL><DEDENT>for up, costs in self.transducer.operation_costs.items():<EOL><INDENT>for low, cost in costs.items():<EOL><INDENT>if up == low:<EOL><INDENT>continue<EOL><DEDENT>if up != '<STR_LIT>':<EOL><INDENT>removal_cost = cost / len(up)<EOL>for a in up:<EOL><INDENT>removal_costs[a] = min(removal_costs[a], removal_cost)<EOL><DEDENT><DEDENT>if low != '<STR_LIT>':<EOL><INDENT>insertion_cost = cost / len(low)<EOL>for a in low:<EOL><INDENT>insertion_costs[a] = min(insertion_costs[a], insertion_cost)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self._absense_costs_by_node = _precompute_absense_costs(<EOL>self.dictionary, removal_costs, insertion_costs,<EOL>self.euristics, self.allow_spaces)<EOL>self._temporary_euristics = [dict() for i in range(len(self.dictionary))]<EOL>

Предвычисляет будущие символы и стоимости операций с ними
для h-эвристики


def _euristic_h_function(self, suffix, index):

if self.euristics > <NUM_LIT:0>:<EOL><INDENT>suffix = suffix[:self.euristics]<EOL><DEDENT>index_temporary_euristics = self._temporary_euristics[index]<EOL>cost = index_temporary_euristics.get(suffix, None)<EOL>if cost is not None:<EOL><INDENT>return cost<EOL><DEDENT>absense_costs = self._absense_costs_by_node[index]<EOL>data = self.dictionary.data[index]<EOL>costs = np.zeros(dtype=np.float64, shape=(self.euristics,))<EOL>for i, a in enumerate(suffix):<EOL><INDENT>costs[i:] += absense_costs[a][i:]<EOL><DEDENT>cost = max(costs)<EOL>index_temporary_euristics[suffix] = cost<EOL>return cost<EOL>

Вычисление h-эвристики из работы Hulden,2009 для текущей вершины словаря

Аргументы:
----------
suffix : string
    непрочитанный суффикс входного слова
index : int
    индекс текущего узла в словаре

Возвращает:
-----------
cost : float
    оценка снизу для стоимости замены,
    приводящей к входному слову с суффиксом suffix,
    если прочитанный префикс слова без опечатки
    привёл в вершину с номером index


up_costs = self.operation_costs.get(up, None)<EOL>if up_costs is None:<EOL><INDENT>return np.inf<EOL><DEDENT>cost = up_costs.get(low, np.inf)<EOL>return cost<EOL>

Возвращает стоимость элементарной трансдукции up->low
или np.inf, если такой элементарной трансдукции нет

Аргументы:
----------
up, low : string
    элементы элементарной трансдукции

Возвращает:
-----------
cost : float
    стоимость элементарной трансдукции up->low
    (np.inf, если такая трансдукция отсутствует)


<EOL>inversed_transducer = SegmentTransducer(self.alphabet, operation_costs=dict())<EOL>inversed_transducer.operation_costs = self._reversed_operation_costs<EOL>inversed_transducer._reversed_operation_costs = self.operation_costs<EOL>inversed_transducer.max_low_length = self.max_up_length<EOL>inversed_transducer.max_up_length = self.max_low_length<EOL>inversed_transducer.max_low_lengths_by_up = self.max_up_lengths_by_low<EOL>inversed_transducer.max_up_lengths_by_low = self.max_low_lengths_by_up<EOL>return inversed_transducer<EOL>

Строит пробразователь, задающий обратное конечное преобразование


def distance(self, first, second, return_transduction = False):

if return_transduction:<EOL><INDENT>add_pred = (lambda x, y: (y == np.inf or x < y))<EOL><DEDENT>else:<EOL><INDENT>add_pred = (lambda x, y: (y == np.inf or x <= y))<EOL><DEDENT>clear_pred = (lambda x, y: (y < np.inf and x < y))<EOL>update_func = lambda x, y: min(x, y)<EOL>costs, backtraces = self._fill_levenshtein_table(first, second,<EOL>update_func, add_pred, clear_pred)<EOL>final_cost = costs[-<NUM_LIT:1>][-<NUM_LIT:1>]<EOL>if final_cost == np.inf:<EOL><INDENT>transductions = [None]<EOL><DEDENT>elif return_transduction:<EOL><INDENT>transductions = self._backtraces_to_transductions(first, second, backtraces,<EOL>final_cost, return_cost=False)<EOL><DEDENT>if return_transduction:<EOL><INDENT>return final_cost, transductions<EOL><DEDENT>else:<EOL><INDENT>return final_cost<EOL><DEDENT>

Вычисляет трансдукцию минимальной стоимости,
отображающую first в second

Аргументы:
-----------
first : string
second : string
    Верхний и нижний элементы трансдукции

return_transduction : bool (optional, default=False)
    следует ли возвращать трансдукцию минимального веса
    (см. возвращаемое значение)

Возвращает:
-----------
(final_cost, transductions) : tuple(float, list)
    если return_transduction=True, то возвращает
    минимальную стоимость трансдукции, переводящей first в second
    и список трансдукций с данной стоимостью

final_cost : float
    если return_transduction=False, то возвращает
    минимальную стоимость трансдукции, переводящей first в second


def transduce(self, first, second, threshold):

add_pred = (lambda x, y: x <= threshold)<EOL>clear_pred =(lambda x, y: False)<EOL>update_func = (lambda x, y: min(x, y))<EOL>costs, backtraces = self._fill_levenshtein_table(first, second,<EOL>update_func, add_pred, clear_pred,<EOL>threshold=threshold)<EOL>result = self._backtraces_to_transductions(first, second,<EOL>backtraces, threshold, return_cost=True)<EOL>return result<EOL>

Возвращает все трансдукции, переводящие first в second,
чья стоимость не превышает threshold

Возвращает:
----------
result : list
    список вида [(трансдукция, стоимость)]


def lower_transductions(self, word, max_cost, return_cost=True):

prefixes = [[] for i in range(len(word) + <NUM_LIT:1>)]<EOL>prefixes[<NUM_LIT:0>].append(((), <NUM_LIT:0.0>))<EOL>for pos in range(len(prefixes)):<EOL><INDENT>prefixes[pos] = self._perform_insertions(prefixes[pos], max_cost)<EOL>max_upperside_length = min(len(word) - pos, self.max_up_length)<EOL>for upperside_length in range(<NUM_LIT:1>, max_upperside_length + <NUM_LIT:1>):<EOL><INDENT>up = word[pos: pos + upperside_length]<EOL>for low, low_cost in self.operation_costs.get(up, dict()).items():<EOL><INDENT>for transduction, cost in prefixes[pos]:<EOL><INDENT>new_cost = cost + low_cost<EOL>if new_cost <= max_cost:<EOL><INDENT>new_transduction = transduction +(up, low)<EOL>prefixes[pos + upperside_length].append((new_transduction, new_cost))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>answer = sorted(prefixes[-<NUM_LIT:1>], key=(lambda x: x[<NUM_LIT:0>]))<EOL>if return_cost:<EOL><INDENT>return answer<EOL><DEDENT>else:<EOL><INDENT>return [elem[<NUM_LIT:0>] for elem in answer]<EOL><DEDENT>

Возвращает все трансдукции с верхним элементом word,
    чья стоимость не превышает max_cost

`   Возвращает:
    ----------
    result : list
        список вида [(трансдукция, стоимость)], если return_cost=True
        список трансдукций, если return_cost=False
        список отсортирован в порядке возрастания стоимости трансдукции


def _fill_levenshtein_table(self, first, second, update_func, add_pred, clear_pred,<EOL>threshold=None):

m, n = len(first), len(second)<EOL>if threshold is None:<EOL><INDENT>threshold = <NUM_LIT:0.0><EOL>for a, b in zip(first, second):<EOL><INDENT>threshold += self.get_operation_cost(a, b)<EOL><DEDENT>if m > n:<EOL><INDENT>for a in first[n: ]:<EOL><INDENT>threshold += self.get_operation_cost(a, '<STR_LIT>')<EOL><DEDENT><DEDENT>elif m < n:<EOL><INDENT>for b in second[m: ]:<EOL><INDENT>threshold += self.get_operation_cost('<STR_LIT>', b)<EOL><DEDENT><DEDENT>threshold *= <NUM_LIT:2><EOL><DEDENT>costs = np.zeros(shape=(m + <NUM_LIT:1>, n + <NUM_LIT:1>), dtype=np.float64)<EOL>costs[:] = np.inf<EOL>backtraces = [None] * (m + <NUM_LIT:1>)<EOL>for i in range(m + <NUM_LIT:1>):<EOL><INDENT>backtraces[i] = [[] for j in range(n + <NUM_LIT:1>)]<EOL><DEDENT>costs[<NUM_LIT:0>][<NUM_LIT:0>] = <NUM_LIT:0.0><EOL>for i in range(m + <NUM_LIT:1>):<EOL><INDENT>for i_right in range(i, min(i + self.max_up_length, m) + <NUM_LIT:1>):<EOL><INDENT>up = first[i: i_right]<EOL>max_low_length = self.max_low_lengths_by_up.get(up, -<NUM_LIT:1>)<EOL>if max_low_length == -<NUM_LIT:1>: <EOL><INDENT>continue<EOL><DEDENT>up_costs = self.operation_costs[up]<EOL>for j in range(n + <NUM_LIT:1>):<EOL><INDENT>if costs[i][j] > threshold:<EOL><INDENT>continue<EOL><DEDENT>if len(backtraces[i][j]) == <NUM_LIT:0> and i + j > <NUM_LIT:0>:<EOL><INDENT>continue <EOL><DEDENT>for j_right in range((j if i_right > i else j + <NUM_LIT:1>),<EOL>min(j + max_low_length, n) + <NUM_LIT:1>):<EOL><INDENT>low = second[j: j_right]<EOL>curr_cost = up_costs.get(low, np.inf)<EOL>old_cost = costs[i_right][j_right]<EOL>new_cost = costs[i][j] + curr_cost<EOL>if new_cost > threshold:<EOL><INDENT>continue<EOL><DEDENT>if add_pred(new_cost, old_cost):<EOL><INDENT>if clear_pred(new_cost, old_cost):<EOL><INDENT>backtraces[i_right][j_right] = []<EOL><DEDENT>costs[i_right][j_right] = update_func(new_cost, old_cost)<EOL>backtraces[i_right][j_right].append((i, j))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return costs, backtraces<EOL>

Функция, динамически заполняющая таблицу costs стоимости трансдукций,
costs[i][j] --- минимальная стоимость трансдукции,
переводящей first[:i] в second[:j]

Аргументы:
----------
first, second : string
    Верхний и нижний элементы трансдукции
update_func : callable, float*float -> bool
    update_func(x, y) возвращает новое значение в ячейке таблицы costs,
    если старое значение --- y, а потенциально новое значение --- x
    везде update_func = min
add_pred : callable : float*float -> bool
    add_pred(x, y) возвращает, производится ли добавление
    нового элемента p стоимости x в ячейку backtraces[i][j]
    в зависимости от значения costs[i][j]=y и текущей стоимости x
clear_pred : callable : float*float -> bool
    clear_pred(x, y) возвращает, производится ли очистка
    ячейки backtraces[i][j] в зависимости от значения costs[i][j]=y
    и текущей стоимости x элемента p, добавляемого в эту ячейку

Возвращает:
-----------
costs : array, dtype=float, shape=(len(first)+1, len(second)+1)
    массив, в ячейке с индексами i, j которого хранится
    минимальная стоимость трансдукции, переводящей first[:i] в second[:j]
backtraces : array, dtype=list, shape=(len(first)+1, len(second)+1)
    массив, в ячейке с индексами i, j которого хранятся
    обратные ссылки на предыдущую ячейку в оптимальной трансдукции,
    приводящей в ячейку backtraces[i][j]


_reversed_operation_costs = dict()<EOL>for up, costs in self.operation_costs.items():<EOL><INDENT>for low, cost in costs.items():<EOL><INDENT>if low not in _reversed_operation_costs:<EOL><INDENT>_reversed_operation_costs[low] = dict()<EOL><DEDENT>_reversed_operation_costs[low][up] = cost<EOL><DEDENT><DEDENT>self._reversed_operation_costs = _reversed_operation_costs<EOL>

Заполняет массив _reversed_operation_costs
на основе имеющегося массива operation_costs


self.max_up_length =(max(len(up) for up in self.operation_costs)<EOL>if len(self.operation_costs) > <NUM_LIT:0> else -<NUM_LIT:1>)<EOL>self.max_low_length =(max(len(low) for low in self._reversed_operation_costs)<EOL>if len(self._reversed_operation_costs) > <NUM_LIT:0> else -<NUM_LIT:1>)<EOL>self.max_low_lengths_by_up, self.max_up_lengths_by_low = dict(), dict()<EOL>for up, costs in self.operation_costs.items():<EOL><INDENT>self.max_low_lengths_by_up[up] =max(len(low) for low in costs) if len(costs) > <NUM_LIT:0> else -<NUM_LIT:1><EOL><DEDENT>for low, costs in self._reversed_operation_costs.items():<EOL><INDENT>self.max_up_lengths_by_low[low] =max(len(up) for up in costs) if len(costs) > <NUM_LIT:0> else -<NUM_LIT:1><EOL><DEDENT>

Вычисляет максимальную длину элемента low
в элементарной трансдукции (up, low) для каждого up
и максимальную длину элемента up
в элементарной трансдукции (up, low) для каждого low


def _backtraces_to_transductions(self, first, second, backtraces, threshold, return_cost=False):

m, n = len(first), len(second)<EOL>agenda = [None] * (m + <NUM_LIT:1>)<EOL>for i in range(m + <NUM_LIT:1>):<EOL><INDENT>agenda[i] = [[] for j in range(n+<NUM_LIT:1>)]<EOL><DEDENT>agenda[m][n] = [((), <NUM_LIT:0.0>)]<EOL>for i_right in range(m, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>for j_right in range(n, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>current_agenda = agenda[i_right][j_right]<EOL>if len(current_agenda) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>for (i, j) in backtraces[i_right][j_right]:<EOL><INDENT>up, low = first[i:i_right], second[j:j_right]<EOL>add_cost = self.operation_costs[up][low]<EOL>for elem, cost in current_agenda:<EOL><INDENT>new_cost = cost + add_cost<EOL>if new_cost <= threshold: <EOL><INDENT>agenda[i][j].append((((up, low),) + elem, new_cost))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if return_cost:<EOL><INDENT>return agenda[<NUM_LIT:0>][<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return [elem[<NUM_LIT:0>] for elem in agenda[<NUM_LIT:0>][<NUM_LIT:0>]]<EOL><DEDENT>

Восстанавливает трансдукции по таблице обратных ссылок

Аргументы:
----------
first, second : string
    верхние и нижние элементы трансдукции
backtraces : array-like, dtype=list, shape=(len(first)+1, len(second)+1)
    таблица обратных ссылок
threshold : float
    порог для отсева трансдукций,
    возвращаются только трансдукции стоимостью <= threshold
return_cost : bool (optional, default=False)
    если True, то вместе с трансдукциями возвращается их стоимость

Возвращает:
-----------
result : list
    список вида [(трансдукция, стоимость)], если return_cost=True
    и вида [трансдукция], если return_cost=False,
    содержащий все трансдукции, переводящие first в second,
    чья стоимость не превышает threshold


def _perform_insertions(self, initial, max_cost):

queue = list(initial)<EOL>final = initial<EOL>while len(queue) > <NUM_LIT:0>:<EOL><INDENT>transduction, cost = queue[<NUM_LIT:0>]<EOL>queue = queue[<NUM_LIT:1>:]<EOL>for string, string_cost in self.operation_costs["<STR_LIT>"].items():<EOL><INDENT>new_cost = cost + string_cost<EOL>if new_cost <= max_cost:<EOL><INDENT>new_transduction = transduction + ("<STR_LIT>", string)<EOL>final.append((new_transduction, new_cost))<EOL>queue.append((new_transduction, new_cost))<EOL><DEDENT><DEDENT><DEDENT>return final<EOL>

возвращает все трансдукции стоимости <= max_cost,
которые можно получить из элементов initial

Аргументы:
----------
initial : list of tuples
    список исходных трансдукций вида [(трансдукция, стоимость)]
max_cost : float
    максимальная стоимость трансдукции

Возвращает:
-----------
final : list of tuples
    финальный список трансдукций вида [(трансдукция, стоимость)]


self.operation_costs = dict()<EOL>self.operation_costs["<STR_LIT>"] = {c: <NUM_LIT:1.0> for c in list(self.alphabet) + ['<STR_LIT:U+0020>']}<EOL>for a in self.alphabet:<EOL><INDENT>current_costs = {c: <NUM_LIT:1.0> for c in self.alphabet}<EOL>current_costs[a] = <NUM_LIT:0.0><EOL>current_costs["<STR_LIT>"] = <NUM_LIT:1.0><EOL>if allow_spaces:<EOL><INDENT>current_costs["<STR_LIT:U+0020>"] = <NUM_LIT:1.0><EOL><DEDENT>self.operation_costs[a] = current_costs<EOL><DEDENT>for a, b in itertools.permutations(self.alphabet, <NUM_LIT:2>):<EOL><INDENT>self.operation_costs[a + b] = {b + a: <NUM_LIT:1.0>}<EOL><DEDENT>if allow_spaces:<EOL><INDENT>self.operation_costs["<STR_LIT:U+0020>"] = {c: <NUM_LIT:1.0> for c in self.alphabet}<EOL>self.operation_costs["<STR_LIT:U+0020>"]["<STR_LIT>"] = <NUM_LIT:1.0><EOL><DEDENT>

sets 1.0 cost for every replacement, insertion, deletion and transposition


def __call__(self, batch: Iterable[Iterable[str]], *args, **kwargs) -> List[List[List[Tuple[float, str]]]]:

return [self._infer_instance(tokens) for tokens in batch]<EOL>

Propose candidates for tokens in sentences

        Args:
            batch: batch of tokenized sentences

        Returns:
            batch of lists of probabilities and candidates for every token


def precompute_future_symbols(trie, n, allow_spaces=False):

if n == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>if trie.is_terminated and trie.precompute_symbols:<EOL><INDENT>return<EOL><DEDENT>for index, final in enumerate(trie.final):<EOL><INDENT>trie.data[index] = [set() for i in range(n)]<EOL><DEDENT>for index, (node_data, final) in enumerate(zip(trie.data, trie.final)):<EOL><INDENT>node_data[<NUM_LIT:0>] = set(trie._get_letters(index))<EOL>if allow_spaces and final:<EOL><INDENT>node_data[<NUM_LIT:0>].add("<STR_LIT:U+0020>")<EOL><DEDENT><DEDENT>for d in range(<NUM_LIT:1>, n):<EOL><INDENT>for index, (node_data, final) in enumerate(zip(trie.data, trie.final)):<EOL><INDENT>children = set(trie._get_children(index))<EOL>for child in children:<EOL><INDENT>node_data[d] |= trie.data[child][d - <NUM_LIT:1>]<EOL><DEDENT>if allow_spaces and final:<EOL><INDENT>node_data[d] |= trie.data[trie.root][d - <NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT>trie.terminated = True<EOL>

Collecting possible continuations of length <= n for every node


with open(outfile, "<STR_LIT:w>", encoding="<STR_LIT:utf8>") as fout:<EOL><INDENT>attr_values = [getattr(self, attr) for attr in Trie.ATTRS]<EOL>attr_values.append(any(x is not None for x in self.data))<EOL>fout.write("<STR_LIT>".format(<EOL>"<STR_LIT:U+0020>".join("<STR_LIT:T>" if x else "<STR_LIT:F>" for x in attr_values),<EOL>self.nodes_number, self.root))<EOL>fout.write("<STR_LIT:U+0020>".join(str(a) for a in self.alphabet) + "<STR_LIT:\n>")<EOL>for index, label in enumerate(self.final):<EOL><INDENT>letters = self._get_letters(index, return_indexes=True)<EOL>children = self._get_children(index)<EOL>fout.write("<STR_LIT>".format(<EOL>"<STR_LIT:T>" if label else "<STR_LIT:F>", "<STR_LIT:U+0020>".join("<STR_LIT>".format(*elem)<EOL>for elem in zip(letters, children))))<EOL><DEDENT>if self.precompute_symbols is not None:<EOL><INDENT>for elem in self.data:<EOL><INDENT>fout.write("<STR_LIT::>".join("<STR_LIT:U+002C>".join(<EOL>map(str, symbols)) for symbols in elem) + "<STR_LIT:\n>")<EOL><DEDENT><DEDENT><DEDENT>return<EOL>

Сохраняет дерево для дальнейшего использования


self._descendance_cash = [dict() for _ in self.graph]<EOL>self.descend = self._descend_cashed<EOL>

Включает кэширование запросов к descend


if self.is_terminated:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>if s == "<STR_LIT>":<EOL><INDENT>self._set_final(self.root)<EOL>return<EOL><DEDENT>curr = self.root<EOL>for i, a in enumerate(s):<EOL><INDENT>code = self.alphabet_codes[a]<EOL>next = self.graph[curr][code]<EOL>if next == Trie.NO_NODE:<EOL><INDENT>curr = self._add_descendant(curr, s[i:])<EOL>break<EOL><DEDENT>else:<EOL><INDENT>curr = next<EOL><DEDENT><DEDENT>self._set_final(curr)<EOL>return self<EOL>

Добавление строки s в префиксный бор


branch, word, indexes = [self.root], [], [<NUM_LIT:0>]<EOL>letters_with_children = [self._get_children_and_letters(self.root)]<EOL>while len(branch) > <NUM_LIT:0>:<EOL><INDENT>if self.is_final(branch[-<NUM_LIT:1>]):<EOL><INDENT>yield "<STR_LIT>".join(word)<EOL><DEDENT>while indexes[-<NUM_LIT:1>] == len(letters_with_children[-<NUM_LIT:1>]):<EOL><INDENT>indexes.pop()<EOL>letters_with_children.pop()<EOL>branch.pop()<EOL>if len(indexes) == <NUM_LIT:0>:<EOL><INDENT>raise StopIteration()<EOL><DEDENT>word.pop()<EOL><DEDENT>next_letter, next_child = letters_with_children[-<NUM_LIT:1>][indexes[-<NUM_LIT:1>]]<EOL>indexes[-<NUM_LIT:1>] += <NUM_LIT:1><EOL>indexes.append(<NUM_LIT:0>)<EOL>word.append(next_letter)<EOL>branch.append(next_child)<EOL>letters_with_children.append(self._get_children_and_letters(branch[-<NUM_LIT:1>]))<EOL><DEDENT>

Возвращает итератор по словам, содержащимся в боре


return self.final[index]<EOL>

Аргументы
---------
index: int, номер вершины

Возвращает
----------
True: если index --- номер финальной вершины


def find_partitions(self, s, max_count=<NUM_LIT:1>):

curr_agenda = [(self.root, [], <NUM_LIT:0>)]<EOL>for i, a in enumerate(s):<EOL><INDENT>next_agenda = []<EOL>for curr, borders, cost in curr_agenda:<EOL><INDENT>if cost >= max_count:<EOL><INDENT>continue<EOL><DEDENT>child = self.graph[curr][self.alphabet_codes[a]]<EOL>if child == Trie.NO_NODE:<EOL><INDENT>continue<EOL><DEDENT>next_agenda.append((child, borders, cost))<EOL>if self.is_final(child):<EOL><INDENT>next_agenda.append((self.root, borders + [i+<NUM_LIT:1>], cost+<NUM_LIT:1>))<EOL><DEDENT><DEDENT>curr_agenda = next_agenda<EOL><DEDENT>answer = []<EOL>for curr, borders, cost in curr_agenda:<EOL><INDENT>if curr == self.root:<EOL><INDENT>borders = [<NUM_LIT:0>] + borders<EOL>answer.append([s[left:borders[i+<NUM_LIT:1>]] for i, left in enumerate(borders[:-<NUM_LIT:1>])])<EOL><DEDENT><DEDENT>return answer<EOL>

Находит все разбиения s = s_1 ... s_m на словарные слова s_1, ..., s_m
для m <= max_count


def _add_empty_child(self, parent, code, final=False):

self.graph[parent][code] = self.nodes_number<EOL>self.graph.append(self._make_default_node())<EOL>self.data.append(None)<EOL>self.final.append(final)<EOL>self.nodes_number += <NUM_LIT:1><EOL>return (self.nodes_number - <NUM_LIT:1>)<EOL>

Добавление ребёнка к вершине parent по символу с кодом code


for a in s:<EOL><INDENT>curr = self.graph[curr][self.alphabet_codes[a]]<EOL>if curr == Trie.NO_NODE:<EOL><INDENT>break<EOL><DEDENT><DEDENT>return curr<EOL>

Спуск из вершины curr по строке s


if s == "<STR_LIT>":<EOL><INDENT>return curr<EOL><DEDENT>curr_cash = self._descendance_cash[curr]<EOL>answer = curr_cash.get(s, None)<EOL>if answer is not None:<EOL><INDENT>return answer<EOL><DEDENT>res = curr<EOL>for a in s:<EOL><INDENT>res = self.graph[res][self.alphabet_codes[a]]<EOL>if res == Trie.NO_NODE:<EOL><INDENT>break<EOL><DEDENT><DEDENT>curr_cash[s] = res<EOL>return res<EOL>

Спуск из вершины curr по строке s с кэшированием


self.final[curr] = True<EOL>

Делает состояние curr завершающим


def _get_letters(self, index, return_indexes=False):

if self.dict_storage:<EOL><INDENT>answer = list(self.graph[index].keys())<EOL><DEDENT>else:<EOL><INDENT>answer =  [i for i, elem in enumerate(self.graph[index])<EOL>if elem != Trie.NO_NODE]<EOL><DEDENT>if not return_indexes:<EOL><INDENT>answer = [(self.alphabet[i] if i >= <NUM_LIT:0> else "<STR_LIT:U+0020>") for i in answer]<EOL><DEDENT>return answer<EOL>

Извлекает все метки выходных рёбер вершины с номером index


if self.dict_storage:<EOL><INDENT>return list(self.graph[index].values())<EOL><DEDENT>else:<EOL><INDENT>return [elem for elem in self.graph[index] if elem != Trie.NO_NODE]<EOL><DEDENT>

Извлекает всех потомков вершины с номером index


order, stack = [], []<EOL>stack.append(trie.root)<EOL>colors = ['<STR_LIT>'] * len(trie)<EOL>while len(stack) > <NUM_LIT:0>:<EOL><INDENT>index = stack[-<NUM_LIT:1>]<EOL>color = colors[index]<EOL>if color == '<STR_LIT>': <EOL><INDENT>colors[index] = '<STR_LIT>'<EOL>for child in trie._get_children(index):<EOL><INDENT>if child != Trie.NO_NODE and colors[child] == '<STR_LIT>':<EOL><INDENT>stack.append(child)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if color == '<STR_LIT>':<EOL><INDENT>colors[index] = '<STR_LIT>'<EOL>order.append(index)<EOL><DEDENT>stack = stack[:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>return order<EOL>

Обратная топологическая сортировка


def process_word(word: str, to_lower: bool = False,<EOL>append_case: Optional[str] = None) -> Tuple[str]:

if all(x.isupper() for x in word) and len(word) > <NUM_LIT:1>:<EOL><INDENT>uppercase = "<STR_LIT>"<EOL><DEDENT>elif word[<NUM_LIT:0>].isupper():<EOL><INDENT>uppercase = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>uppercase = None<EOL><DEDENT>if to_lower:<EOL><INDENT>word = word.lower()<EOL><DEDENT>if word.isdigit():<EOL><INDENT>answer = ["<STR_LIT>"]<EOL><DEDENT>elif word.startswith("<STR_LIT>") or word.startswith("<STR_LIT>"):<EOL><INDENT>answer = ["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>answer = list(word)<EOL><DEDENT>if to_lower and uppercase is not None:<EOL><INDENT>if append_case == "<STR_LIT>":<EOL><INDENT>answer = [uppercase] + answer<EOL><DEDENT>elif append_case == "<STR_LIT>":<EOL><INDENT>answer = answer + [uppercase]<EOL><DEDENT><DEDENT>return tuple(answer)<EOL>

Converts word to a tuple of symbols, optionally converts it to lowercase
    and adds capitalization label.

    Args:
        word: input word
        to_lower: whether to lowercase
        append_case: whether to add case mark
            ('<FIRST_UPPER>' for first capital and '<ALL_UPPER>' for all caps)

    Returns:
        a preprocessed word


def __call__(self, batch: Union[List[str], Tuple[str]]) -> StrUTF8EncoderInfo:

if isinstance(batch, (list, tuple)):<EOL><INDENT>if isinstance(batch[-<NUM_LIT:1>], str):<EOL><INDENT>return self._encode_chars(batch)<EOL><DEDENT>else:<EOL><INDENT>return [self(line) for line in batch]<EOL><DEDENT><DEDENT>raise RuntimeError(f'<STR_LIT>'<EOL>f'<STR_LIT>')<EOL>

Recursively search for strings in a list and utf8 encode

        Args:
            batch: a string or a list containing strings

        Returns:
            the same structure where all strings are utf8 encoded


chars_ids = [self._word_to_char_ids(cur_word)<EOL>for cur_word in sentence]<EOL>return self._wrap_in_s_char(chars_ids)<EOL>

Encode the sentence as a white space delimited string of tokens.


return len(self)<EOL>


def __call__(self, contexts_raw: Tuple[str, ...], questions_raw: Tuple[str, ...],<EOL>**kwargs) -> Tuple[<EOL>List[str], List[List[str]], List[List[List[str]]],<EOL>List[List[int]], List[List[int]],<EOL>List[str], List[List[str]], List[List[List[str]]],<EOL>List[List[Tuple[int, int]]]<EOL>]:

contexts = []<EOL>contexts_tokens = []<EOL>contexts_chars = []<EOL>contexts_r2p = []<EOL>contexts_p2r = []<EOL>questions = []<EOL>questions_tokens = []<EOL>questions_chars = []<EOL>spans = []<EOL>for c_raw, q_raw in zip(contexts_raw, questions_raw):<EOL><INDENT>c, r2p, p2r = SquadPreprocessor.preprocess_str(c_raw, return_mapping=True)<EOL>c_tokens = [token.replace("<STR_LIT>", '<STR_LIT:">').replace("<STR_LIT>", '<STR_LIT:">') for token in word_tokenize(c)][:self.context_limit]<EOL>c_chars = [list(token)[:self.char_limit] for token in c_tokens]<EOL>q = SquadPreprocessor.preprocess_str(q_raw)<EOL>q_tokens = [token.replace("<STR_LIT>", '<STR_LIT:">').replace("<STR_LIT>", '<STR_LIT:">') for token in word_tokenize(q)][:self.question_limit]<EOL>q_chars = [list(token)[:self.char_limit] for token in q_tokens]<EOL>contexts.append(c)<EOL>contexts_tokens.append(c_tokens)<EOL>contexts_chars.append(c_chars)<EOL>contexts_r2p.append(r2p)<EOL>contexts_p2r.append(p2r)<EOL>questions.append(q)<EOL>questions_tokens.append(q_tokens)<EOL>questions_chars.append(q_chars)<EOL>spans.append(SquadPreprocessor.convert_idx(c, c_tokens))<EOL><DEDENT>return contexts, contexts_tokens, contexts_chars, contexts_r2p, contexts_p2r,questions, questions_tokens, questions_chars, spans<EOL>

Performs preprocessing of context and question
        Args:
            contexts_raw: batch of contexts to preprocess
            questions_raw: batch of questions to preprocess

        Returns:
            context: batch of processed contexts
            contexts_tokens: batch of tokenized contexts
            contexts_chars: batch of tokenized and split on chars contexts
            contexts_r2p: batch of mappings from raw context to processed context
            contexts_p2r: batch of mappings from procesesd context to raw context
            questions: batch of processed questions
            questions_tokens: batch of tokenized questions
            questions_chars: batch of tokenized and split on chars questions
            spans: batch of mapping tokens to position in context


@staticmethod<EOL><INDENT>def preprocess_str(line: str, return_mapping: bool = False) -> Union[Tuple[str, List[int], List[int]], str]:<DEDENT>

if not return_mapping:<EOL><INDENT>return '<STR_LIT>'.join(c for c in line if not unicodedata.combining(c)).replace("<STR_LIT>", '<STR_LIT>').replace("<STR_LIT>", '<STR_LIT>')<EOL><DEDENT>r2p = [len(line)] * (len(line) + <NUM_LIT:1>)<EOL>p2r = [len(line)] * (len(line) + <NUM_LIT:1>)<EOL>s = '<STR_LIT>'<EOL>for i, c in enumerate(line):<EOL><INDENT>if unicodedata.combining(c):<EOL><INDENT>r2p[i] = -<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>s += c<EOL>r2p[i] = len(s) - <NUM_LIT:1><EOL>p2r[len(s) - <NUM_LIT:1>] = i<EOL><DEDENT><DEDENT>return s.replace("<STR_LIT>", '<STR_LIT>').replace("<STR_LIT>", '<STR_LIT>'), r2p, p2r<EOL>

Removes unicode and other characters from str

        Args:
            line: string to process
            return_mapping: return mapping from line to preprocessed line or not

        Returns:
            preprocessed line, raw2preprocessed mapping, preprocessed2raw mapping


def __call__(self, answers_raw: Tuple[List[str], ...], answers_start: Tuple[List[int], ...],<EOL>r2ps: List[List[int]], spans: List[List[Tuple[int, int]]],<EOL>**kwargs) -> Tuple[List[List[str]], List[List[int]], List[List[int]]]:

answers = []<EOL>start = []<EOL>end = []<EOL>for ans_raw, ans_st, r2p, span in zip(answers_raw, answers_start, r2ps, spans):<EOL><INDENT>start.append([])<EOL>end.append([])<EOL>answers.append([])<EOL>for a_raw, a_st in zip(ans_raw, ans_st):<EOL><INDENT>ans = SquadPreprocessor.preprocess_str(a_raw)<EOL>ans_st = r2p[a_st]<EOL>ans_end = ans_st + len(ans)<EOL>answer_span = []<EOL>for idx, sp in enumerate(span):<EOL><INDENT>if not (ans_end <= sp[<NUM_LIT:0>] or ans_st >= sp[<NUM_LIT:1>]):<EOL><INDENT>answer_span.append(idx)<EOL><DEDENT><DEDENT>if len(answer_span) != <NUM_LIT:0>:<EOL><INDENT>y1, y2 = answer_span[<NUM_LIT:0>], answer_span[-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>y1, y2 = -<NUM_LIT:1>, -<NUM_LIT:1><EOL><DEDENT>start[-<NUM_LIT:1>].append(y1)<EOL>end[-<NUM_LIT:1>].append(y2)<EOL>answers[-<NUM_LIT:1>].append(ans)<EOL><DEDENT><DEDENT>return answers, start, end<EOL>

Processes answers for SQuAD dataset

        Args:
            answers_raw: list of str [batch_size x number_of_answers]
            answers_start: start position of answer (in chars) [batch_size x number_of_answers]
            r2ps: mapping from raw context to processed context
            spans: mapping tokens to position in context

        Returns:
            processed answer text, start position in tokens, end position in tokens
            [batch_size x number_of_answers]


def __call__(self, contexts: List[List[str]], questions: List[List[str]]) -> Tuple[np.ndarray, np.ndarray]:

if self.level == '<STR_LIT>':<EOL><INDENT>c_idxs = np.zeros([len(contexts), self.context_limit], dtype=np.int32)<EOL>q_idxs = np.zeros([len(questions), self.question_limit], dtype=np.int32)<EOL>for i, context in enumerate(contexts):<EOL><INDENT>for j, token in enumerate(context):<EOL><INDENT>c_idxs[i, j] = self._get_idx(token)<EOL><DEDENT><DEDENT>for i, question in enumerate(questions):<EOL><INDENT>for j, token in enumerate(question):<EOL><INDENT>q_idxs[i, j] = self._get_idx(token)<EOL><DEDENT><DEDENT><DEDENT>elif self.level == '<STR_LIT>':<EOL><INDENT>c_idxs = np.zeros([len(contexts), self.context_limit, self.char_limit], dtype=np.int32)<EOL>q_idxs = np.zeros([len(questions), self.question_limit, self.char_limit], dtype=np.int32)<EOL>for i, context in enumerate(contexts):<EOL><INDENT>for j, token in enumerate(context):<EOL><INDENT>for k, char in enumerate(token):<EOL><INDENT>c_idxs[i, j, k] = self._get_idx(char)<EOL><DEDENT><DEDENT><DEDENT>for i, question in enumerate(questions):<EOL><INDENT>for j, token in enumerate(question):<EOL><INDENT>for k, char in enumerate(token):<EOL><INDENT>q_idxs[i, j, k] = self._get_idx(char)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return c_idxs, q_idxs<EOL>

Transforms tokens/chars to indices.

        Args:
            contexts: batch of list of tokens in context
            questions: batch of list of tokens in question

        Returns:
            transformed contexts and questions


for e in (el, el.lower(), el.capitalize(), el.upper()):<EOL><INDENT>if e in self.token2idx_dict:<EOL><INDENT>return self.token2idx_dict[e]<EOL><DEDENT><DEDENT>return <NUM_LIT:1><EOL>

Returns idx for el (token or char).

        Args:
            el: token or character

        Returns:
            idx in vocabulary


def __call__(self, ans_start: Tuple[int, ...], ans_end: Tuple[int, ...], contexts: Tuple[str, ...],<EOL>p2rs: List[List[int]], spans: List[List[Tuple[int, int]]],<EOL>**kwargs) -> Tuple[List[str], List[int], List[int]]:

answers = []<EOL>start = []<EOL>end = []<EOL>for a_st, a_end, c, p2r, span in zip(ans_start, ans_end, contexts, p2rs, spans):<EOL><INDENT>if a_st == -<NUM_LIT:1> or a_end == -<NUM_LIT:1>:<EOL><INDENT>start.append(-<NUM_LIT:1>)<EOL>end.append(-<NUM_LIT:1>)<EOL>answers.append('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>start.append(p2r[span[a_st][<NUM_LIT:0>]])<EOL>end.append(p2r[span[a_end][<NUM_LIT:1>]])<EOL>answers.append(c[start[-<NUM_LIT:1>]:end[-<NUM_LIT:1>]])<EOL><DEDENT><DEDENT>return answers, start, end<EOL>

Extracts answer from context using predicted answer positions.

        Args:
            ans_start: predicted start position in processed context: list of ints with len(ans_start) == batch_size
            ans_end: predicted end position in processed context
            contexts: raw contexts
            p2rs: mapping from processed context to raw
            spans: tokens positions in context

        Returns:
            postprocessed answer text, start position in raw context, end position in raw context


sanitized_batch = []<EOL>for utterance in tokens_batch:<EOL><INDENT>sanitized_utterance = []<EOL>for token in utterance:<EOL><INDENT>token = unicodedata.normalize('<STR_LIT>', token)<EOL>sanitized_utterance.append(token.translate(self.combining_characters))<EOL><DEDENT>sanitized_batch.append(sanitized_utterance)<EOL><DEDENT>return sanitized_batch<EOL>

Takes batch of tokens and returns the batch with sanitized tokens


def __call__(self, batch_docs: List[Union[str, List[str]]]) ->List[Union[List[str], List[List[str]]]]:

result = []<EOL>for docs in batch_docs:<EOL><INDENT>batch_chunks = []<EOL>if isinstance(docs, str):<EOL><INDENT>docs = [docs]<EOL><DEDENT>for doc in docs:<EOL><INDENT>if self.paragraphs:<EOL><INDENT>split_doc = doc.split('<STR_LIT>')<EOL>split_doc = [sd.strip() for sd in split_doc]<EOL>split_doc = list(filter(lambda x: len(x) > <NUM_LIT>, split_doc))<EOL>batch_chunks.append(split_doc)<EOL><DEDENT>else:<EOL><INDENT>doc_chunks = []<EOL>if self.keep_sentences:<EOL><INDENT>sentences = sent_tokenize(doc)<EOL>n_tokens = <NUM_LIT:0><EOL>keep = []<EOL>for s in sentences:<EOL><INDENT>n_tokens += len(s.split())<EOL>if n_tokens > self.tokens_limit:<EOL><INDENT>if keep:<EOL><INDENT>doc_chunks.append('<STR_LIT:U+0020>'.join(keep))<EOL>n_tokens = <NUM_LIT:0><EOL>keep.clear()<EOL><DEDENT><DEDENT>keep.append(s)<EOL><DEDENT>if keep:<EOL><INDENT>doc_chunks.append('<STR_LIT:U+0020>'.join(keep))<EOL><DEDENT>batch_chunks.append(doc_chunks)<EOL><DEDENT>else:<EOL><INDENT>split_doc = doc.split()<EOL>doc_chunks = [split_doc[i:i + self.tokens_limit] for i in<EOL>range(<NUM_LIT:0>, len(split_doc), self.tokens_limit)]<EOL>batch_chunks.append(doc_chunks)<EOL><DEDENT><DEDENT><DEDENT>result.append(batch_chunks)<EOL><DEDENT>if self.flatten_result:<EOL><INDENT>if isinstance(result[<NUM_LIT:0>][<NUM_LIT:0>], list):<EOL><INDENT>for i in range(len(result)):<EOL><INDENT>flattened = list(chain.from_iterable(result[i]))<EOL>result[i] = flattened<EOL><DEDENT><DEDENT><DEDENT>return result<EOL>

Make chunks from a batch of documents. There can be several documents in each batch.
        Args:
            batch_docs: a batch of documents / a batch of lists of documents
        Returns:
            chunks of docs, flattened or not


def __call__(self, batch_s: List[str], ref: List[str]) -> List[List[str]]:

res = []<EOL>for s, r in zip(batch_s, ref):<EOL><INDENT>res.append([s] * len(r))<EOL><DEDENT>return res<EOL>

Multiply each string in a provided batch of strings.

        Args:
            batch_s: a batch of strings to be multiplied
            ref: a reference to obtain a length of the resulting list

        Returns:
            a multiplied s as list


def __call__(self, texts_a: List[str], texts_b: Optional[List[str]] = None) -> List[InputFeatures]:

if texts_b is None:<EOL><INDENT>texts_b = [None] * len(texts_a)<EOL><DEDENT>examples = [InputExample(unique_id=<NUM_LIT:0>, text_a=text_a, text_b=text_b)<EOL>for text_a, text_b in zip(texts_a, texts_b)]<EOL>return convert_examples_to_features(examples, self.max_seq_length, self.tokenizer)<EOL>

Call Bert convert_examples_to_features function to tokenize and create masks.

        texts_a and texts_b are separated by [SEP] token

        Args:
            texts_a: list of texts,
            texts_b: list of texts, it could be None, e.g. single sentence classification task

        Returns:
            batch of InputFeatures with subtokens, subtoken ids, subtoken mask, segment mask.


def __call__(self, batch: List[List[int]], **kwargs) -> Union[List[List[np.ndarray]], List[np.ndarray]]:

one_hotted_batch = []<EOL>for utt in batch:<EOL><INDENT>if isinstance(utt, list):<EOL><INDENT>one_hotted_utt = self._to_one_hot(utt, self._depth)<EOL><DEDENT>elif isinstance(utt, int):<EOL><INDENT>if self._pad_zeros or self.single_vector:<EOL><INDENT>one_hotted_utt = self._to_one_hot([utt], self._depth)<EOL><DEDENT>else:<EOL><INDENT>one_hotted_utt = self._to_one_hot([utt], self._depth).reshape(-<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if self.single_vector:<EOL><INDENT>one_hotted_utt = np.sum(one_hotted_utt, axis=<NUM_LIT:0>)<EOL><DEDENT>one_hotted_batch.append(one_hotted_utt)<EOL><DEDENT>if self._pad_zeros:<EOL><INDENT>one_hotted_batch = zero_pad(one_hotted_batch)<EOL><DEDENT>return one_hotted_batch<EOL>

Convert given batch of list of labels to one-hot representation of the batch.

Args:
    batch: list of samples, where each sample is a list of integer labels.
    **kwargs: additional arguments

Returns:
    if ``single_vector``, list of one-hot representations of each sample,
    otherwise, list of lists of one-hot representations of each label in a sample


def __call__(self, batch: Union[str, list, tuple]) -> StrTokenReverserInfo:

if isinstance(batch, (list, tuple)):<EOL><INDENT>batch = batch.copy()<EOL><DEDENT>if self.tokenized:<EOL><INDENT>if isinstance(batch, (list, tuple)):<EOL><INDENT>if isinstance(batch[-<NUM_LIT:1>], str):<EOL><INDENT>return self._reverse_tokens(batch)<EOL><DEDENT>else:<EOL><INDENT>return [self(line) for line in batch]<EOL><DEDENT><DEDENT>raise RuntimeError(f'<STR_LIT>'<EOL>f'<STR_LIT>'<EOL>f'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>if isinstance(batch, (list, tuple)):<EOL><INDENT>return [self(line) for line in batch]<EOL><DEDENT>else:<EOL><INDENT>return self._reverse_str(batch)<EOL><DEDENT><DEDENT>

Recursively search for strings in a list and convert them to strings with reversed token positions

        Args:
            batch: a string or a list containing strings

        Returns:
            the same structure where all strings tokens are reversed


def __call__(self, tokens_batch, **kwargs):

lemma_batch = []<EOL>for utterance in tokens_batch:<EOL><INDENT>lemma_utterance = []<EOL>for token in utterance:<EOL><INDENT>p = self.lemmatizer.parse(token)[<NUM_LIT:0>]<EOL>lemma_utterance.append(p.normal_form)<EOL><DEDENT>lemma_batch.append(lemma_utterance)<EOL><DEDENT>return lemma_batch<EOL>

Takes batch of tokens and returns the lemmatized tokens.


def __call__(self, batch: List[str], **kwargs) -> List[str]:

f = [x.lower() for x in batch]<EOL>f = [re.sub("<STR_LIT>", "<STR_LIT:U+0020>", x) for x in f]<EOL>f = [re.sub('<STR_LIT>', '<STR_LIT:U+0020>', x) for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [re.sub("<STR_LIT>", "<STR_LIT>", x) for x in f]<EOL>f = [re.sub("<STR_LIT>", "<STR_LIT:U+0020>", x) for x in f]<EOL>f = [re.sub("<STR_LIT>", "<STR_LIT:U+0020>", x) for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT:U+0020>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT:U+0020>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT:U+0020>") for x in f]<EOL>f = [x.replace("<STR_LIT>", "<STR_LIT:U+0020>") for x in f]<EOL>f = [re.sub("<STR_LIT>", "<STR_LIT>", x) for x in f]<EOL>f = [re.sub(r'<STR_LIT>' + string.printable + r'<STR_LIT>', r'<STR_LIT>', x).strip() for x in f]<EOL>if self.remove_punctuation:<EOL><INDENT>f = [re.sub(r'<STR_LIT>' + string.punctuation + '<STR_LIT>', '<STR_LIT:U+0020>', x) for x in f]<EOL><DEDENT>f = [re.sub('<STR_LIT>', '<STR_LIT:U+0020>', x) for x in f]<EOL>return f<EOL>

Preprocess given batch

Args:
    batch: list of text samples
    **kwargs: additional arguments

Returns:
    list of preprocessed text samples


def extract_money(self, doc: spacy.tokens.Doc) -> Tuple[List, Tuple[float, float]]:

matches = self.matcher(doc)<EOL>money_range: Tuple = ()<EOL>doc_no_money = list(doc)<EOL>negated = False<EOL>for match_id, start, end in matches:<EOL><INDENT>string_id = self.model.vocab.strings[match_id]<EOL>span = doc[start:end]<EOL>for child in doc[start].children:<EOL><INDENT>if child.dep_ == '<STR_LIT>':<EOL><INDENT>negated = True<EOL><DEDENT><DEDENT>num_token = [token for token in span if token.like_num == True]<EOL>if (string_id == '<STR_LIT>' and negated == False) or (string_id == '<STR_LIT>' and negated == True):<EOL><INDENT>money_range = (<NUM_LIT:0>, float(num_token[<NUM_LIT:0>].text))<EOL><DEDENT>if (string_id == '<STR_LIT>' and negated == False) or (string_id == '<STR_LIT>' and negated == True):<EOL><INDENT>money_range = (float(num_token[<NUM_LIT:0>].text), float(math.inf))<EOL><DEDENT>del doc_no_money[start:end + <NUM_LIT:1>]<EOL><DEDENT>return doc_no_money, money_range<EOL>

Extract money entities and money related tokens from `doc`.

        Parameters:
            doc: a list of tokens with corresponding tags, lemmas, etc.

        Returns:
            doc_no_money: doc with no money related tokens.
            money_range: money range from `money_range[0]` to `money_range[1]` extracted from the doc.


def analyze(self, text: str) -> Iterable:

return self.model(text)<EOL>


def spacy2dict(self, doc: spacy.tokens.Doc, fields: List[str] = None) -> List[Dict[Any, Any]]:

if fields is None:<EOL><INDENT>fields = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:text>']<EOL><DEDENT>return [{field: getattr(token, field) for field in fields} for token in doc]<EOL>

Convert SpaCy doc into list of tokens with `fields` properties only


def filter_nlp(self, tokens: Iterable) -> List[Any]:

res = []<EOL>for word in tokens:<EOL><INDENT>if word.tag_ not in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>res.append(word)<EOL><DEDENT><DEDENT>return res<EOL>


def filter_nlp_title(self, doc: Iterable) -> List[Any]:

return [w for w in doc if w.tag_ in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'] and not w.like_num]<EOL>

Filter item titles according to the POS tags


def lemmas(self, doc: Iterable) -> List[str]:

return [w.get('<STR_LIT>') if isinstance(w, dict) else w.lemma_ for w in doc]<EOL>


def price(self, item: Dict[Any, Any]) -> float:

if '<STR_LIT>' in item:<EOL><INDENT>return float(item['<STR_LIT>'].split('<STR_LIT:$>')[<NUM_LIT:1>].replace("<STR_LIT:U+002C>", "<STR_LIT>"))<EOL><DEDENT>return <NUM_LIT:0><EOL>


def parse_input(self, inp: str) -> Dict[Any, Any]:

state: List = []<EOL>for i in range(len(inp.split()) // <NUM_LIT:2>, <NUM_LIT:0>, -<NUM_LIT:1>):<EOL><INDENT>state.append([inp.split(None, <NUM_LIT:1>)[<NUM_LIT:0>], inp.split(None, <NUM_LIT:1>)[<NUM_LIT:1>].split()[<NUM_LIT:0>]])<EOL>if i > <NUM_LIT:1>:<EOL><INDENT>inp = inp.split(None, <NUM_LIT:2>)[<NUM_LIT:2>]<EOL><DEDENT><DEDENT>return dict(state)<EOL>

Convert space-delimited string into dialog state


def __call__(self, batch: Union[str, list, tuple]):

if isinstance(batch, (list, tuple)):<EOL><INDENT>return [self(line) for line in batch]<EOL><DEDENT>else:<EOL><INDENT>return batch.lower()<EOL><DEDENT>

Recursively search for strings in a list and convert them to lowercase

        Args:
            batch: a string or a list containing strings at some level of nesting

        Returns:
            the same structure where all strings are converted to lowercase


def __init__(self, model_class: str,<EOL>save_path: Union[str, Path] = None,<EOL>load_path: Union[str, Path] = None,<EOL>infer_method: str = "<STR_LIT>",<EOL>ensure_list_output: bool = False,<EOL>**kwargs) -> None:

super().__init__(save_path=save_path, load_path=load_path, **kwargs)<EOL>self.model_class = model_class<EOL>self.model_params = kwargs<EOL>self.model = None<EOL>self.ensure_list_output = ensure_list_output<EOL>self.pipe_params = {}<EOL>for required in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT:name>"]:<EOL><INDENT>self.pipe_params[required] = self.model_params.pop(required, None)<EOL><DEDENT>self.load()<EOL>self.infer_method = getattr(self.model, infer_method)<EOL>

Initialize component with given parameters


n_inputs = len(self.pipe_params["<STR_LIT>"]) if isinstance(self.pipe_params["<STR_LIT>"], list) else <NUM_LIT:1><EOL>x_features = self.compose_input_data(args[:n_inputs])<EOL>if len(args) > n_inputs:<EOL><INDENT>y_ = np.squeeze(np.array(args[-<NUM_LIT:1>]))<EOL><DEDENT>else:<EOL><INDENT>y_ = None<EOL><DEDENT>try:<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class))<EOL>self.model.fit(x_features, y_)<EOL><DEDENT>except TypeError or ValueError:<EOL><INDENT>if issparse(x_features):<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class))<EOL>self.model.fit(x_features.todense(), y_)<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class))<EOL>self.model.fit(csr_matrix(x_features), y_)<EOL><DEDENT><DEDENT>return<EOL>

Fit model on the given data

Args:
    *args: list of x-inputs and, optionally, one y-input (the last one) to fit on.
        Possible input (x0, ..., xK, y) or (x0, ..., xK) '
        where K is the number of input data elements (the length of list ``in`` from config). \
        In case of several inputs (K > 1) input features will be stacked. \
        For example, one has x0: (n_samples, n_features0), ..., xK: (n_samples, n_featuresK), \
        then model will be trained on x: (n_samples, n_features0 + ... + n_featuresK).

Returns:
    None


x_features = self.compose_input_data(args)<EOL>try:<EOL><INDENT>predictions = self.infer_method(x_features)<EOL><DEDENT>except TypeError or ValueError:<EOL><INDENT>if issparse(x_features):<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class))<EOL>predictions = self.infer_method(x_features.todense())<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class))<EOL>predictions = self.infer_method(csr_matrix(x_features))<EOL><DEDENT><DEDENT>if isinstance(predictions, list):<EOL><INDENT>predictions_ = [[predictions[j][i][<NUM_LIT:1>] for j in range(len(predictions))] for i in range(x_features.shape[<NUM_LIT:0>])]<EOL>predictions = np.array(predictions_)<EOL><DEDENT>if self.ensure_list_output and len(predictions.shape) == <NUM_LIT:1>:<EOL><INDENT>predictions = predictions.reshape(-<NUM_LIT:1>, <NUM_LIT:1>)<EOL><DEDENT>if issparse(predictions):<EOL><INDENT>return predictions<EOL><DEDENT>else:<EOL><INDENT>return predictions.tolist()<EOL><DEDENT>

Infer on the given data according to given in the config infer method, \
    e.g. ``"predict", "predict_proba", "transform"``

Args:
    *args: list of inputs

Returns:
    predictions, e.g. list of labels, array of probability distribution, sparse array of vectorized samples


log.info("<STR_LIT>".format(self.model_class))<EOL>model_function = cls_from_str(self.model_class)<EOL>if model_function is None:<EOL><INDENT>raise ConfigError("<STR_LIT>".format(self.model_class))<EOL><DEDENT>given_params = {}<EOL>if self.model_params:<EOL><INDENT>available_params = self.get_function_params(model_function)<EOL>for param_name in self.model_params.keys():<EOL><INDENT>if param_name in available_params:<EOL><INDENT>try:<EOL><INDENT>given_params[param_name] = cls_from_str(self.model_params[param_name])<EOL><DEDENT>except (AttributeError, ValueError, ConfigError):<EOL><INDENT>given_params[param_name] = self.model_params[param_name]<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.model = model_function(**given_params)<EOL>return<EOL>

Initialize ``self.model`` as some sklearn model from scratch with given in ``self.model_params`` parameters.

Returns:
    None


def load(self, fname: str = None) -> None:

if fname is None:<EOL><INDENT>fname = self.load_path<EOL><DEDENT>fname = Path(fname).with_suffix('<STR_LIT>')<EOL>if fname.exists():<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class, str(fname)))<EOL>with open(fname, "<STR_LIT:rb>") as f:<EOL><INDENT>self.model = pickle.load(f)<EOL><DEDENT>warm_start = self.model_params.get("<STR_LIT>", None)<EOL>self.model_params = {param: getattr(self.model, param) for param in self.get_class_attributes(self.model)}<EOL>self.model_class = self.model.__module__ + self.model.__class__.__name__<EOL>log.info("<STR_LIT>".format(self.model_class))<EOL>if warm_start and "<STR_LIT>" in self.model_params.keys():<EOL><INDENT>self.model_params["<STR_LIT>"] = True<EOL>log.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>log.warning("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.warning("<STR_LIT>".format(str(fname)))<EOL>self.init_from_scratch()<EOL><DEDENT>return<EOL>

Initialize ``self.model`` as some sklearn model from saved re-initializing ``self.model_params`` parameters. \
    If in new given parameters ``warm_start`` is set to True and given model admits ``warm_start`` parameter, \
    model will be initilized from saved with opportunity to continue fitting.

Args:
    fname: string name of path to model to load from

Returns:
    None


def save(self, fname: str = None) -> None:

if fname is None:<EOL><INDENT>fname = self.save_path<EOL><DEDENT>fname = Path(fname).with_suffix('<STR_LIT>')<EOL>log.info("<STR_LIT>".format(str(fname)))<EOL>with open(fname, "<STR_LIT:wb>") as f:<EOL><INDENT>pickle.dump(self.model, f, protocol=<NUM_LIT:4>)<EOL><DEDENT>return<EOL>

Save ``self.model`` to the file from ``fname`` or, if not given, ``self.save_path``. \
    If ``self.save_path`` does not have ``.pkl`` extension, then it will be replaced \
    to ``str(Path(self.save_path).stem) + ".pkl"``

Args:
    fname:  string name of path to model to save to

Returns:
    None


@staticmethod<EOL><INDENT>def compose_input_data(x: List[Union[Tuple[Union[np.ndarray, list, spmatrix, str]],<EOL>List[Union[np.ndarray, list, spmatrix, str]],<EOL>np.ndarray, spmatrix]]) -> Union[spmatrix, np.ndarray]:<DEDENT>

x_features = []<EOL>for i in range(len(x)):<EOL><INDENT>if ((isinstance(x[i], tuple) or isinstance(x[i], list) or isinstance(x[i], np.ndarray) and len(x[i]))<EOL>or (issparse(x[i]) and x[i].shape[<NUM_LIT:0>])):<EOL><INDENT>if issparse(x[i][<NUM_LIT:0>]):<EOL><INDENT>x_features.append(vstack(list(x[i])))<EOL><DEDENT>elif isinstance(x[i][<NUM_LIT:0>], np.ndarray) or isinstance(x[i][<NUM_LIT:0>], list):<EOL><INDENT>x_features.append(np.vstack(list(x[i])))<EOL><DEDENT>elif isinstance(x[i][<NUM_LIT:0>], str):<EOL><INDENT>x_features.append(np.array(x[i]))<EOL><DEDENT>else:<EOL><INDENT>raise ConfigError('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT><DEDENT>sparse = False<EOL>for inp in x_features:<EOL><INDENT>if issparse(inp):<EOL><INDENT>sparse = True<EOL><DEDENT><DEDENT>if sparse:<EOL><INDENT>x_features = hstack(list(x_features))<EOL><DEDENT>else:<EOL><INDENT>x_features = np.hstack(list(x_features))<EOL><DEDENT>return x_features<EOL>

Stack given list of different types of inputs to the one matrix. If one of the inputs is a sparse matrix, \
    then output will be also a sparse matrix

Args:
    x: list of data elements

Returns:
    sparse or dense array of stacked data


@staticmethod<EOL><INDENT>def get_function_params(f: Callable) -> List[str]:<DEDENT>

return inspect.getfullargspec(f)[<NUM_LIT:0>]<EOL>

Get list of names of given function's parameters

Args:
    f: function

Returns:
    list of names of given function's parameters


@staticmethod<EOL><INDENT>def get_class_attributes(cls: type) -> List[str]:<DEDENT>

return list(cls.__dict__.keys())<EOL>

Get list of names of given class' attributes

Args:
    cls: class

Returns:
    list of names of given class' attributes


def train_on_batch(self, features: List[InputFeatures], y_st: List[List[int]], y_end: List[List[int]]) -> Dict:

input_ids = [f.input_ids for f in features]<EOL>input_masks = [f.input_mask for f in features]<EOL>input_type_ids = [f.input_type_ids for f in features]<EOL>y_st = [x[<NUM_LIT:0>] for x in y_st]<EOL>y_end = [x[<NUM_LIT:0>] for x in y_end]<EOL>feed_dict = self._build_feed_dict(input_ids, input_masks, input_type_ids, y_st, y_end)<EOL>_, loss = self.sess.run([self.train_op, self.loss], feed_dict=feed_dict)<EOL>return {'<STR_LIT>': loss, '<STR_LIT>': feed_dict[self.learning_rate_ph]}<EOL>

Train model on given batch.
        This method calls train_op using features and labels from y_st and y_end

        Args:
            features: batch of InputFeatures instances
            y_st: batch of lists of ground truth answer start positions
            y_end: batch of lists of ground truth answer end positions

        Returns:
            dict with loss and learning_rate values


def __call__(self, features: List[InputFeatures]) -> Tuple[List[int], List[int], List[float], List[float]]:

input_ids = [f.input_ids for f in features]<EOL>input_masks = [f.input_mask for f in features]<EOL>input_type_ids = [f.input_type_ids for f in features]<EOL>feed_dict = self._build_feed_dict(input_ids, input_masks, input_type_ids)<EOL>st, end, logits, scores = self.sess.run([self.start_pred, self.end_pred, self.yp_logits, self.yp_score], feed_dict=feed_dict)<EOL>return st, end, logits.tolist(), scores.tolist()<EOL>

get predictions using features as input

        Args:
            features: batch of InputFeatures instances

        Returns:
            predictions: start, end positions, logits for answer and no_answer score


def __call__(self, contexts: List[str], questions: List[str], **kwargs) -> Tuple[List[str], List[int], List[float]]:

batch_indices = []<EOL>contexts_to_predict = []<EOL>questions_to_predict = []<EOL>predictions = {}<EOL>for i, (context, question) in enumerate(zip(contexts, questions)):<EOL><INDENT>context_subtokens = self.tokenizer.tokenize(context)<EOL>question_subtokens = self.tokenizer.tokenize(question)<EOL>max_chunk_len = self.max_seq_length - len(question_subtokens) - <NUM_LIT:3><EOL>if max_chunk_len > <NUM_LIT:0> and len(context_subtokens) < max_chunk_len:<EOL><INDENT>number_of_chunks = math.ceil(len(context_subtokens) / max_chunk_len)<EOL>sentences = self.sent_tokenizer(context)<EOL>for chunk in np.array_split(sentences, number_of_chunks):<EOL><INDENT>contexts_to_predict += ['<STR_LIT:U+0020>'.join(chunk)]<EOL>questions_to_predict += [question]<EOL>batch_indices += [i]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>contexts_to_predict += [context]<EOL>questions_to_predict += [question]<EOL>batch_indices += [i]<EOL><DEDENT><DEDENT>for j in range(<NUM_LIT:0>, len(contexts_to_predict), self.batch_size):<EOL><INDENT>c_batch = contexts_to_predict[j: j + self.batch_size]<EOL>q_batch = questions_to_predict[j: j + self.batch_size]<EOL>ind_batch = batch_indices[j: j + self.batch_size]<EOL>a_batch, a_st_batch, logits_batch = self.model(c_batch, q_batch)<EOL>for a, a_st, logits, ind in zip(a_batch, a_st_batch, logits_batch, ind_batch):<EOL><INDENT>if ind in predictions:<EOL><INDENT>predictions[ind] += [(a, a_st, logits)]<EOL><DEDENT>else:<EOL><INDENT>predictions[ind] = [(a, a_st, logits)]<EOL><DEDENT><DEDENT><DEDENT>answers, answer_starts, logits = [], [], []<EOL>for ind in sorted(predictions.keys()):<EOL><INDENT>prediction = predictions[ind]<EOL>best_answer_ind = np.argmax([p[<NUM_LIT:2>] for p in prediction])<EOL>answers += [prediction[best_answer_ind][<NUM_LIT:0>]]<EOL>answer_starts += [prediction[best_answer_ind][<NUM_LIT:1>]]<EOL>logits += [prediction[best_answer_ind][<NUM_LIT:2>]]<EOL><DEDENT>return answers, answer_starts, logits<EOL>

get predictions for given contexts and questions

        Args:
            contexts: batch of contexts
            questions: batch of questions

        Returns:
            predictions: answer, answer start position, logits or scores


def train_on_batch(self, features: List[InputFeatures], y: Union[List[int], List[List[int]]]) -> Dict:

input_ids = [f.input_ids for f in features]<EOL>input_masks = [f.input_mask for f in features]<EOL>input_type_ids = [f.input_type_ids for f in features]<EOL>feed_dict = self._build_feed_dict(input_ids, input_masks, input_type_ids, y)<EOL>_, loss = self.sess.run([self.train_op, self.loss], feed_dict=feed_dict)<EOL>return {'<STR_LIT>': loss, '<STR_LIT>': feed_dict[self.learning_rate_ph]}<EOL>

Train model on given batch.
        This method calls train_op using features and y (labels).

        Args:
            features: batch of InputFeatures
            y: batch of labels (class id or one-hot encoding)

        Returns:
            dict with loss and learning_rate values


def __call__(self, features: List[InputFeatures]) -> Union[List[int], List[List[float]]]:

input_ids = [f.input_ids for f in features]<EOL>input_masks = [f.input_mask for f in features]<EOL>input_type_ids = [f.input_type_ids for f in features]<EOL>feed_dict = self._build_feed_dict(input_ids, input_masks, input_type_ids)<EOL>if not self.return_probas:<EOL><INDENT>pred = self.sess.run(self.y_predictions, feed_dict=feed_dict)<EOL><DEDENT>else:<EOL><INDENT>pred = self.sess.run(self.y_probas, feed_dict=feed_dict)<EOL><DEDENT>return pred<EOL>

Make prediction for given features (texts).

        Args:
            features: batch of InputFeatures

        Returns:
            predicted classes or probabilities of each class


@overrides<EOL><INDENT>def load(self, epoch: Optional[int] = None) -> None:<DEDENT>

path = self.load_path<EOL>if epoch is not None:<EOL><INDENT>path = path.parent / self.epoch_save_path / str(epoch) / path.parts[-<NUM_LIT:1>]<EOL>path.resolve()<EOL>log.info(f'<STR_LIT>')<EOL><DEDENT>path = str(path)<EOL>if tf.train.checkpoint_exists(path):<EOL><INDENT>log.info(f'<STR_LIT>')<EOL>with self.graph.as_default():<EOL><INDENT>saver = tf.train.Saver()<EOL>saver.restore(self.sess, path)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.info(f'<STR_LIT>')<EOL><DEDENT>

Load model parameters from self.load_path


@overrides<EOL><INDENT>def save(self, epoch: Optional[int] = None) -> None:<DEDENT>

path = self.save_path<EOL>if epoch is not None:<EOL><INDENT>path = path.parent / self.epoch_save_path / str(epoch) / path.parts[-<NUM_LIT:1>]<EOL>path.resolve()<EOL>log.info(f'<STR_LIT>')<EOL><DEDENT>path.parent.mkdir(parents=True, exist_ok=True)<EOL>path = str(path)<EOL>log.info(f'<STR_LIT>')<EOL>with self.graph.as_default():<EOL><INDENT>saver = tf.train.Saver()<EOL>saver.save(self.sess, path)<EOL><DEDENT>


def train_on_batch(self,<EOL>x_char_ids: list,<EOL>y_token_ids: list) -> List[float]:

char_ids_batches, reversed_char_ids_batches = x_char_ids<EOL>token_ids_batches, reversed_token_ids_batches = y_token_ids<EOL>feed_dict = self._fill_feed_dict(char_ids_batches, reversed_char_ids_batches,<EOL>token_ids_batches, reversed_token_ids_batches)<EOL>with self.graph.as_default():<EOL><INDENT>loss, _, self.init_state_values = self.sess.run([self.loss, self.train_op, self.final_state_tensors],<EOL>feed_dict)<EOL><DEDENT>return np.mean(loss)<EOL>

This method is called by trainer to make one training step on one batch.

Args:
    x_char_ids:  a batch of char_ids
    y_token_ids: a batch of token_ids

Returns:
    value of loss function on batch


def elmo_export(self, epoch: Optional[int] = None) -> None:

if hasattr(self, '<STR_LIT>'):<EOL><INDENT>self.sess.close()<EOL><DEDENT>path = self.save_path<EOL>if epoch:<EOL><INDENT>from_path = path.parent / self.epoch_save_path / str(epoch) / path.parts[-<NUM_LIT:1>]<EOL>weights_to_path = path.parent / self.dumps_save_path / f'<STR_LIT>'<EOL>tf_hub_to_path = path.parent / self.tf_hub_save_path / f'<STR_LIT>'<EOL>from_path.resolve()<EOL>weights_to_path.resolve()<EOL>tf_hub_to_path.resolve()<EOL>log.info(f'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>from_path = path<EOL>weights_to_path = path.parent / self.dumps_save_path / '<STR_LIT>'<EOL>tf_hub_to_path = path.parent / self.tf_hub_save_path / '<STR_LIT>'<EOL><DEDENT>weights_to_path.parent.mkdir(parents=True, exist_ok=True)<EOL>tf_hub_to_path.parent.mkdir(parents=True, exist_ok=True)<EOL>if tf.train.checkpoint_exists(str(from_path)):<EOL><INDENT>dump_weights(from_path.parent, weights_to_path, self.permanent_options)<EOL>options = copy.deepcopy(self.permanent_options)<EOL>options['<STR_LIT>']['<STR_LIT>'] = <NUM_LIT><EOL>export2hub(weights_to_path, tf_hub_to_path, options)<EOL><DEDENT>

Dump the trained weights from a model to a HDF5 file and export a TF-Hub module.


if hasattr(self, '<STR_LIT>'):<EOL><INDENT>for k in list(self.sess.graph.get_all_collection_keys()):<EOL><INDENT>self.sess.graph.clear_collection(k)<EOL><DEDENT><DEDENT>super().destroy()<EOL>

Delete model from memory

Returns:
    None


def summary_gradient_updates(grads, opt, lr):

<EOL>vars_grads = {}<EOL>for v in tf.trainable_variables():<EOL><INDENT>vars_grads[v.name] = [v, None, None]<EOL><DEDENT>for g, v in grads:<EOL><INDENT>vars_grads[v.name][<NUM_LIT:1>] = g<EOL>vars_grads[v.name][<NUM_LIT:2>] = opt.get_slot(v, '<STR_LIT>')<EOL><DEDENT>ret = []<EOL>for vname, (v, g, a) in vars_grads.items():<EOL><INDENT>if g is None:<EOL><INDENT>continue<EOL><DEDENT>if isinstance(g, tf.IndexedSlices):<EOL><INDENT>updates = lr * g.values<EOL>if a is not None:<EOL><INDENT>updates /= tf.sqrt(tf.gather(a, g.indices))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>updates = lr * g<EOL>if a is not None:<EOL><INDENT>updates /= tf.sqrt(a)<EOL><DEDENT><DEDENT>values_norm = tf.sqrt(tf.reduce_sum(v * v)) + <NUM_LIT><EOL>updates_norm = tf.sqrt(tf.reduce_sum(updates * updates))<EOL>ret.append(tf.summary.scalar('<STR_LIT>' + vname.replace("<STR_LIT::>", "<STR_LIT:_>"), updates_norm / values_norm))<EOL><DEDENT>return ret<EOL>

get summary ops for the magnitude of gradient updates


unique_indices, new_index_positions = tf.unique(indices)<EOL>summed_values = tf.unsorted_segment_sum(values,<EOL>new_index_positions,<EOL>tf.shape(unique_indices)[<NUM_LIT:0>])<EOL>return (summed_values, unique_indices)<EOL>

Sums `values` associated with any non-unique `indices`.
    Args:
      values: A `Tensor` with rank >= 1.
      indices: A one-dimensional integer `Tensor`, indexing into the first
      dimension of `values` (as in an IndexedSlices object).
    Returns:
      A tuple of (`summed_values`, `unique_indices`) where `unique_indices` is a
      de-duplicated version of `indices` and `summed_values` contains the sum of
      `values` slices associated with each unique index.


def dump_weights(tf_save_dir, outfile, options):

def _get_outname(tf_name):<EOL><INDENT>outname = re.sub('<STR_LIT>', '<STR_LIT>', tf_name)<EOL>outname = outname.lstrip('<STR_LIT>')<EOL>outname = re.sub('<STR_LIT>', '<STR_LIT>', outname)<EOL>outname = re.sub('<STR_LIT>', '<STR_LIT>', outname)<EOL>outname = re.sub('<STR_LIT>', '<STR_LIT>', outname)<EOL>outname = re.sub('<STR_LIT>', '<STR_LIT>', outname)<EOL>if '<STR_LIT>' in outname:<EOL><INDENT>if '<STR_LIT>' in outname:<EOL><INDENT>outname = re.sub('<STR_LIT>', '<STR_LIT>', outname)<EOL><DEDENT>else:<EOL><INDENT>outname = re.sub('<STR_LIT>', '<STR_LIT>', outname)<EOL>outname = re.sub('<STR_LIT>', '<STR_LIT>', outname)<EOL><DEDENT><DEDENT>return outname<EOL><DEDENT>ckpt_file = tf.train.latest_checkpoint(tf_save_dir)<EOL>config = tf.ConfigProto(allow_soft_placement=True)<EOL>with tf.Graph().as_default():<EOL><INDENT>with tf.Session(config=config) as sess:<EOL><INDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>LanguageModel(options, False)  <EOL>loader = tf.train.Saver()<EOL>loader.restore(sess, ckpt_file)<EOL><DEDENT>with h5py.File(outfile, '<STR_LIT:w>') as fout:<EOL><INDENT>for v in tf.trainable_variables():<EOL><INDENT>if v.name.find('<STR_LIT>') >= <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>outname = _get_outname(v.name)<EOL>shape = v.get_shape().as_list()<EOL>dset = fout.create_dataset(outname, shape, dtype='<STR_LIT>')<EOL>values = sess.run([v])[<NUM_LIT:0>]<EOL>dset[...] = values<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Dump the trained weights from a model to a HDF5 file.


batch_size = self.options['<STR_LIT>']<EOL>unroll_steps = self.options['<STR_LIT>']<EOL>projection_dim = self.options['<STR_LIT>']['<STR_LIT>']<EOL>cnn_options = self.options['<STR_LIT>']<EOL>filters = cnn_options['<STR_LIT>']<EOL>n_filters = sum(f[<NUM_LIT:1>] for f in filters)<EOL>max_chars = cnn_options['<STR_LIT>']<EOL>char_embed_dim = cnn_options['<STR_LIT>']['<STR_LIT>']<EOL>n_chars = cnn_options['<STR_LIT>']<EOL>if n_chars != <NUM_LIT>:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if cnn_options['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>activation = tf.nn.tanh<EOL><DEDENT>elif cnn_options['<STR_LIT>'] == '<STR_LIT:relu>':<EOL><INDENT>activation = tf.nn.relu<EOL><DEDENT>self.tokens_characters = tf.placeholder(DTYPE_INT,<EOL>shape=(batch_size, unroll_steps, max_chars),<EOL>name='<STR_LIT>')<EOL>with tf.device("<STR_LIT>"):<EOL><INDENT>self.embedding_weights = tf.get_variable("<STR_LIT>", [n_chars, char_embed_dim],<EOL>dtype=DTYPE,<EOL>initializer=tf.random_uniform_initializer(-<NUM_LIT:1.0>, <NUM_LIT:1.0>))<EOL>self.char_embedding = tf.nn.embedding_lookup(self.embedding_weights,<EOL>self.tokens_characters)<EOL>if self.bidirectional:<EOL><INDENT>self.tokens_characters_reverse = tf.placeholder(DTYPE_INT,<EOL>shape=(batch_size, unroll_steps, max_chars),<EOL>name='<STR_LIT>')<EOL>self.char_embedding_reverse = tf.nn.embedding_lookup(<EOL>self.embedding_weights, self.tokens_characters_reverse)<EOL><DEDENT><DEDENT>def make_convolutions(inp, reuse):<EOL><INDENT>with tf.variable_scope('<STR_LIT>', reuse=reuse):<EOL><INDENT>convolutions = []<EOL>for i, (width, num) in enumerate(filters):<EOL><INDENT>if cnn_options['<STR_LIT>'] == '<STR_LIT:relu>':<EOL><INDENT>w_init = tf.random_uniform_initializer(<EOL>minval=-<NUM_LIT>, maxval=<NUM_LIT>)<EOL><DEDENT>elif cnn_options['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>w_init = tf.random_normal_initializer(<EOL>mean=<NUM_LIT:0.0>,<EOL>stddev=np.sqrt(<NUM_LIT:1.0> / (width * char_embed_dim))<EOL>)<EOL><DEDENT>w = tf.get_variable(<EOL>"<STR_LIT>" % i,<EOL>[<NUM_LIT:1>, width, char_embed_dim, num],<EOL>initializer=w_init,<EOL>dtype=DTYPE)<EOL>b = tf.get_variable(<EOL>"<STR_LIT>" % i, [num], dtype=DTYPE,<EOL>initializer=tf.constant_initializer(<NUM_LIT:0.0>))<EOL>conv = tf.nn.conv2d(inp, w,<EOL>strides=[<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>],<EOL>padding="<STR_LIT>") + b<EOL>conv = tf.nn.max_pool(conv, [<NUM_LIT:1>, <NUM_LIT:1>, max_chars - width + <NUM_LIT:1>, <NUM_LIT:1>],<EOL>[<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>], '<STR_LIT>')<EOL>conv = activation(conv)<EOL>conv = tf.squeeze(conv, squeeze_dims=[<NUM_LIT:2>])<EOL>convolutions.append(conv)<EOL><DEDENT><DEDENT>return tf.concat(convolutions, <NUM_LIT:2>)<EOL><DEDENT>reuse = tf.get_variable_scope().reuse<EOL>embedding = make_convolutions(self.char_embedding, reuse)<EOL>self.token_embedding_layers = [embedding]<EOL>if self.bidirectional:<EOL><INDENT>embedding_reverse = make_convolutions(<EOL>self.char_embedding_reverse, True)<EOL><DEDENT>n_highway = cnn_options.get('<STR_LIT>')<EOL>use_highway = n_highway is not None and n_highway > <NUM_LIT:0><EOL>use_proj = n_filters != projection_dim<EOL>if use_highway or use_proj:<EOL><INDENT>embedding = tf.reshape(embedding, [-<NUM_LIT:1>, n_filters])<EOL>if self.bidirectional:<EOL><INDENT>embedding_reverse = tf.reshape(embedding_reverse,<EOL>[-<NUM_LIT:1>, n_filters])<EOL><DEDENT><DEDENT>if use_proj:<EOL><INDENT>assert n_filters > projection_dim<EOL>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>W_proj_cnn = tf.get_variable(<EOL>"<STR_LIT>", [n_filters, projection_dim],<EOL>initializer=tf.random_normal_initializer(<EOL>mean=<NUM_LIT:0.0>, stddev=np.sqrt(<NUM_LIT:1.0> / n_filters)),<EOL>dtype=DTYPE)<EOL>b_proj_cnn = tf.get_variable(<EOL>"<STR_LIT>", [projection_dim],<EOL>initializer=tf.constant_initializer(<NUM_LIT:0.0>),<EOL>dtype=DTYPE)<EOL><DEDENT><DEDENT>def high(x, ww_carry, bb_carry, ww_tr, bb_tr):<EOL><INDENT>carry_gate = tf.nn.sigmoid(tf.matmul(x, ww_carry) + bb_carry)<EOL>transform_gate = tf.nn.relu(tf.matmul(x, ww_tr) + bb_tr)<EOL>return carry_gate * transform_gate + (<NUM_LIT:1.0> - carry_gate) * x<EOL><DEDENT>if use_highway:<EOL><INDENT>highway_dim = n_filters<EOL>for i in range(n_highway):<EOL><INDENT>with tf.variable_scope('<STR_LIT>' % i):<EOL><INDENT>W_carry = tf.get_variable(<EOL>'<STR_LIT>', [highway_dim, highway_dim],<EOL>initializer=tf.random_normal_initializer(<EOL>mean=<NUM_LIT:0.0>, stddev=np.sqrt(<NUM_LIT:1.0> / highway_dim)),<EOL>dtype=DTYPE)<EOL>b_carry = tf.get_variable(<EOL>'<STR_LIT>', [highway_dim],<EOL>initializer=tf.constant_initializer(-<NUM_LIT>),<EOL>dtype=DTYPE)<EOL>W_transform = tf.get_variable(<EOL>'<STR_LIT>', [highway_dim, highway_dim],<EOL>initializer=tf.random_normal_initializer(<EOL>mean=<NUM_LIT:0.0>, stddev=np.sqrt(<NUM_LIT:1.0> / highway_dim)),<EOL>dtype=DTYPE)<EOL>b_transform = tf.get_variable(<EOL>'<STR_LIT>', [highway_dim],<EOL>initializer=tf.constant_initializer(<NUM_LIT:0.0>),<EOL>dtype=DTYPE)<EOL><DEDENT>embedding = high(embedding, W_carry, b_carry,<EOL>W_transform, b_transform)<EOL>if self.bidirectional:<EOL><INDENT>embedding_reverse = high(embedding_reverse,<EOL>W_carry, b_carry,<EOL>W_transform, b_transform)<EOL><DEDENT>self.token_embedding_layers.append(tf.reshape(embedding,<EOL>[batch_size, unroll_steps, highway_dim]))<EOL><DEDENT><DEDENT>if use_proj:<EOL><INDENT>embedding = tf.matmul(embedding, W_proj_cnn) + b_proj_cnn<EOL>if self.bidirectional:<EOL><INDENT>embedding_reverse = tf.matmul(embedding_reverse, W_proj_cnn)+ b_proj_cnn<EOL><DEDENT>self.token_embedding_layers.append(<EOL>tf.reshape(embedding, [batch_size, unroll_steps, projection_dim])<EOL>)<EOL><DEDENT>if use_highway or use_proj:<EOL><INDENT>shp = [batch_size, unroll_steps, projection_dim]<EOL>embedding = tf.reshape(embedding, shp)<EOL>if self.bidirectional:<EOL><INDENT>embedding_reverse = tf.reshape(embedding_reverse, shp)<EOL><DEDENT><DEDENT>self.embedding = embedding<EOL>if self.bidirectional:<EOL><INDENT>self.embedding_reverse = embedding_reverse<EOL><DEDENT>

options contains key 'char_cnn': {

'n_characters': 262,

# includes the start / end characters
'max_characters_per_token': 50,

'filters': [
    [1, 32],
    [2, 32],
    [3, 64],
    [4, 128],
    [5, 256],
    [6, 512],
    [7, 512]
],
'activation': 'tanh',

# for the character embedding
'embedding': {'dim': 16}

# for highway layers
# if omitted, then no highway layers
'n_highway': 2,
}


batch_size = self.options['<STR_LIT>']<EOL>unroll_steps = self.options['<STR_LIT>']<EOL>n_tokens_vocab = self.options['<STR_LIT>']<EOL>def _get_next_token_placeholders(suffix):<EOL><INDENT>name = '<STR_LIT>' + suffix<EOL>id_placeholder = tf.placeholder(DTYPE_INT,<EOL>shape=(batch_size, unroll_steps),<EOL>name=name)<EOL>return id_placeholder<EOL><DEDENT>self.next_token_id = _get_next_token_placeholders('<STR_LIT>')<EOL>if self.bidirectional:<EOL><INDENT>self.next_token_id_reverse = _get_next_token_placeholders(<EOL>'<STR_LIT>')<EOL><DEDENT>softmax_dim = self.options['<STR_LIT>']['<STR_LIT>']<EOL>if self.share_embedding_softmax:<EOL><INDENT>self.softmax_W = self.embedding_weights<EOL><DEDENT>with tf.variable_scope('<STR_LIT>'), tf.device('<STR_LIT>'):<EOL><INDENT>softmax_init = tf.random_normal_initializer(<NUM_LIT:0.0>, <NUM_LIT:1.0> / np.sqrt(softmax_dim))<EOL>if not self.share_embedding_softmax:<EOL><INDENT>self.softmax_W = tf.get_variable(<EOL>'<STR_LIT>', [n_tokens_vocab, softmax_dim],<EOL>dtype=DTYPE,<EOL>initializer=softmax_init<EOL>)<EOL><DEDENT>self.softmax_b = tf.get_variable(<EOL>'<STR_LIT:b>', [n_tokens_vocab],<EOL>dtype=DTYPE,<EOL>initializer=tf.constant_initializer(<NUM_LIT:0.0>))<EOL><DEDENT>self.individual_train_losses = []<EOL>self.individual_eval_losses = []<EOL>if self.bidirectional:<EOL><INDENT>next_ids = [self.next_token_id, self.next_token_id_reverse]<EOL><DEDENT>else:<EOL><INDENT>next_ids = [self.next_token_id]<EOL><DEDENT>for id_placeholder, lstm_output_flat in zip(next_ids, lstm_outputs):<EOL><INDENT>next_token_id_flat = tf.reshape(id_placeholder, [-<NUM_LIT:1>, <NUM_LIT:1>])<EOL>with tf.control_dependencies([lstm_output_flat]):<EOL><INDENT>sampled_losses = tf.nn.sampled_softmax_loss(self.softmax_W, self.softmax_b,<EOL>next_token_id_flat, lstm_output_flat,<EOL>self.options['<STR_LIT>'],<EOL>self.options['<STR_LIT>'],<EOL>num_true=<NUM_LIT:1>)<EOL>output_scores = tf.matmul(<EOL>lstm_output_flat,<EOL>tf.transpose(self.softmax_W)<EOL>) + self.softmax_b<EOL>losses = tf.nn.sparse_softmax_cross_entropy_with_logits(<EOL>logits=output_scores,<EOL>labels=tf.squeeze(next_token_id_flat, squeeze_dims=[<NUM_LIT:1>])<EOL>)<EOL><DEDENT>sampled_losses = tf.reshape(sampled_losses, [self.options['<STR_LIT>'], -<NUM_LIT:1>])<EOL>losses = tf.reshape(losses, [self.options['<STR_LIT>'], -<NUM_LIT:1>])<EOL>self.individual_train_losses.append(tf.reduce_mean(sampled_losses, axis=<NUM_LIT:1>))<EOL>self.individual_eval_losses.append(tf.reduce_mean(losses, axis=<NUM_LIT:1>))<EOL><DEDENT>if self.bidirectional:<EOL><INDENT>self.total_train_loss = <NUM_LIT:0.5> * (self.individual_train_losses[<NUM_LIT:0>] + self.individual_train_losses[<NUM_LIT:1>])<EOL>self.total_eval_loss = <NUM_LIT:0.5> * (self.individual_eval_losses[<NUM_LIT:0>] + self.individual_eval_losses[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>self.total_train_loss = self.individual_train_losses[<NUM_LIT:0>]<EOL>self.total_eval_loss = self.individual_eval_losses[<NUM_LIT:0>]<EOL><DEDENT>

Create:
    self.total_loss: total loss op for training
    self.softmax_W, softmax_b: the softmax variables
    self.next_token_id / _reverse: placeholders for gold input


def module_fn():<EOL><INDENT>"""<STR_LIT>"""<EOL>_bos_id = <NUM_LIT><EOL>_eos_id = <NUM_LIT><EOL>_bow_id = <NUM_LIT><EOL>_eow_id = <NUM_LIT><EOL>_pad_id = <NUM_LIT><EOL>_max_word_length = <NUM_LIT:50><EOL>_parallel_iterations = <NUM_LIT:10><EOL>_max_batch_size = <NUM_LIT><EOL>id_dtype = tf.int32<EOL>id_nptype = np.int32<EOL>max_word_length = tf.constant(_max_word_length, dtype=id_dtype, name='<STR_LIT>')<EOL>version = tf.constant('<STR_LIT>', dtype=tf.string, name='<STR_LIT:version>')<EOL>def _make_bos_eos(c):<EOL><INDENT>r = np.zeros([_max_word_length], dtype=id_nptype)<EOL>r[:] = _pad_id<EOL>r[<NUM_LIT:0>] = _bow_id<EOL>r[<NUM_LIT:1>] = c<EOL>r[<NUM_LIT:2>] = _eow_id<EOL>return tf.constant(r, dtype=id_dtype)<EOL><DEDENT>bos_ids = _make_bos_eos(_bos_id)<EOL>eos_ids = _make_bos_eos(_eos_id)<EOL>def token2ids(token):<EOL><INDENT>with tf.name_scope("<STR_LIT>"):<EOL><INDENT>char_ids = tf.decode_raw(token, tf.uint8, name='<STR_LIT>')<EOL>char_ids = tf.cast(char_ids, tf.int32, name='<STR_LIT>')<EOL>char_ids = tf.strided_slice(char_ids, [<NUM_LIT:0>], [max_word_length - <NUM_LIT:2>],<EOL>[<NUM_LIT:1>], name='<STR_LIT>')<EOL>ids_num = tf.shape(char_ids)[<NUM_LIT:0>]<EOL>fill_ids_num = (_max_word_length - <NUM_LIT:2>) - ids_num<EOL>pads = tf.fill([fill_ids_num], _pad_id)<EOL>bow_token_eow_pads = tf.concat([[_bow_id], char_ids, [_eow_id], pads],<EOL><NUM_LIT:0>, name='<STR_LIT>')<EOL>return bow_token_eow_pads<EOL><DEDENT><DEDENT>def sentence_tagging_and_padding(sen_dim):<EOL><INDENT>with tf.name_scope("<STR_LIT>"):<EOL><INDENT>sen = sen_dim[<NUM_LIT:0>]<EOL>dim = sen_dim[<NUM_LIT:1>]<EOL>extra_dim = tf.shape(sen)[<NUM_LIT:0>] - dim<EOL>sen = tf.slice(sen, [<NUM_LIT:0>, <NUM_LIT:0>], [dim, max_word_length], name='<STR_LIT>')<EOL>bos_sen_eos = tf.concat([[bos_ids], sen, [eos_ids]], <NUM_LIT:0>, name='<STR_LIT>')<EOL>bos_sen_eos_plus_one = bos_sen_eos + <NUM_LIT:1><EOL>bos_sen_eos_pads = tf.pad(bos_sen_eos_plus_one, [[<NUM_LIT:0>, extra_dim], [<NUM_LIT:0>, <NUM_LIT:0>]],<EOL>"<STR_LIT>", name='<STR_LIT>')<EOL>return bos_sen_eos_pads<EOL><DEDENT><DEDENT>tokens = tf.placeholder(shape=(None, None), dtype=tf.string, name='<STR_LIT>')<EOL>sequence_len = tf.placeholder(shape=(None, ), dtype=tf.int32, name='<STR_LIT>')<EOL>tok_shape = tf.shape(tokens)<EOL>line_tokens = tf.reshape(tokens, shape=[-<NUM_LIT:1>], name='<STR_LIT>')<EOL>with tf.device('<STR_LIT>'):<EOL><INDENT>tok_ids = tf.map_fn(<EOL>token2ids,<EOL>line_tokens,<EOL>dtype=tf.int32, back_prop=False, parallel_iterations=_parallel_iterations,<EOL>name='<STR_LIT>')<EOL><DEDENT>tok_ids = tf.reshape(tok_ids, [tok_shape[<NUM_LIT:0>], tok_shape[<NUM_LIT:1>], -<NUM_LIT:1>], name='<STR_LIT>')<EOL>with tf.device('<STR_LIT>'):<EOL><INDENT>sen_ids = tf.map_fn(<EOL>sentence_tagging_and_padding,<EOL>(tok_ids, sequence_len),<EOL>dtype=tf.int32, back_prop=False, parallel_iterations=_parallel_iterations,<EOL>name='<STR_LIT>')<EOL><DEDENT>bilm = BidirectionalLanguageModel(options, str(weight_file),<EOL>max_batch_size=_max_batch_size)<EOL>embeddings_op = bilm(sen_ids)<EOL>elmo_output = weight_layers('<STR_LIT>', embeddings_op, l2_coef=<NUM_LIT:0.0>)<EOL>weighted_op = elmo_output['<STR_LIT>']<EOL>mean_op = elmo_output['<STR_LIT>']<EOL>word_emb = elmo_output['<STR_LIT>']<EOL>lstm_outputs1 = elmo_output['<STR_LIT>']<EOL>lstm_outputs2 = elmo_output['<STR_LIT>']<EOL>hub.add_signature("<STR_LIT>", {"<STR_LIT>": tokens, "<STR_LIT>": sequence_len},<EOL>{"<STR_LIT>": weighted_op,<EOL>"<STR_LIT:default>": mean_op,<EOL>"<STR_LIT>": word_emb,<EOL>"<STR_LIT>": lstm_outputs1,<EOL>"<STR_LIT>": lstm_outputs2,<EOL>"<STR_LIT:version>": version})<EOL>def_strings = tf.placeholder(shape=(None), dtype=tf.string)<EOL>def_tokens_sparse = tf.string_split(def_strings)<EOL>def_tokens_dense = tf.sparse_to_dense(sparse_indices=def_tokens_sparse.indices,<EOL>output_shape=def_tokens_sparse.dense_shape,<EOL>sparse_values=def_tokens_sparse.values,<EOL>default_value='<STR_LIT>'<EOL>)<EOL>def_mask = tf.not_equal(def_tokens_dense, '<STR_LIT>')<EOL>def_int_mask = tf.cast(def_mask, dtype=tf.int32)<EOL>def_sequence_len = tf.reduce_sum(def_int_mask, axis=-<NUM_LIT:1>)<EOL>def_tok_shape = tf.shape(def_tokens_dense)<EOL>def_line_tokens = tf.reshape(def_tokens_dense, shape=[-<NUM_LIT:1>], name='<STR_LIT>')<EOL>with tf.device('<STR_LIT>'):<EOL><INDENT>def_tok_ids = tf.map_fn(<EOL>token2ids,<EOL>def_line_tokens,<EOL>dtype=tf.int32, back_prop=False, parallel_iterations=_parallel_iterations,<EOL>name='<STR_LIT>')<EOL><DEDENT>def_tok_ids = tf.reshape(def_tok_ids, [def_tok_shape[<NUM_LIT:0>], def_tok_shape[<NUM_LIT:1>], -<NUM_LIT:1>], name='<STR_LIT>')<EOL>with tf.device('<STR_LIT>'):<EOL><INDENT>def_sen_ids = tf.map_fn(<EOL>sentence_tagging_and_padding,<EOL>(def_tok_ids, def_sequence_len),<EOL>dtype=tf.int32, back_prop=False, parallel_iterations=_parallel_iterations,<EOL>name='<STR_LIT>')<EOL><DEDENT>def_embeddings_op = bilm(def_sen_ids)<EOL>def_elmo_output = weight_layers('<STR_LIT>', def_embeddings_op, l2_coef=<NUM_LIT:0.0>, reuse=True)<EOL>def_weighted_op = def_elmo_output['<STR_LIT>']<EOL>def_mean_op = def_elmo_output['<STR_LIT>']<EOL>def_word_emb = def_elmo_output['<STR_LIT>']<EOL>def_lstm_outputs1 = def_elmo_output['<STR_LIT>']<EOL>def_lstm_outputs2 = def_elmo_output['<STR_LIT>']<EOL>hub.add_signature("<STR_LIT:default>", {"<STR_LIT>": def_strings},<EOL>{"<STR_LIT>": def_weighted_op,<EOL>"<STR_LIT:default>": def_mean_op,<EOL>"<STR_LIT>": def_word_emb,<EOL>"<STR_LIT>": def_lstm_outputs1,<EOL>"<STR_LIT>": def_lstm_outputs2,<EOL>"<STR_LIT:version>": version})<EOL><DEDENT>return hub.create_module_spec(module_fn)<EOL>

Makes a module spec.

    Args:
      options: LM hyperparameters.
      weight_file: location of the hdf5 file with LM weights.

    Returns:
      A module spec object used for constructing a TF-Hub module.


def export2hub(weight_file, hub_dir, options):

spec = make_module_spec(options, str(weight_file))<EOL>try:<EOL><INDENT>with tf.Graph().as_default():<EOL><INDENT>module = hub.Module(spec)<EOL>with tf.Session() as sess:<EOL><INDENT>sess.run(tf.global_variables_initializer())<EOL>if hub_dir.exists():<EOL><INDENT>shutil.rmtree(hub_dir)<EOL><DEDENT>module.export(str(hub_dir), sess)<EOL><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>pass<EOL><DEDENT>


def _pretrained_initializer(varname, weight_file, embedding_weight_file=None):

weight_name_map = {}<EOL>for i in range(<NUM_LIT:2>):<EOL><INDENT>for j in range(<NUM_LIT:8>):  <EOL><INDENT>root = '<STR_LIT>'.format(i, j)<EOL>weight_name_map[root + '<STR_LIT>'] =root + '<STR_LIT>'<EOL>weight_name_map[root + '<STR_LIT>'] =root + '<STR_LIT>'<EOL>weight_name_map[root + '<STR_LIT>'] =root + '<STR_LIT>'<EOL><DEDENT><DEDENT>varname_in_file = varname[<NUM_LIT:5>:]<EOL>if varname_in_file.startswith('<STR_LIT>'):<EOL><INDENT>varname_in_file = weight_name_map[varname_in_file]<EOL><DEDENT>if varname_in_file == '<STR_LIT>':<EOL><INDENT>with h5py.File(embedding_weight_file, '<STR_LIT:r>') as fin:<EOL><INDENT>embed_weights = fin[varname_in_file][...]<EOL>weights = np.zeros(<EOL>(embed_weights.shape[<NUM_LIT:0>] + <NUM_LIT:1>, embed_weights.shape[<NUM_LIT:1>]),<EOL>dtype=DTYPE<EOL>)<EOL>weights[<NUM_LIT:1>:, :] = embed_weights<EOL><DEDENT><DEDENT>else:<EOL><INDENT>with h5py.File(weight_file, '<STR_LIT:r>') as fin:<EOL><INDENT>if varname_in_file == '<STR_LIT>':<EOL><INDENT>char_embed_weights = fin[varname_in_file][...]<EOL>weights = np.zeros(<EOL>(char_embed_weights.shape[<NUM_LIT:0>] + <NUM_LIT:1>,<EOL>char_embed_weights.shape[<NUM_LIT:1>]),<EOL>dtype=DTYPE<EOL>)<EOL>weights[<NUM_LIT:1>:, :] = char_embed_weights<EOL><DEDENT>else:<EOL><INDENT>weights = fin[varname_in_file][...]<EOL><DEDENT><DEDENT><DEDENT>def ret(shape, **kwargs):<EOL><INDENT>if list(shape) != list(weights.shape):<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>".format(<EOL>varname_in_file, shape, weights.shape)<EOL>)<EOL><DEDENT>return weights<EOL><DEDENT>return ret<EOL>

We'll stub out all the initializers in the pretrained LM with
a function that loads the weights from the file