Project the tokenized prediction back to the original text .

def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False):
    """Project the tokenized prediction back to the original text."""

    # When we created the data, we kept track of the alignment between original
    # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So
    # now `orig_text` contains the span of our original text corresponding to the
    # span that we predicted.
    #
    # However, `orig_text` may contain extra characters that we don't want in
    # our prediction.
    #
    # For example, let's say:
    #   pred_text = steve smith
    #   orig_text = Steve Smith's
    #
    # We don't want to return `orig_text` because it contains the extra "'s".
    #
    # We don't want to return `pred_text` because it's already been normalized
    # (the SQuAD eval script also does punctuation stripping/lower casing but
    # our tokenizer does additional normalization like stripping accent
    # characters).
    #
    # What we really want to return is "Steve Smith".
    #
    # Therefore, we have to apply a semi-complicated alignment heuristic between
    # `pred_text` and `orig_text` to get a character-to-character alignment. This
    # can fail in certain cases in which case we just return `orig_text`.

    def _strip_spaces(text):
        ns_chars = []
        ns_to_s_map = collections.OrderedDict()
        for (i, c) in enumerate(text):
            if c == " ":
                continue
            ns_to_s_map[len(ns_chars)] = i
            ns_chars.append(c)
        ns_text = "".join(ns_chars)
        return (ns_text, ns_to_s_map)

    # We first tokenize `orig_text`, strip whitespace from the result
    # and `pred_text`, and check if they are the same length. If they are
    # NOT the same length, the heuristic has failed. If they are the same
    # length, we assume the characters are one-to-one aligned.
    tokenizer = BasicTokenizer(do_lower_case=do_lower_case)

    tok_text = " ".join(tokenizer.tokenize(orig_text))

    start_position = tok_text.find(pred_text)
    if start_position == -1:
        if verbose_logging:
            logger.info(
                "Unable to find text: '%s' in '%s'" % (pred_text, orig_text))
        return orig_text
    end_position = start_position + len(pred_text) - 1

    (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text)
    (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text)

    if len(orig_ns_text) != len(tok_ns_text):
        if verbose_logging:
            logger.info("Length not equal after stripping spaces: '%s' vs '%s'",
                        orig_ns_text, tok_ns_text)
        return orig_text

    # We then project the characters in `pred_text` back to `orig_text` using
    # the character-to-character alignment.
    tok_s_to_ns_map = {}
    for (i, tok_index) in tok_ns_to_s_map.items():
        tok_s_to_ns_map[tok_index] = i

    orig_start_position = None
    if start_position in tok_s_to_ns_map:
        ns_start_position = tok_s_to_ns_map[start_position]
        if ns_start_position in orig_ns_to_s_map:
            orig_start_position = orig_ns_to_s_map[ns_start_position]

    if orig_start_position is None:
        if verbose_logging:
            logger.info("Couldn't map start position")
        return orig_text

    orig_end_position = None
    if end_position in tok_s_to_ns_map:
        ns_end_position = tok_s_to_ns_map[end_position]
        if ns_end_position in orig_ns_to_s_map:
            orig_end_position = orig_ns_to_s_map[ns_end_position]

    if orig_end_position is None:
        if verbose_logging:
            logger.info("Couldn't map end position")
        return orig_text

    output_text = orig_text[orig_start_position:(orig_end_position + 1)]
    return output_text


def _get_best_indexes(logits, n_best_size):
    """Get the n-best logits from a list."""
    index_and_score = sorted(
        enumerate(logits), key=lambda x: x[1], reverse=True)

    best_indexes = []
    for i in range(len(index_and_score)):
        if i >= n_best_size:
            break
        best_indexes.append(index_and_score[i][0])
    return best_indexes

def _get_best_indexes ( logits , n_best_size ) : index_and_score = sorted ( enumerate ( logits ) , key = lambda x : x [ 1 ] , reverse = True ) best_indexes = [ ] for i in range ( len ( index_and_score ) ) : if i >= n_best_size : break best_indexes . append ( index_and_score [ i ] [ 0 ] ) return best_indexes


Compute softmax probability over raw logits .

def _compute_softmax(scores):
    """Compute softmax probability over raw logits."""
    if not scores:
        return []

    max_score = None
    for score in scores:
        if max_score is None or score > max_score:
            max_score = score

    exp_scores = []
    total_sum = 0.0
    for score in scores:
        x = math.exp(score - max_score)
        exp_scores.append(x)
        total_sum += x

    probs = []
    for score in exp_scores:
        probs.append(score / total_sum)
    return probs

def _compute_softmax ( scores ) : if not scores : return [ ] max_score = None for score in scores : if max_score is None or score > max_score : max_score = score exp_scores = [ ] total_sum = 0.0 for score in scores : x = math . exp ( score - max_score ) exp_scores . append ( x ) total_sum += x probs = [ ] for score in exp_scores : probs . append ( score / total_sum ) return probs


Construct a layernorm module in the TF style ( epsilon inside the square root ) .

def __init__(self, config, variance_epsilon=1e-12):
        """Construct a layernorm module in the TF style (epsilon inside the square root).
        """
        super(BERTLayerNorm, self).__init__()
        self.gamma = nn.Parameter(torch.ones(config.hidden_size))
        self.beta = nn.Parameter(torch.zeros(config.hidden_size))
        self.variance_epsilon = variance_epsilon

def __init__ ( self , config , variance_epsilon = 1e-12 ) : super ( BERTLayerNorm , self ) . __init__ ( ) self . gamma = nn . Parameter ( torch . ones ( config . hidden_size ) ) self . beta = nn . Parameter ( torch . zeros ( config . hidden_size ) ) self . variance_epsilon = variance_epsilon


Find the mximum size of the initial context ( links are not included )

def create_examples(jsn, graph_retriever_config):

    task = graph_retriever_config.task

    examples = []

    '''
    Find the mximum size of the initial context (links are not included)
    '''
    graph_retriever_config.max_context_size = 0

    for data in jsn:

        guid = data['q_id']
        question = data['question']
        context = data['context'] # {context title: paragraph}
        all_linked_paras_dic = {} # {context title: {linked title: paragraph}}

        '''
        Use TagMe-based context at test time.
        '''
        if graph_retriever_config.tagme:
            assert 'tagged_context' in data

            '''
            Reformat "tagged_context" if needed (c.f. the "context" case above)
            '''
            if type(data['tagged_context']) == list:
                tagged_context = {c[0]: c[1] for c in data['tagged_context']}
                data['tagged_context'] = tagged_context

            '''
            Append valid paragraphs from "tagged_context" to "context"
            '''
            for tagged_title in data['tagged_context']:
                tagged_text = data['tagged_context'][tagged_title]
                if tagged_title not in context and tagged_title is not None and tagged_title.strip() != '' and tagged_text is not None and tagged_text.strip() != '':
                    context[tagged_title] = tagged_text

        '''
        Clean "context" by removing invalid paragraphs
        '''
        removed_keys = []
        for title in context:
            if title is None or title.strip() == '' or context[title] is None or context[title].strip() == '':
                removed_keys.append(title)
        for key in removed_keys:
            context.pop(key)

        all_paras = {}
        for title in context:
            all_paras[title] = context[title]

        if graph_retriever_config.expand_links:
            expand_links(context, all_linked_paras_dic, all_paras)

        graph_retriever_config.max_context_size = max(graph_retriever_config.max_context_size, len(context))

        examples.append(InputExample(guid = guid,
                                     q = question,
                                     c = context,
                                     para_dic = all_linked_paras_dic,
                                     s_g = None, r_g = None, all_r_g = None,
                                     all_paras = all_paras))

    return examples


Loads a data file into a list of InputBatch s .

def convert_examples_to_features(examples, max_seq_length, max_para_num, graph_retriever_config, tokenizer):
    """Loads a data file into a list of `InputBatch`s."""

    max_para_num = graph_retriever_config.max_context_size
    graph_retriever_config.max_para_num = max(graph_retriever_config.max_para_num, max_para_num)

    max_steps = graph_retriever_config.max_select_num

    DUMMY = [0] * max_seq_length
    features = []

    for (ex_index, example) in enumerate(examples):
        tokens_q = tokenize_question(example.question, tokenizer)

        title2index = {}
        input_ids = []
        input_masks = []
        segment_ids = []

        titles_list = list(example.context.keys())
        for p in titles_list:

            if len(input_ids) == max_para_num:
                break

            if p in title2index:
                continue

            title2index[p] = len(title2index)
            example.title_order.append(p)
            p = example.context[p]

            input_ids_, input_masks_, segment_ids_ = tokenize_paragraph(p, tokens_q, max_seq_length, tokenizer)
            input_ids.append(input_ids_)
            input_masks.append(input_masks_)
            segment_ids.append(segment_ids_)

        num_paragraphs_no_links = len(input_ids)

        assert len(input_ids) <= max_para_num

        num_paragraphs = len(input_ids)

        output_masks = [([1.0] * len(input_ids) + [0.0] * (max_para_num - len(input_ids) + 1)) for _ in range(max_para_num + 2)]

        assert len(example.context) == num_paragraphs_no_links
        for i in range(len(output_masks[0])):
            if i >= num_paragraphs_no_links:
                output_masks[0][i] = 0.0

        for i in range(len(input_ids)):
            output_masks[i+1][i] = 0.0

        padding = [DUMMY] * (max_para_num - len(input_ids))
        input_ids += padding
        input_masks += padding
        segment_ids += padding

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_masks=input_masks,
                              segment_ids=segment_ids,
                              output_masks = output_masks,
                              num_paragraphs = num_paragraphs,
                              num_steps = -1,
                              ex_index = ex_index))

    return features

def convert_examples_to_features ( examples , max_seq_length , max_para_num , graph_retriever_config , tokenizer ) : max_para_num = graph_retriever_config . max_context_size graph_retriever_config . max_para_num = max ( graph_retriever_config . max_para_num , max_para_num ) max_steps = graph_retriever_config . max_select_num DUMMY = [ 0 ] * max_seq_length features = [ ] for ( ex_index , example ) in enumerate ( examples ) : tokens_q = tokenize_question ( example . question , tokenizer ) title2index = { } input_ids = [ ] input_masks = [ ] segment_ids = [ ] titles_list = list ( example . context . keys ( ) ) for p in titles_list : if len ( input_ids ) == max_para_num : break if p in title2index : continue title2index [ p ] = len ( title2index ) example . title_order . append ( p ) p = example . context [ p ] input_ids_ , input_masks_ , segment_ids_ = tokenize_paragraph ( p , tokens_q , max_seq_length , tokenizer ) input_ids . append ( input_ids_ ) input_masks . append ( input_masks_ ) segment_ids . append ( segment_ids_ ) num_paragraphs_no_links = len ( input_ids ) assert len ( input_ids ) <= max_para_num num_paragraphs = len ( input_ids ) output_masks = [ ( [ 1.0 ] * len ( input_ids ) + [ 0.0 ] * ( max_para_num - len ( input_ids ) + 1 ) ) for _ in range ( max_para_num + 2 ) ] assert len ( example . context ) == num_paragraphs_no_links for i in range ( len ( output_masks [ 0 ] ) ) : if i >= num_paragraphs_no_links : output_masks [ 0 ] [ i ] = 0.0 for i in range ( len ( input_ids ) ) : output_masks [ i + 1 ] [ i ] = 0.0 padding = [ DUMMY ] * ( max_para_num - len ( input_ids ) ) input_ids += padding input_masks += padding segment_ids += padding features . append ( InputFeatures ( input_ids = input_ids , input_masks = input_masks , segment_ids = segment_ids , output_masks = output_masks , num_paragraphs = num_paragraphs , num_steps = - 1 , ex_index = ex_index ) ) return features


Retrieve articles with their corresponding hyperlinked titles . Due to efficiency we sample top k articles and then sample top l paragraphs from each article . ( so eventually we get k * l paragraphs with tfidf - based pruning . ) We also store the hyperlinked titles for each paragraph .

def get_article_tfidf_with_hyperlinked_titles(self, q_id,question, args):
        """
        Retrieve articles with their corresponding hyperlinked titles.
        Due to efficiency, we sample top k articles, and then sample top l paragraphs from each article.
        (so, eventually we get k*l paragraphs with tfidf-based pruning.)
        We also store the hyperlinked titles for each paragraph.
        """

        tfidf_limit, pruning_l, prune_after_agg = args.tfidf_limit, args.pruning_l, args.prune_after_agg
        doc_names, _ = self.ranker.closest_docs(question, k=tfidf_limit)
        context, hyper_linked_titles = self.load_sampled_para_text_and_linked_titles(
            doc_names, question, pruning_l, prune_after_agg)

        if args.tagme is True and args.tagme_api_key is not None:
            # if add TagMe
            tagged_context = self.load_sampled_tagged_para_text(
                question, pruning_l, args.tagme_api_key)

            return [{"question": question,
                     "context": context,
                     "tagged_context": tagged_context,
                     "all_linked_para_title_dic": hyper_linked_titles,
                     "q_id": q_id}]
        else:
            return [{"question": question,
                    "context": context,
                    "all_linked_para_title_dic": hyper_linked_titles,
                    "q_id": q_id}]