Upload 15 files

.gitattributes

@@ -32,3 +32,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+outputs/bm25_stage1/bm25_index filter=lfs diff=lfs merge=lfs -text
+outputs/bm25_stage1/tfidf filter=lfs diff=lfs merge=lfs -text
+processed/wikipedia_chungta_cleaned.csv filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,59 @@
+import json
+from text_utils import *
+import pandas as pd
+from qa_model import *
+from bm25_utils import *
+from pairwise_model import *
+
+df_wiki_windows = pd.read_csv("./processed/wikipedia_chungta_cleaned.csv")
+df_wiki = pd.read_csv("./processed/wikipedia_chungta_short.csv")
+df_wiki.title = df_wiki.title.apply(str)
+
+
+
+entity_dict = json.load(open("./processed/entities.json"))
+new_dict = dict()
+for key, val in entity_dict.items():
+    val = val.replace("wiki/", "").replace("_", " ")
+    entity_dict[key] = val
+    key = preprocess(key)
+    new_dict[key.lower()] = val
+entity_dict.update(new_dict)
+title2idx = dict([(x.strip(), y) for x, y in zip(df_wiki.title, df_wiki.index.values)])
+
+
+qa_model = QAEnsembleModel_modify("letrunglinh/qa_pnc", entity_dict)
+pairwise_model_stage1 = PairwiseModel_modify("nguyenvulebinh/vi-mrc-base")
+
+bm25_model_stage1 = BM25Gensim("./outputs/bm25_stage1/", entity_dict, title2idx)
+
+
+def get_answer_e2e(question):
+    #Bm25 retrieval for top200 candidates
+    query = preprocess(question).lower()
+    top_n, bm25_scores = bm25_model_stage1.get_topk_stage1(query, topk=200)
+    titles = [preprocess(df_wiki_windows.title.values[i]) for i in top_n]
+    pre_texts = [preprocess(df_wiki_windows.text.values[i]) for i in top_n]
+
+    #Reranking with pairwise model for top10
+    question = preprocess(question)
+    ranking_preds = pairwise_model_stage1.stage1_ranking(question, pre_texts)
+
+    ranking_scores = ranking_preds * bm25_scores
+    
+    #Question answering
+    best_idxs = np.argsort(ranking_scores)[-10:]
+    ranking_scores = np.array(ranking_scores)[best_idxs]
+    texts = np.array(pre_texts)[best_idxs]
+
+    best_answer = qa_model(question, texts, ranking_scores)
+    
+    if best_answer is None:
+        return pre_texts[0]
+    
+    return best_answer
+
+if __name__ == "__main__":
+    # result = get_answer_e2e("OKR là gì?")
+    # print(result)
+    gr.Interface(fn=get_answer_e2e, inputs=["text"], outputs=["textbox"]).launch()

bm25_utils.py

@@ -0,0 +1,40 @@
+import numpy as np
+from tqdm.auto import tqdm
+
+tqdm.pandas()
+from gensim.corpora import Dictionary
+from gensim.models import TfidfModel
+from gensim.similarities import SparseMatrixSimilarity
+from text_utils import preprocess
+
+
+class BM25Gensim:
+    def __init__(self, checkpoint_path, entity_dict, title2idx):
+        self.dictionary = Dictionary.load(checkpoint_path + "/dict")
+        self.tfidf_model = SparseMatrixSimilarity.load(checkpoint_path + "/tfidf")
+        self.bm25_index = TfidfModel.load(checkpoint_path + "/bm25_index")
+        self.title2idx = title2idx
+        self.entity_dict = entity_dict
+
+    def get_topk_stage1(self, query, topk=100):
+        tokenized_query = query.split()
+        tfidf_query = self.tfidf_model[self.dictionary.doc2bow(tokenized_query)]
+        scores = self.bm25_index[tfidf_query]
+        top_n = np.argsort(scores)[::-1][:topk]
+        return top_n, scores[top_n]
+
+    def get_topk_stage2(self, x, raw_answer=None, topk=50):
+        x = str(x)
+        query = preprocess(x, max_length=128).lower().split()
+        tfidf_query = self.tfidf_model[self.dictionary.doc2bow(query)]
+        scores = self.bm25_index[tfidf_query]
+        top_n = list(np.argsort(scores)[::-1][:topk])
+        if raw_answer is not None:
+            raw_answer = raw_answer.strip()
+            if raw_answer in self.entity_dict:
+                title = self.entity_dict[raw_answer].replace("wiki/", "").replace("_", " ")
+                extra_id = self.title2idx.get(title, -1)
+                if extra_id != -1 and extra_id not in top_n:
+                    top_n.append(extra_id)
+        scores = scores[top_n]
+        return np.array(top_n), np.array(scores)

graph_utils.py

@@ -0,0 +1,110 @@
+import networkx as nx
+import numpy as np
+from cdlib import algorithms
+
+
+# these functions are heavily influenced by the HF squad_metrics.py script
+def normalize_text(s):
+    """Removing articles and punctuation, and standardizing whitespace are all typical text processing steps."""
+    import string, re
+
+    def remove_articles(text):
+        regex = re.compile(r"\b(a|an|the)\b", re.UNICODE)
+        return re.sub(regex, " ", text)
+
+    def white_space_fix(text):
+        return " ".join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def compute_exact_match(prediction, truth):
+    return int(normalize_text(prediction) == normalize_text(truth))
+
+
+def compute_f1(prediction, truth):
+    pred_tokens = normalize_text(prediction).split()
+    truth_tokens = normalize_text(truth).split()
+
+    # if either the prediction or the truth is no-answer then f1 = 1 if they agree, 0 otherwise
+    if len(pred_tokens) == 0 or len(truth_tokens) == 0:
+        return int(pred_tokens == truth_tokens)
+
+    common_tokens = set(pred_tokens) & set(truth_tokens)
+
+    # if there are no common tokens then f1 = 0
+    if len(common_tokens) == 0:
+        return 0
+
+    prec = len(common_tokens) / len(pred_tokens)
+    rec = len(common_tokens) / len(truth_tokens)
+
+    return 2 * (prec * rec) / (prec + rec)
+
+
+def is_date_or_num(answer):
+    answer = answer.lower().split()
+    for w in answer:
+        w = w.strip()
+        if w.isnumeric() or w in ["ngày", "tháng", "năm"]:
+            return True
+    return False
+
+
+def find_best_cluster(answers, best_answer, thr=0.79):
+    if len(answers) == 0:  # or best_answer not in answers:
+        return best_answer
+    elif len(answers) == 1:
+        return answers[0]
+    dists = np.zeros((len(answers), len(answers)))
+    for i in range(len(answers) - 1):
+        for j in range(i + 1, len(answers)):
+            a1 = answers[i].lower().strip()
+            a2 = answers[j].lower().strip()
+            if is_date_or_num(a1) or is_date_or_num(a2):
+                # print(a1, a2)
+                if a1 == a2 or ("tháng" in a1 and a1 in a2) or ("tháng" in a2 and a2 in a1):
+                    dists[i, j] = 1
+                    dists[j, i] = 1
+                # continue
+            elif a1 == a2 or (a1 in a2) or (a2 in a1) or compute_f1(a1.lower(), a2.lower()) >= thr:
+                dists[i, j] = 1
+                dists[j, i] = 1
+    # print(dists)
+    try:
+        thr = 1
+        dups = np.where(dists >= thr)
+        dup_strs = []
+        edges = []
+        for i, j in zip(dups[0], dups[1]):
+            if i != j:
+                edges.append((i, j))
+        G = nx.Graph()
+        for i, answer in enumerate(answers):
+            G.add_node(i, content=answer)
+        G.add_edges_from(edges)
+        partition = algorithms.louvain(G)
+        max_len_comm = np.max([len(x) for x in partition.communities])
+        best_comms = []
+        for comm in partition.communities:
+            # print([answers[i] for i in comm])
+            if len(comm) == max_len_comm:
+                best_comms.append([answers[i] for i in comm])
+        # if len(best_comms) > 1:
+        #     return best_answer
+        for comm in best_comms:
+            if best_answer in comm:
+                return best_answer
+        mid = len(best_comms[0]) // 2
+        # print(mid, sorted(best_comms[0], key = len))
+        return sorted(best_comms[0], key=len)[mid]
+    except Exception as e:
+        print(e, "Disconnected graph")
+        return best_answer

openvino_stage1/stage1.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac91eb96b7cb06441c544d46f1a64b9f44c7bec4f629e91c1e28cee61570262a
+size 1109819632

outputs/bm25_stage1/bm25_index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35c10a1b81b1a458ca41cd3fcc73ab1dcedf0606024aa57b94a99e21e1820fbd
+size 28408446

outputs/bm25_stage1/tfidf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed2df8ff211b311b6f99b1f9f7d5f34ad24139fbedb608aa04465a37aae56b58
+size 1525507

pairwise_model.py

@@ -0,0 +1,139 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader
+from transformers import AutoModel, AutoConfig, AutoTokenizer
+import pandas as pd
+from optimum.intel import OVModelForQuestionAnswering
+import openvino.inference_engine as ie
+import os
+import gradio as gr
+
+AUTH_TOKEN = "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"
+
+tokenizer = AutoTokenizer.from_pretrained('nguyenvulebinh/vi-mrc-base',
+                                          use_auth_token=AUTH_TOKEN)
+pad_token_id = tokenizer.pad_token_id
+
+# Load the model
+model_xml = "openvino_stage1/stage1.xml"
+model_bin = "openvino_stage1/stage1.bin"
+# Create an Inference Engine object
+ie_core = ie.IECore()
+# Read the IR files"
+net = ie_core.read_network(model=model_xml, weights=model_bin)
+
+class PairwiseModel_modify(nn.Module):
+    def __init__(self, model_name, max_length=384, batch_size=16, device="cpu"):
+        super(PairwiseModel_modify, self).__init__()
+        self.max_length = max_length
+        self.batch_size = batch_size
+        self.device = device
+        # self.model = AutoModel.from_pretrained(model_name , use_auth_token=AUTH_TOKEN)
+        self.config = AutoConfig.from_pretrained(model_name, use_auth_token=AUTH_TOKEN)
+        self.fc = nn.Linear(768, 1).to(self.device)
+
+    def forward(self, ids, masks):
+        # Export the model to ONNX format
+        input_feed = {"input_ids": ids.cpu().numpy().astype(np.int64), "attention_mask": masks.cpu().numpy().astype(np.int64)}
+        # Specify the input shapes (batch_size, max_sequence_length)
+        input_shapes = {"input_ids": ids.shape, "attention_mask": masks.shape}
+
+        # Set the input shapes in the network
+        net.reshape(input_shapes)
+
+        # Load the network with the specified input shapes
+        exec_net = ie_core.load_network(network=net, device_name="CPU")
+        outputs = exec_net.infer(input_feed)
+
+        # Get the output tensor and apply the linear layer
+        out = torch.from_numpy(outputs["output"]).to(self.device)
+        out = out[:, 0]
+        return out
+
+    def stage1_ranking(self, question, texts):
+        tmp = pd.DataFrame()
+        tmp["text"] = [" ".join(x.split()) for x in texts]
+        tmp["question"] = question
+        valid_dataset = SiameseDatasetStage1(tmp, tokenizer, self.max_length, is_test=True)
+        valid_loader = DataLoader(valid_dataset, batch_size=self.batch_size, collate_fn=collate_fn,
+                                  num_workers=0, shuffle=False, pin_memory=True)
+        preds = []
+        with torch.no_grad():
+            bar = enumerate(valid_loader)
+            for step, data in bar:
+                ids = data["ids"].to(self.device)
+                masks = data["masks"].to(self.device)
+                preds.append(torch.sigmoid(self(ids, masks)).view(-1))
+            preds = torch.concat(preds)
+        return preds.cpu().numpy()
+    
+
+class SiameseDatasetStage1(Dataset):
+
+    def __init__(self, df, tokenizer, max_length, is_test=False):
+        self.df = df
+        self.max_length = max_length
+        self.tokenizer = tokenizer
+        self.is_test = is_test
+        self.content1 = tokenizer.batch_encode_plus(list(df.question.values), max_length=max_length, truncation=True)[
+            "input_ids"]
+        self.content2 = tokenizer.batch_encode_plus(list(df.text.values), max_length=max_length, truncation=True)[
+            "input_ids"]
+        if not self.is_test:
+            self.targets = self.df.label
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, index):
+        return {
+            'ids1': torch.tensor(self.content1[index], dtype=torch.long),
+            'ids2': torch.tensor(self.content2[index][1:], dtype=torch.long),
+            'target': torch.tensor(0) if self.is_test else torch.tensor(self.targets[index], dtype=torch.float)
+        }
+
+
+class SiameseDatasetStage2(Dataset):
+
+    def __init__(self, df, tokenizer, max_length, is_test=False):
+        self.df = df
+        self.max_length = max_length
+        self.tokenizer = tokenizer
+        self.is_test = is_test
+        self.df["content1"] = self.df.apply(lambda row: row.question + f" {tokenizer.sep_token} " + row.answer, axis=1)
+        self.df["content2"] = self.df.apply(lambda row: row.title + f" {tokenizer.sep_token} " + row.candidate, axis=1)
+        self.content1 = tokenizer.batch_encode_plus(list(df.content1.values), max_length=max_length, truncation=True)[
+            "input_ids"]
+        self.content2 = tokenizer.batch_encode_plus(list(df.content2.values), max_length=max_length, truncation=True)[
+            "input_ids"]
+        if not self.is_test:
+            self.targets = self.df.label
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, index):
+        return {
+            'ids1': torch.tensor(self.content1[index], dtype=torch.long),
+            'ids2': torch.tensor(self.content2[index][1:], dtype=torch.long),
+            'target': torch.tensor(0) if self.is_test else torch.tensor(self.targets[index], dtype=torch.float)
+        }
+
+
+def collate_fn(batch):
+    ids = [torch.cat([x["ids1"], x["ids2"]]) for x in batch]
+    targets = [x["target"] for x in batch]
+    max_len = np.max([len(x) for x in ids])
+    masks = []
+    for i in range(len(ids)):
+        if len(ids[i]) < max_len:
+            ids[i] = torch.cat((ids[i], torch.tensor([pad_token_id, ] * (max_len - len(ids[i])), dtype=torch.long)))
+        masks.append(ids[i] != pad_token_id)
+    # print(tokenizer.decode(ids[0]))
+    outputs = {
+        "ids": torch.vstack(ids),
+        "masks": torch.vstack(masks),
+        "target": torch.vstack(targets).view(-1)
+    }
+    return outputs

processed/wikipedia_chungta_cleaned.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8968d3a8c99c9844b1aaf6b2c174830305473e41a85e08058ceceb95dc8ae921
+size 66457234

qa_model.py

@@ -0,0 +1,65 @@
+from pathlib import Path
+from transformers import AutoTokenizer, pipeline
+from optimum.onnxruntime import ORTModelForQuestionAnswering
+import numpy as np
+import torch
+import torch.nn as nn
+from transformers import AutoModelForQuestionAnswering, pipeline
+from text_utils import post_process_answer
+from graph_utils import find_best_cluster
+from optimum.intel import OVModelForQuestionAnswering
+import os
+import json
+from text_utils import * 
+
+
+# os.environ['HTTP_PROXY'] = 'http://proxy.hcm.fpt.vn:80'
+class QAEnsembleModel_modify(nn.Module):
+
+    # def __init__(self, model_name, model_checkpoints, entity_dict,
+    #              thr=0.1, device="cuda:0"):
+    def __init__(self, model_name, entity_dict,
+                 thr=0.1, device="cpu"):
+        super(QAEnsembleModel_modify, self).__init__()
+        self.nlps = []
+        # model_checkpoint = "./data/qa_model_robust.bin"
+        AUTH_TOKEN = "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"
+        # model_checkpoint = "letrunglinh/qa_pnc"
+        model_convert = OVModelForQuestionAnswering.from_pretrained(model_name, export= True, use_auth_token= AUTH_TOKEN)
+        # model_convert.half()
+        # model_convert.compile()
+        nlp = pipeline('question-answering', model=model_convert,
+                        tokenizer=model_name)
+        self.nlps.append(nlp)
+        self.entity_dict = entity_dict
+        self.thr = thr
+
+    def forward(self, question, texts, ranking_scores=None):
+        if ranking_scores is None:
+            ranking_scores = np.ones((len(texts),))
+
+        curr_answers = []
+        curr_scores = []
+        best_score = 0
+        for i, nlp in enumerate(self.nlps):
+            for text, score in zip(texts, ranking_scores):
+                QA_input = {
+                    'question': question,
+                    'context': text
+                }
+                res = nlp(QA_input)
+                print(res)
+                if res["score"] > self.thr:
+                    curr_answers.append(res["answer"])
+                    curr_scores.append(res["score"])
+                res["score"] = res["score"] * score
+                if i == 0:
+                    if res["score"] > best_score:
+                        answer = res["answer"]
+                        best_score = res["score"]
+        if len(curr_answers) == 0:
+            return None
+        curr_answers = [post_process_answer(x, self.entity_dict) for x in curr_answers]
+        answer = post_process_answer(answer, self.entity_dict)
+        new_best_answer = post_process_answer(find_best_cluster(curr_answers, answer), self.entity_dict)
+        return new_best_answer

text_utils.py

@@ -0,0 +1,82 @@
+import json
+from glob import glob
+import re
+from nltk import word_tokenize as lib_tokenizer
+import string
+
+
+def preprocess(x, max_length=-1, remove_puncts=False):
+    x = nltk_tokenize(x)
+    x = x.replace("\n", " ")
+    if remove_puncts:
+        x = "".join([i for i in x if i not in string.punctuation])
+    if max_length > 0:
+        x = " ".join(x.split()[:max_length])
+    return x
+
+
+def nltk_tokenize(x):
+    return " ".join(word_tokenize(strip_context(x))).strip()
+
+
+def post_process_answer(x, entity_dict):
+    if type(x) is not str:
+        return x
+    try:
+        x = strip_answer_string(x)
+    except:
+        return "NaN"
+    x = "".join([c for c in x if c not in string.punctuation])
+    x = " ".join(x.split())
+    y = x.lower()
+    if len(y) > 1 and y.split()[0].isnumeric() and ("tháng" not in x):
+        return y.split()[0]
+    if not (x.isnumeric() or "ngày" in x or "tháng" in x or "năm" in x):
+        if len(x.split()) <= 2:
+            return entity_dict.get(x.lower(), x)
+        else:
+            return x
+    else:
+        return y
+
+
+dict_map = dict({})
+
+
+def word_tokenize(text):
+    global dict_map
+    words = text.split()
+    words_norm = []
+    for w in words:
+        if dict_map.get(w, None) is None:
+            dict_map[w] = ' '.join(lib_tokenizer(w)).replace('``', '"').replace("''", '"')
+        words_norm.append(dict_map[w])
+    return words_norm
+
+
+def strip_answer_string(text):
+    text = text.strip()
+    while text[-1] in '.,/><;:\'"[]{}+=-_)(*&^!~`':
+        if text[0] != '(' and text[-1] == ')' and '(' in text:
+            break
+        if text[-1] == '"' and text[0] != '"' and text.count('"') > 1:
+            break
+        text = text[:-1].strip()
+    while text[0] in '.,/><;:\'"[]{}+=-_)(*&^!~`':
+        if text[0] == '"' and text[-1] != '"' and text.count('"') > 1:
+            break
+        text = text[1:].strip()
+    text = text.strip()
+    return text
+
+
+def strip_context(text):
+    text = text.replace('\n', ' ')
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+def check_number(x):
+    x = str(x).lower()
+    return (x.isnumeric() or "ngày" in x or "tháng" in x or "năm" in x)