Upload 10 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+sentence_embeddings_index_no_citation.faiss filter=lfs diff=lfs merge=lfs -text

.streamlit/secrets.toml

@@ -0,0 +1 @@
+OPENAI_API_KEY = "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"

App3.py

@@ -0,0 +1,61 @@
+from openai import OpenAI
+import streamlit as st
+import faiss
+from sentence_transformers import SentenceTransformer
+import pickle
+import re
+from transformers import pipeline
+
+
+
+
+st.title("Vietnamese Legal Question Answering System")
+
+with open('articles.pkl', 'rb') as file:
+    articles = pickle.load(file)
+
+index_loaded = faiss.read_index("sentence_embeddings_index_no_citation.faiss")
+
+if 'model_embedding' not in st.session_state:
+    print("ERROR")
+    st.session_state.model_embedding = SentenceTransformer('bkai-foundation-models/vietnamese-bi-encoder')
+
+
+
+# Replace this with your own checkpoint
+model_checkpoint = "model"
+question_answerer = pipeline("question-answering", model=model_checkpoint)
+def question_answering(question):
+    print(question)
+    query_sentence = [question]
+    query_embedding = st.session_state.model_embedding.encode(query_sentence)
+    k = 5
+    D, I = index_loaded.search(query_embedding.astype('float32'), k)  # D is distances, I is indices
+    answer = [question_answerer(question=query_sentence[0], context=articles[I[0][i]], max_answer_len = 512) for i in range(k)]
+    best_answer = max(answer, key=lambda x: x['score'])
+    print(best_answer['answer'])
+    if best_answer['score'] > 0.5:
+        return best_answer['answer']
+    return f"Tôi không chắc lắm nhưng có lẽ câu trả lời là: {best_answer['answer']}"
+
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"])
+
+
+def clean_answer(s):
+    # Sử dụng regex để loại bỏ tất cả các ký tự đặc biệt ở cuối chuỗi
+    return re.sub(r'[^a-zA-Z0-9]+$', '', s)
+    
+if prompt := st.chat_input("What is up?"):
+        st.session_state.messages.append({"role": "user", "content": prompt})
+        with st.chat_message("user"):
+            st.markdown(prompt)
+        response = clean_answer(question_answering(prompt))
+        with st.chat_message("assistant"):
+            st.markdown(response)
+        
+        st.session_state.messages.append({"role": "assistant", "content": response})

articles.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ab327307b8445059e2651ec102473c81b909f393fe0e7ab3b08fa54574cbef3
+size 112645333

model/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "nguyenvulebinh/vi-mrc-large",
+  "architectures": [
+    "RobertaForQuestionAnswering"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

model/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c754249d7b05fac9b7f4fa8e9655c331e2a39e1d250163a71f12e939f2ac1725
+size 2235510893

model/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

model/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63c73edbc2edd6c83927ce77f0f51131ae759bae511b7b1a0258cb8161f56dd0
+size 17082924

model/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "nguyenvulebinh/vi-mrc-large",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}

model/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ba966667f50e05cba202f5926f0a98d9d1bf217623435eaf15f0b00cd7cfc44
+size 3515

sentence_embeddings_index_no_citation.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ad281e6fdb666da835ef1163bff941bc21344cafec6809f58e823e4ac9dc8b
+size 199836717