Init

.dockerignore

@@ -0,0 +1,12 @@
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+*.db
+*.sqlite
+*.log
+.DS_Store
+.env
+venv
+*.bat
+desktop.ini

.gitignore

@@ -0,0 +1 @@
+*.bat

Dockerfile

@@ -0,0 +1,32 @@
+# Use an official Python runtime as a parent image
+FROM nvidia/cuda:12.3.1-runtime-ubuntu22.04 
+# Set Python to use unbuffered mode
+ENV PYTHONUNBUFFERED 1
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends python3-pip python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+# Set the working directory in the container
+RUN mkdir /var/www
+ENV HOME /var/www
+WORKDIR /var/www
+
+# Copy the current directory contents into the container at /app
+COPY . /var/www
+
+RUN pip install -r requirements.txt
+RUN pip install torch==1.12.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html
+# RUN python -c "import nltk; nltk.download('punkt')"
+# RUN python -c "import nltk; nltk.download('stopwords')"
+# RUN python -m nltk.downloader -d $HOME/nltk_data punkt stopwords
+
+EXPOSE 7866
+
+RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=true \
+    HF_TOKEN=$(cat /run/secrets/HF_TOKEN) && export HF_TOKEN
+
+# Define environment variable
+ENV FLASK_APP app.py \
+    HF_MODEL_NAME "muryshev/e5_large_rus_finetuned_20240120_122822_ep6"
+
+# Run app.py when the container launches
+CMD flask run --host=0.0.0.0 --port=7869

app.py

@@ -0,0 +1,26 @@
+import json
+from flask import Flask, jsonify, request
+from semantic_search import SemanticSearch
+
+search = SemanticSearch('cuda')
+search.test_search()
+
+app = Flask(__name__)
+app.config['JSON_AS_ASCII'] = False
+
+@app.route('/health', methods=['GET'])
+def health():
+    return jsonify({"status": "ok"})
+
+@app.route('/search', methods=['POST'])
+def search_route():
+    data = request.get_json()
+    query = data.get('query', '')
+    top = data.get('top', 10)
+    titles, docs, scores = search.search(query, top)
+    result = [{'title': str(item1), 'text': str(item2), 'relevance': str(item3)} for item1, item2, item3 in zip(titles, docs, scores)]
+    return jsonify(result)
+
+if __name__ == '__main__':
+    
+    app.run(debug=False, host='0.0.0.0', port=7868)

legal_info_search_utils/metrics.py

@@ -0,0 +1,47 @@
+import numpy as np
+
+
+def calculate_metrics_at_k(pred, true, k, dynamic_topk=False):
+    precisions_at_k = []
+    recalls_at_k = []
+    f1_scores_at_k = []
+    
+    for query_id in pred:
+        if dynamic_topk:
+            k = len(set(pred[query_id]))
+        
+        retrieved_documents = set(pred[query_id][:k])
+        relevant_documents = set(true[query_id])
+        true_positives = len(retrieved_documents.intersection(relevant_documents))
+
+        if not len(retrieved_documents) and not len(relevant_documents):
+            precisions_at_k.append(1)
+            recalls_at_k.append(1)
+            f1_scores_at_k.append(1)
+            continue
+        
+        # precision
+        precision_at_k = true_positives / k if k else 0
+        precisions_at_k.append(precision_at_k)
+        
+        # recall
+        recall_at_k = true_positives / len(relevant_documents) if relevant_documents else 0
+        recalls_at_k.append(recall_at_k)
+        
+        # f1
+        if precision_at_k + recall_at_k > 0:
+            f1_at_k = 2 * (precision_at_k * recall_at_k) / (precision_at_k + recall_at_k)
+        else:
+            f1_at_k = 0
+        f1_scores_at_k.append(f1_at_k)
+        
+    # Average Precision@k, Recall@k, and F1@k
+    avg_precision_at_k = np.mean(precisions_at_k) if precisions_at_k else 0
+    avg_recall_at_k = np.mean(recalls_at_k) if recalls_at_k else 0
+    avg_f1_at_k = np.mean(f1_scores_at_k) if f1_scores_at_k else 0
+    
+    return {
+        f"avg_precision@{k}": avg_precision_at_k, 
+        f"avg_recall@{k}": avg_recall_at_k,
+        f"avg_f1@{k}": avg_f1_at_k
+    }

legal_info_search_utils/utils.py

@@ -0,0 +1,254 @@
+import json
+import copy
+import torch
+import torch.nn.functional as F
+import numpy as np
+import faiss
+
+from tqdm import tqdm
+from torch.utils.data import Dataset, DataLoader
+from torch.cuda.amp import autocast
+
+
+all_types_but_courts = ['НКРФ', 'ГКРФ', 'ТКРФ', 'Федеральный закон', 'Письмо Минфина', 
+                        'Письмо ФНС', 'Приказ ФНС', 'Постановление Правительства']
+
+
+class FaissDocsDataset(Dataset):
+    def __init__(self, data):
+        self.data = data
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        return self.data[idx]
+
+
+def preprocess_inputs(inputs, device):
+    return {k: v[:, 0, :].to(device) for k, v in inputs.items()}
+
+
+def get_subsets_for_db(subsets, data_ids, all_docs):
+    subsets = [data_ids[ss_name] for ss_name in subsets]
+    subsets = [x for ss in subsets for x in ss]
+
+    all_docs_db = {k: v for k, v in all_docs.items() 
+                if v['id'] in subsets}
+    unique_refs = set([ref for doc in all_docs_db.values() 
+                       for ref, text in doc['added_refs'].items()])
+    db_data = {ref: text for doc in all_docs_db.values() 
+               for ref, text in doc['added_refs'].items() if ref in unique_refs}
+    return db_data
+
+
+def get_subsets_for_qa(subsets, data_ids, all_docs):
+    subsets = [data_ids[ss_name] for ss_name in subsets]
+    subsets = [x for ss in subsets for x in ss]
+    all_docs_qa = {k: v for k, v in all_docs.items() 
+                   if v['id'] in subsets}
+    return all_docs_qa
+
+
+def filter_db_data_types(text_parts, db_data_in):
+    filtered_db_data = {}
+    db_data = copy.deepcopy(db_data_in)
+    check_if_courts = 'Суды' in text_parts
+    for ref, text in db_data.items():
+        check_not_other = not any([True for x in all_types_but_courts if x in ref])
+        court_condition = check_if_courts and check_not_other
+        
+        if court_condition or any([True for x in text_parts if x in ref]):
+            filtered_db_data[ref] = text
+    return filtered_db_data
+
+
+def filter_qa_data_types(text_parts, all_docs_in):
+    filtered_all_docs = {}
+    all_docs = copy.deepcopy(all_docs_in)
+    for doc_key, doc in all_docs.items():
+        if not len(doc['added_refs']):
+            filtered_all_docs[doc_key] = doc
+            continue
+
+        filtered_refs = {}
+        check_if_courts = 'Суды' in text_parts
+        for ref, text in doc['added_refs'].items():
+            check_not_other = not any([True for x in all_types_but_courts if x in ref])
+            court_condition = check_if_courts and check_not_other
+            
+            if court_condition or any([True for x in text_parts if x in ref]):
+                filtered_refs[ref] = text
+        
+        filtered_all_docs[doc_key] = doc
+        filtered_all_docs[doc_key]['added_refs'] = filtered_refs
+    return filtered_all_docs
+
+
+def db_tokenization(filtered_db_data, tokenizer, max_len=510):
+    index_keys = {}
+    index_toks = {}    
+    for key_idx, (ref, text) in enumerate(tqdm(filtered_db_data.items(), 
+                                               desc="Tokenizing DB refs")):
+        index_keys[key_idx] = ref
+        text = "passage: " + text
+        index_toks[key_idx] = tokenizer(text, return_tensors="pt", 
+                                        padding='max_length', truncation=True, 
+                                        max_length=max_len)
+    return index_keys, index_toks
+
+
+def qa_tokenization(all_docs_qa, tokenizer, max_len=510):
+    ss_docs = []    
+    for doc in tqdm(all_docs_qa.values(), desc="Tokenizing QA docs"):
+        text = doc['title'] + '\n' + doc['question']
+        text = "query: " + text
+        text = tokenizer(text, return_tensors="pt", 
+                         padding='max_length', truncation=True, 
+                         max_length=max_len)
+        ss_docs.append([text, list(doc['added_refs'].keys())])
+
+    val_questions = [x[0] for x in ss_docs]
+    val_refs = {idx: x[1] for idx, x in enumerate(ss_docs)}
+
+    return val_questions, val_refs
+
+
+def query_tokenization(text, tokenizer, max_len=510):
+    text = "query: " + text
+    text = tokenizer(text, return_tensors="pt", 
+                     padding='max_length', truncation=True, 
+                     max_length=max_len)
+    return text
+
+
+def query_embed_extraction(tokens, model, do_normalization=True):
+    model.eval()
+    device = model.device
+    with torch.no_grad():
+        with autocast():
+            inputs = {k: v[:, :].to(device) for k, v in tokens.items()}
+            outputs = model(**inputs)
+            embedding = outputs.last_hidden_state[:, 0].cpu()
+
+            if do_normalization:
+                embedding = F.normalize(embedding, dim=-1)
+    return embedding.numpy()
+
+
+def extract_text_embeddings(index_toks, val_questions, model, 
+                            do_normalization=True, faiss_batch_size=16):
+    faiss_dataset = FaissDocsDataset(list(index_toks.values()))
+    db_data_loader = DataLoader(faiss_dataset, batch_size=faiss_batch_size)
+
+    ss_val_dataset = FaissDocsDataset(val_questions)
+    qu_data_loader = DataLoader(ss_val_dataset, batch_size=faiss_batch_size)
+
+    model.eval()
+    device = model.device
+    docs_embeds = []
+    questions_embeds = []
+    with torch.no_grad():
+        for batch in tqdm(db_data_loader, desc="db_embeds_extraction"):
+            with autocast():
+                outputs = model(**preprocess_inputs(batch, device))
+                docs_embeds.extend(outputs.last_hidden_state[:, 0].cpu())
+
+        for batch in tqdm(qu_data_loader, desc="qu_embeds_extraction"):
+            with autocast():
+                outputs = model(**preprocess_inputs(batch, device))
+                questions_embeds.extend(outputs.last_hidden_state[:, 0].cpu())
+
+    docs_embeds_faiss = [torch.unsqueeze(x, 0) for x in docs_embeds]
+    docs_embeds_faiss = torch.cat(docs_embeds_faiss)
+
+    questions_embeds_faiss = [torch.unsqueeze(x, 0) for x in questions_embeds]
+    questions_embeds_faiss = torch.cat(questions_embeds_faiss)
+
+    if do_normalization:
+        docs_embeds_faiss = F.normalize(docs_embeds_faiss, dim=-1)
+        questions_embeds_faiss = F.normalize(questions_embeds_faiss, dim=-1)
+
+    return docs_embeds_faiss.numpy(), questions_embeds_faiss.numpy()
+
+
+def run_semantic_search(index, model, tokenizer, filtered_db_data, all_docs_qa, 
+                        do_normalization=True, faiss_batch_size=16, topk=100):
+    index_keys, index_toks = db_tokenization(filtered_db_data, tokenizer)
+    val_questions, val_refs = qa_tokenization(all_docs_qa, tokenizer)
+    docs_embeds_faiss, questions_embeds_faiss = extract_text_embeddings(index_toks, 
+        val_questions, model, do_normalization, faiss_batch_size)
+    index.add(docs_embeds_faiss)
+
+    pred = {}
+    true = {}
+    all_distances = []
+    for idx, (q_embed, refs) in enumerate(zip(questions_embeds_faiss, val_refs.values())):
+        distances, indices = index.search(np.expand_dims(q_embed, 0), topk)
+        pred[idx] = [index_keys[x] for x in indices[0]]
+        true[idx] = list(refs)
+        all_distances.append(distances)
+
+    return pred, true, all_distances
+
+
+def filter_ref_parts(ref_dict, filter_parts):
+    filtered_dict = {}
+    for k, refs in ref_dict.items():
+        filtered_refs = [" ".join([x for x in ref.split() if not any([True for part in filter_parts if part in x])]) 
+                         for ref in refs]
+        filtered_dict[k] = filtered_refs
+    
+    return filtered_dict
+
+
+def get_final_metrics(pred, true, categories, top_k_values, 
+                      metrics_func, dynamic_topk=False):
+    metrics = {}
+    for top_k in top_k_values:
+        ctg_metrics = {}
+        for ctg in categories:
+            ctg_pred, ctg_true = get_exact_ctg_data(pred, true, ctg)
+            metrics_at_k = metrics_func(ctg_pred, ctg_true, top_k, dynamic_topk)
+            for mk in metrics_at_k.keys():
+                metrics_at_k[mk] = round(metrics_at_k[mk] * 100, 6)
+            ctg_metrics[ctg] = metrics_at_k
+
+        metrics[top_k] = ctg_metrics
+    return metrics
+
+
+def get_exact_ctg_data(pred_in, true_in, ctg):
+    if ctg == "all":
+        return pred_in, true_in
+    
+    out_pred = {}
+    out_true = {}
+    check_if_courts = ctg == "Суды"
+    for idx, (pred, true) in zip(true_in.keys(), zip(pred_in.values(), true_in.values())):
+        if check_if_courts:
+            ctg_refs_true = [ref for ref in true 
+                             if not any([True for x in all_types_but_courts if x in ref])]
+            ctg_refs_pred = [ref for ref in pred 
+                             if not any([True for x in all_types_but_courts if x in ref])]
+        else:
+            ctg_refs_true = [ref for ref in true if ctg in ref]
+            ctg_refs_pred = [ref for ref in pred if ctg in ref]
+        
+        out_true[idx] = ctg_refs_true
+        out_pred[idx] = ctg_refs_pred
+    return out_pred, out_true
+
+
+def print_metrics(metrics, ref_categories):
+    first_ctg = metrics[list(metrics.keys())[0]]
+    metric_tags = list(first_ctg[list(first_ctg.keys())[0]].keys())
+    metric_tags = [x.split('@')[0] for x in metric_tags]
+    print('\t', *metric_tags, sep='\t')
+
+    for ctg, ctg_short in ref_categories.items():
+        for top_k, vals in metrics.items():
+            for ctg_tag, ctg_val in vals.items():
+                if ctg_tag == ctg:
+                    ctg_vals_str = ["{:.3f}".format(x).zfill(6) for x in ctg_val.values()]
+                    print(f"{ctg_short}@{top_k}", *ctg_vals_str, sep='\t\t')

requirements.txt

@@ -0,0 +1,11 @@
+# torch==1.12.1+cu113
+numpy==1.24.3
+faiss-cpu==1.7.4
+# tqdm==4.65.0
+transformers==4.29.2
+# huggingface-hub==0.14.1
+# protobuf==3.20.3
+# sentencepiece==0.1.99
+# six==1.16.0
+# tokenizers==0.13.3
+flask==3.0.0

semantic_search.py

@@ -0,0 +1,229 @@
+import os
+import json
+import torch
+import pickle
+import numpy as np
+import faiss
+
+from transformers import AutoTokenizer, AutoModel
+from legal_info_search_utils.utils import get_subsets_for_db, get_subsets_for_qa
+from legal_info_search_utils.utils import filter_db_data_types, filter_qa_data_types
+from legal_info_search_utils.utils import db_tokenization, qa_tokenization
+from legal_info_search_utils.utils import extract_text_embeddings, filter_ref_parts
+from legal_info_search_utils.utils import print_metrics, get_final_metrics
+from legal_info_search_utils.utils import query_tokenization, query_embed_extraction
+from legal_info_search_utils.metrics import calculate_metrics_at_k
+
+
+global_data_path = os.environ.get("GLOBAL_DATA_PATH", "legal_info_search_data/")
+global_model_path = os.environ.get("GLOBAL_MODEL_PATH", 
+                                   "legal_info_search_model/20240120_122822_ep6/")
+
+# размеченные консультации
+data_path_consult = os.environ.get("DATA_PATH_CONSULT", 
+                                   global_data_path + "data_jsons_20240119.pkl")
+
+# id консультаций, побитые на train / valid / test
+data_path_consult_ids = os.environ.get("DATA_PATH_CONSULT_IDS",
+                                       global_data_path + "data_ids.json")
+
+# состав БД
+# $ export DB_SUBSETS='["train", "valid", "test"]'
+db_subsets = os.environ.get("DB_SUBSETS", ["train", "valid", "test"])
+
+# Отбор типов документов. В списке указать те, которые нужно оставить в БД.
+# $ export DB_DATA_TYPES='["НКРФ", "ГКРФ", "ТКРФ"]'
+db_data_types = os.environ.get("DB_DATA_TYPES", [
+    'НКРФ',
+    'ГКРФ',
+    'ТКРФ',
+    'Федеральный закон',
+    'Письмо Минфина',
+    'Письмо ФНС',
+    'Приказ ФНС',
+    'Постановление Правительства'
+])
+
+device = os.environ.get("MODEL_DEVICE", 'cuda' if torch.cuda.is_available() else 'cpu')
+
+# access token huggingface. Если задан, то используется модель с HF
+hf_token = os.environ.get("HF_TOKEN", "")
+hf_model_name = os.environ.get("HF_MODEL_NAME", "")
+
+class SemanticSearch:
+    def __init__(self, device, index_type="IndexFlatIP", do_embedding_norm=True, 
+                 faiss_batch_size=8, do_normalization=True):
+        self.device = device
+        self.do_embedding_norm = do_embedding_norm
+        self.faiss_batch_size = faiss_batch_size
+        self.do_normalization = do_normalization
+        self.load_model()
+        
+        indexes = {
+            "IndexFlatL2": faiss.IndexFlatL2(self.embedding_dim),
+            "IndexFlatIP": faiss.IndexFlatIP(self.embedding_dim)
+        }
+        self.index = indexes[index_type]
+        self.load_data()
+        self.preproces_data()
+        self.test_search()
+
+    def load_data(self):
+        with open(data_path_consult, "rb") as f:
+            all_docs = pickle.load(f)
+
+        with open(data_path_consult_ids, "r", encoding="utf-8") as f:
+            data_ids = json.load(f)
+
+        db_data = get_subsets_for_db(db_subsets, data_ids, all_docs)
+        filtered_all_docs = filter_qa_data_types(db_data_types, all_docs)
+        
+        self.filtered_db_data = filter_db_data_types(db_data_types, db_data)
+        self.all_docs_qa = get_subsets_for_qa(["valid"], data_ids, filtered_all_docs)
+
+    def load_model(self):
+        if hf_token and hf_model_name:
+            self.tokenizer = AutoTokenizer.from_pretrained(hf_model_name, use_auth_token=True)
+            self.model = AutoModel.from_pretrained(hf_model_name, use_auth_token=True).to(self.device)
+        else:
+            self.tokenizer = AutoTokenizer.from_pretrained(global_model_path)
+            self.model = AutoModel.from_pretrained(global_model_path).to(self.device)
+            
+        self.max_len = self.tokenizer.max_len_single_sentence
+        self.embedding_dim = self.model.config.hidden_size
+
+    def preproces_data(self):
+        index_keys, index_toks = db_tokenization(self.filtered_db_data, self.tokenizer)
+        val_questions, val_refs = qa_tokenization(self.all_docs_qa, self.tokenizer)
+        docs_embeds_faiss, questions_embeds_faiss = extract_text_embeddings(index_toks, 
+            val_questions, self.model, self.do_normalization, self.faiss_batch_size)
+
+        self.index.add(docs_embeds_faiss)
+        self.index_keys = index_keys
+        self.index_toks = index_toks
+        self.val_questions = val_questions
+        self.val_refs = val_refs
+        self.docs_embeds_faiss = docs_embeds_faiss
+        self.questions_embeds_faiss = questions_embeds_faiss
+        self.optimal_params = {
+            'НКРФ': {
+            'thresh': 0.613793, 'sim_factor': 0.878947, 'diff_n': 0},
+            'ГКРФ': {
+            'thresh': 0.758620, 'sim_factor': 0.878947, 'diff_n': 0},
+            'ТКРФ': {
+            'thresh': 0.734482, 'sim_factor': 0.9, 'diff_n': 0},
+            'Федеральный закон': {
+            'thresh': 0.734482, 'sim_factor': 0.5, 'diff_n': 0},
+            'Письмо Минфина': {
+            'thresh': 0.782758, 'sim_factor': 0.5, 'diff_n': 0},
+            'Письмо ФНС': {
+            'thresh': 0.879310, 'sim_factor': 0.5, 'diff_n': 0},
+            'Приказ ФНС': {
+            'thresh': 0.806896, 'sim_factor': 0.5, 'diff_n': 0},
+            'Постановление Правительства': {
+            'thresh': 0.782758, 'sim_factor': 0.5, 'diff_n': 0}
+        }
+        self.ref_categories = {
+            'all': 'all',
+            'НКРФ': 'НКРФ',
+            'ГКРФ': 'ГКРФ',
+            'ТКРФ': 'ТКРФ',
+            'Федеральный закон': 'ФЗ',
+            'Суды': 'Суды',
+            'Письмо Минфина': 'Письмо МФ',
+            'Письмо ФНС': 'Письмо ФНС',
+            'Приказ ФНС': 'Приказ ФНС',
+            'Постановление Правительства': 'Пост. Прав.'
+        }
+
+    def test_search(self):
+        topk = len(self.filtered_db_data)
+        pred_raw = {}
+        true = {}
+        all_distances = []
+        for idx, (q_embed, refs) in enumerate(zip(self.questions_embeds_faiss, 
+                                                  self.val_refs.values())):
+            distances, indices = self.index.search(np.expand_dims(q_embed, 0), topk)
+            pred_raw[idx] = [self.index_keys[x] for x in indices[0]]
+            true[idx] = list(refs)
+            all_distances.append(distances)
+
+        pred = {}
+        for idx, p, d in zip(true.keys(), pred_raw.values(), all_distances):
+            fp, fs = self.search_results_filtering(p, d[0])
+            pred[idx] = fp
+
+        # раскомментировать нужное. Если всё закомментировано - метрики 
+        # посчтаются "как есть", с учетом полной иерархии
+        filter_parts = [
+        #     "абз.", 
+        #     "пп.", 
+        #     "п."
+        ]
+        filtered_pred = filter_ref_parts(pred, filter_parts)
+        filtered_true = filter_ref_parts(true, filter_parts)
+
+        metrics = get_final_metrics(filtered_pred, filtered_true, 
+            self.ref_categories.keys(), [0], 
+            metrics_func=calculate_metrics_at_k, dynamic_topk=True)
+        
+        print_metrics(metrics, self.ref_categories)
+
+        
+    def search_results_filtering(self, pred, dists):
+        all_ctg_preds = []
+        all_scores = []
+        for ctg in db_data_types:
+            ctg_thresh = self.optimal_params[ctg]['thresh']
+            ctg_sim_factor = self.optimal_params[ctg]['sim_factor']
+            ctg_diff_n = self.optimal_params[ctg]['diff_n']
+            
+            ctg_preds = [(ref, dist) for ref, dist in zip(pred, dists) 
+                         if ctg in ref and dist > ctg_thresh]
+
+            sorted_pd = sorted(ctg_preds, key=lambda x: x[1], reverse=True)
+            sorted_preds = [x[0] for x in sorted_pd]
+            sorted_dists = [x[1] for x in sorted_pd]
+
+            if len(sorted_dists):
+                diffs = np.diff(sorted_dists, ctg_diff_n)
+                if len(diffs):
+                    n_preds = np.argmax(diffs) + ctg_diff_n + 1
+                else:
+                    n_preds = 0
+
+                if len(sorted_dists) > 1:
+                    ratios = (sorted_dists[1:] / sorted_dists[0]) >= ctg_sim_factor
+                    ratios = np.array([True, *ratios])
+                else:
+                    ratios = np.array([True])
+
+                main_preds = np.array(sorted_preds)[np.where(ratios)].tolist()
+                scores = np.array(sorted_dists)[np.where(ratios)].tolist()
+                if ctg_diff_n > 0 and n_preds > 0:
+                    main_preds = main_preds[:n_preds]
+                    scores = scores[:n_preds]
+            else:
+                main_preds = []
+                scores = []
+
+            all_ctg_preds.extend(main_preds)
+            all_scores.extend(scores)
+
+        sorted_values = [(ref, score) for ref, score in zip(all_ctg_preds, all_scores)]
+        sorted_values = sorted(sorted_values, key=lambda x: x[1], reverse=True)
+        sorted_preds = [x[0] for x in sorted_values]
+        sorted_scores = [x[1] for x in sorted_values]
+
+        return sorted_preds, sorted_scores
+
+    def search(self, query, top=10):
+        query_tokens = query_tokenization(query, self.tokenizer)
+        query_embeds = query_embed_extraction(query_tokens, self.model, 
+                                              self.do_normalization)
+        distances, indices = self.index.search(query_embeds, len(self.filtered_db_data))
+        pred = [self.index_keys[x] for x in indices[0]]
+        preds, scores = self.search_results_filtering(pred, distances[0])
+        docs = [self.filtered_db_data[ref] for ref in preds]
+
+        return preds[:top], docs[:top], scores[:top]

semantic_search_service_data/data_ids.json

@@ -0,0 +1,532 @@
+{
+    "train": [
+        "FDD21013-5C46-43C1-A5AE-000168446B38",
+        "1555CB0C-5F06-4FA0-8275-013B5BF62E12",
+        "0C8A5363-992C-489C-9B56-01F89495BB29",
+        "D6D62A0D-4F79-47E0-A593-0446D2BB5434",
+        "837D142E-C749-4B2A-8408-0568D8895B4B",
+        "10489897-031A-4321-A2DF-05C5AE57E460",
+        "2275F47A-20CD-4B0F-942A-05ED31833A72",
+        "C2EA18F5-FB94-4971-AFC8-05F132D7D47C",
+        "F73317B6-45F4-40CE-B154-09DD7439BD72",
+        "F84371E6-A95F-4F70-9753-0A371C8197CD",
+        "5B7C4D9F-DD0F-4D53-BEDE-0B3B4537C5EA",
+        "95B76A97-2DAB-4471-8518-0B50C8CFAA25",
+        "BCA2F97F-8B86-4E76-98E9-0C2C7A1A1D86",
+        "307D0E38-B93A-4439-9E4B-0D6727EF88B3",
+        "49DEFDE9-3CEC-4321-BC66-0DDAEA054BEF",
+        "47C11DB6-C22D-4F65-BB18-0E59F97D847D",
+        "C10B0B9D-EA97-4F1D-8451-0F98765F51F3",
+        "6583F334-9BBB-4340-B00A-1150AE362588",
+        "3FE39BDA-21CF-40EF-9467-1198EE313064",
+        "63E7BEA3-73DD-47E1-9AFC-1275AADF09BE",
+        "C6661801-C219-4A74-9006-128380D6B103",
+        "4FEBB63E-6CF7-4A14-907E-12CA46A9212D",
+        "3850987C-7FA3-40F3-BC20-13FBAFB49442",
+        "24B85580-3AD0-4DE2-B587-156DA620D2DC",
+        "C4F7457D-4712-49F0-9495-162BD07EC600",
+        "E0C12A09-2900-462C-A1A7-16796936F56A",
+        "7E6D2DA4-2207-4277-8757-16A441EAB2F3",
+        "EB5CEB3A-FA96-4EB2-8F28-16A82FC4FA5E",
+        "BCACDA21-054B-4C12-9CD7-16DE9C78911D",
+        "742564C4-F09C-49FC-BFF6-1967E2F822B8",
+        "5BED334D-5E4D-49BD-AF40-19AC1D70BBCB",
+        "B99B7ADD-7A45-42FD-9039-1A1D8B72A720",
+        "7C9079D7-4A23-4542-AD89-1C9F71E57C7D",
+        "D1AA0E8D-1BCF-45DB-8358-1CD245A3C527",
+        "BFFEEB75-9B2E-438F-B193-1D6811CCFEBC",
+        "328978A2-0F8A-4C1C-BFFA-1DBDC6C5BA67",
+        "D758CCDD-7EF5-4032-892D-1E2481881ADD",
+        "42BE6659-D200-4E82-ADC0-1E61B95D2746",
+        "92B33F61-75F0-4666-B76F-1F0430B8D938",
+        "06B2915D-EF56-4D13-8F0B-1F9394515F9E",
+        "3855CA1D-FCCB-497F-A43F-1FB0479880C6",
+        "A94D18A0-87D1-4F34-837E-201AD38C8057",
+        "9A31EA29-85CD-4DAB-A64D-21DBB4C10EAC",
+        "F7F593C3-CF02-4BE0-8C43-22A1DBBB6DFC",
+        "725B4227-1532-4707-9B15-236C9F7A5808",
+        "4A421D6F-1CCE-418D-AE4F-253CD4D3DC4B",
+        "BC6731DB-C8A6-4708-9CFE-25CFD38C5C43",
+        "214E4C44-618B-4BED-AF58-25EEE8926DF1",
+        "E40D42B8-7FFC-4535-97CE-27296645203F",
+        "A76B2376-C23D-4F37-954E-2754F9E5973D",
+        "4DC9F702-0321-42C2-BC49-2839C6CC8DA4",
+        "BE9D9417-CE89-467D-81BA-2892E6474E45",
+        "8494B9EF-7967-4F35-9EFC-28A0D4B916E5",
+        "4CD8F683-F325-423A-AA01-28BC7528021A",
+        "D2A7D890-E789-417C-A272-294CF92F4DAB",
+        "362F1D85-68C1-4D85-86C7-2A8F405E3F54",
+        "FB3E98FE-750E-4170-AB52-2B0DCF8D06FD",
+        "C79A8C7B-F695-4E64-93B9-2CAA73F30882",
+        "EE973895-441B-4FE3-BA2C-2D067E376610",
+        "F992E2DE-C312-4B36-AA92-2E8F830F028C",
+        "636585ED-E16E-49AD-BF84-2EE83D3E2E78",
+        "D5F11820-A15B-43C5-9270-2F80C7289342",
+        "62D4FC85-7D7A-4267-B4F7-30F2E5DB0E74",
+        "9487E335-C13B-4F64-81A4-31BC6732F2FE",
+        "D8B570C6-3C96-4A75-B5BA-32000E07C5A0",
+        "49387B7B-09EB-4B0F-BE22-3276B21A6C05",
+        "7434D6E4-6F26-4648-8ACE-34C53C09B212",
+        "6D9ACC37-190C-4FE5-B549-37097BF9C3EE",
+        "368C7AB9-8340-45D8-8DEC-371037E211C9",
+        "FDCC6284-DDB5-4557-A085-372BFCD6F905",
+        "584A3EDD-3563-4032-8FD8-375733264317",
+        "0BACA3DD-E8C8-4310-A74F-375D4F56295B",
+        "D0667F53-E3DC-4605-B853-37B4C5FAB3C4",
+        "02CEA5D6-5324-449E-93F3-37C0045CEE76",
+        "66435C75-9A7C-4860-B074-3877A96CCAA9",
+        "021FCC9E-28D9-48AF-B563-394940ACF5BA",
+        "C25EDBAD-5CE2-4246-8874-3A0AECBD38E4",
+        "40FF87E5-9122-4D1C-86DD-3AFDB5354E67",
+        "19E4A31C-671F-40A2-9152-3B353190EC45",
+        "ED117BEC-87EF-43F1-8DE1-3B5F227C00DC",
+        "515D11C6-8E12-4937-AB42-3BEF3DDC9D1C",
+        "B73B86C6-3109-45C0-93C5-3CD7314A82C3",
+        "EC7DC550-8413-4944-ADF5-3D057A79700E",
+        "78C3C939-F42D-45FF-9589-3D2B87E5622E",
+        "D529C996-A3CB-4469-946E-3D58A39200A9",
+        "80EDB612-28EC-46AD-9EEE-3E640BE3BD0C",
+        "FC6B67C7-2DB4-443A-BA35-3E6B8C96936F",
+        "7C189FAF-9779-4B6D-8616-3E7CD2A30F34",
+        "5231547E-420A-4FA6-B0DC-3EC446C6E040",
+        "E41349BE-469C-46E7-917F-4050092911DD",
+        "C77F95C2-DFD9-4521-BE70-40B083357041",
+        "760D08B7-6A22-425E-B195-426165392802",
+        "8B62AC1D-CB36-4636-8D6C-42B37A194665",
+        "AB9A0B32-04B4-4430-AE84-42BF77D5BF4A",
+        "AD5D7361-BBB5-4D6F-8B01-42FDC47F7AA4",
+        "57E460EE-C104-4E03-98B3-4336950B30DC",
+        "C9F6E32F-37B1-460B-9B1F-439D2D4DDBCC",
+        "68454817-0AA5-4FA9-AC69-44F399FC1D41",
+        "F2555245-3B62-421A-85FF-453BD810C59D",
+        "2B3F8E9F-5690-466D-B342-457CC2B91971",
+        "E3B81F18-ADB3-4F6B-927A-45E065FC35FB",
+        "EC02B794-1F4D-42FB-A447-4679DB9AFDD0",
+        "9F8E0584-AB18-4C9C-960E-46B43942D47B",
+        "7590E18F-DA8B-4315-9A7F-48020F48C229",
+        "BC60BDD9-72FB-45ED-82F2-480C51299324",
+        "3EB5392B-342D-4965-82D3-4821E6C42295",
+        "D58426FD-3B76-4CE5-83F1-48C66CC1CF80",
+        "A1CE0348-41EF-40A6-A01C-49270C66BC21",
+        "3157F262-F93C-429B-990D-49C14B08D263",
+        "C09FA1BD-EA0B-412F-8CC6-49C8F18F32B3",
+        "2EC5C73C-9F24-47C3-BFFF-4A52DC376131",
+        "D76F2F02-980B-4837-895D-4AF762F8320C",
+        "6264E033-9443-4CFF-BABB-4BC73F253808",
+        "94C4E0A5-4C53-4DE0-B4B7-4C433710F6F8",
+        "8A34FE88-54FB-424C-BCC2-4C9F7DA6443C",
+        "2CD8E91F-300C-48EE-BEA2-4E62D736D115",
+        "70952BA1-9BC4-4511-AA85-5050281ABAE7",
+        "569AF151-5A4B-4F60-A171-50E4DF03A13E",
+        "94B64693-314E-43F2-986E-515C5837183F",
+        "F1C032F4-928D-4B08-BCF4-52A4F4100FC9",
+        "9381B463-8987-48DA-8F4B-52AFA795A799",
+        "B597654F-63B6-4D2D-815E-5314A1473C07",
+        "CBD190BF-D9D5-45E3-86CD-532D08AF028C",
+        "3FF7D5E8-8100-4C1C-B413-54A272B32AFF",
+        "9FBF08FC-539F-4494-BCB2-564450A24650",
+        "E4F54836-A111-4DC9-B41D-56939A7B3370",
+        "0A5FE04F-1D6E-41AB-AAAF-576FA9C9F093",
+        "CCAAD3B8-EA08-4975-9329-57B6B47E0B45",
+        "0B72C345-B7A4-43D6-96DC-581C7C8ABFDA",
+        "631334A6-FC0A-49B7-9EEC-583F5928FA32",
+        "8528B707-1B55-4D04-839C-58966E5B51B9",
+        "5D2D4851-3421-4A57-94A9-589813259533",
+        "8DC6AB5D-F872-4A09-B3C5-58E4F673D200",
+        "71D39F1D-28F0-41AA-8FB8-597D04833ABC",
+        "1E141B4B-4A7B-4CAC-B166-598F9D3E9E22",
+        "B285F057-FAB0-49FB-9701-5B7E02C7904D",
+        "D039F1CC-C701-4035-9E67-5C5F2776B2B8",
+        "52B08DFF-76F7-488B-958C-5C73308D9C01",
+        "7BF518D0-44D4-41D6-B6FA-5D6060C6817F",
+        "F49A531B-CAE1-4F5C-BC9A-5DABACCF9D7E",
+        "57DF168A-B326-489D-9515-5DD1EE0ED526",
+        "3874F909-E22A-4E6E-BBEF-601855FE2BF5",
+        "EB050AED-E192-4179-926A-601A93F22A65",
+        "59F3C0DA-7A2D-4DCA-9B76-6030B5A5BF0F",
+        "48A27226-E547-4EEF-91FF-6036B592FF2F",
+        "A3EDAD53-2238-41F8-A526-604713B8831D",
+        "509EA3DC-D1B8-40C0-9523-6182BFE53A9C",
+        "66A1C6A9-D07E-4F56-80DD-61CDE9DA8743",
+        "D696DFE0-51DA-4E48-A0D9-61EC27EC8C95",
+        "DCA41856-CD48-4FF5-9491-627307B430AB",
+        "DBE17E97-EF59-4C3C-A7EC-628F5C3AF06B",
+        "FB8C1252-0D6E-4DDF-A287-631714A0CF35",
+        "CFC234BE-A913-4B82-8735-63A52E52F6A9",
+        "A8BEEE0D-EA7F-45FE-BC78-6429D0B55B01",
+        "C811D787-4A64-4442-A964-643FED06195A",
+        "9CC4A0E2-96C9-481B-BAD8-64F726197C1A",
+        "30A7CFA5-E238-4ECE-8332-65ABBB3399DD",
+        "0DA64DDF-C0CA-4616-A972-65B906B549EF",
+        "6C40C0D8-DD4F-4E8A-A2F7-660D44C5F84E",
+        "93285157-8DEC-4B46-AD63-6624A95A4A99",
+        "59F72B9C-2D5E-4049-997C-6663FE0C31CB",
+        "8411E9B5-654B-4F12-B657-66ABF8485EF8",
+        "EEC317FF-1CD0-45BF-AA5E-66D02B6253E1",
+        "836A02EA-81F1-4F38-8A0B-671A3180D0C3",
+        "B470A21C-1C04-4057-8FAD-684841983832",
+        "F1BDCBB7-A538-46FF-B97C-686B12348306",
+        "244BBE94-DB18-410A-B86D-6916831C39B5",
+        "DCB22FAB-821C-417B-BCFD-69BB81AF6348",
+        "622FA79B-54FC-4F18-90E3-69BE0EDBDAEA",
+        "2821A2D4-2416-4C40-9917-6AC63A855A39",
+        "6B49D52D-686B-42B8-B943-6B6E49D1A6D9",
+        "EC9A249B-FA82-4DAB-AD21-6BC3D35CA0A6",
+        "0ABE0E82-FD7E-4AED-9FF7-6C17F4E551AB",
+        "2EC04F83-69D0-4838-8B96-6D1EE849C4CB",
+        "627781A5-917A-49B7-AA45-6D82A8DAFE47",
+        "6BA38F04-2044-4544-B76A-6FB6D074549D",
+        "6A1EFE6C-2F5B-4586-9E78-71B5E0332016",
+        "1B375B7F-21B8-41C8-B988-7200500DF6DC",
+        "26C4D737-B2BC-4F59-92D6-727AF4AA6873",
+        "08BF73B3-2147-483E-9166-7298467DB5E1",
+        "61171B20-CC4D-4EC2-98E7-73008E6A1F29",
+        "030F8C8E-D261-4A30-8D06-731443F105E7",
+        "D601E02F-7BA5-4510-80D8-7358B866D49A",
+        "A05D06F5-ADC4-4020-B2AE-737967A54EA7",
+        "3A7A702D-0903-4B8A-AAE5-738E4058C1D1",
+        "3CDAEE2F-56C5-4E3E-84A5-73BA5DDFF99B",
+        "780DD27D-2B1B-46D3-B3B0-748BA91AE586",
+        "19983C3D-4FCA-4514-A335-74B9BB6BD052",
+        "55EF1A53-26DF-415A-B73A-753C55F2DFB5",
+        "97E8BA1E-1307-47FB-A332-755C1CB260A7",
+        "8F696F9D-7A16-4916-B51B-7739293704CD",
+        "37F47EDB-91C1-44F1-9A6B-79423AF92550",
+        "1CD9DE1B-A4ED-44E5-B075-797BD0A3FEB0",
+        "89386E88-C68C-47F5-9BC5-79AB9145E7F2",
+        "A18EB15A-BE12-448A-9AB4-79EE4C823994",
+        "D639964E-CCFD-41F6-A9D4-7A54EA40159A",
+        "9CDA588F-CCF9-4FF1-8450-7A5D10F0F468",
+        "EDEA2FF3-E040-43C1-A67D-7B7882CBBE65",
+        "802DEE10-C0E5-4CC8-B998-7BCBBBE30F5C",
+        "C5579A90-21C7-4815-94D4-7C8AB5AAF69F",
+        "A7D5A4F3-0108-4554-BC18-7F2911D8A4BA",
+        "37A59CFA-458D-4B95-82EB-7F3E533596EB",
+        "B2A17054-C238-4AFE-9848-7FCDA65FA8E9",
+        "F9F291C8-4303-46D4-B9FE-80A31A3EC8C2",
+        "1AD62D50-B88D-406E-AC14-816A1141DE09",
+        "7E307B29-42D0-4C02-8662-83DC1D410A09",
+        "E42E0F38-E1AD-4BEC-B6D4-8472882DE97C",
+        "2B4265CE-983B-41AB-B80B-8492E0DEAAC2",
+        "A1E17A41-C890-436C-BFC5-84D9BF9E6E06",
+        "8DB8639E-B982-43DD-B223-84DC2DB4F5D6",
+        "6A72A751-A28E-40AE-BF17-8501A3F12E77",
+        "C030CCB3-3CD7-415E-A83A-85650D33A348",
+        "B724A59F-116F-4958-AA9B-85CBB7098286",
+        "028B2F37-8F25-4E52-91B7-88A796B09AE9",
+        "EEC2AE3D-5D7F-43E3-BE5F-8A4F491BFD69",
+        "D99808CD-C8F6-4E83-914C-8A76FE54913D",
+        "D0048FFB-812D-47F9-A180-8BA213F46529",
+        "17643930-9E35-4E82-8E4B-8BB1F7A300BE",
+        "96F48375-CF45-4096-8367-8EBB4933742A",
+        "6C951588-71A9-49EF-8F47-8F7190653165",
+        "FFF4656B-D31D-4917-97CF-9016AA529458",
+        "7B456692-217A-4B76-9840-907C23C63E42",
+        "D8435DA8-ABCE-4FB6-AD11-90DEE6DA613E",
+        "AAF64B53-DFF7-42C5-ACF6-91239DA50FFB",
+        "B0A16157-426C-451C-A88B-91B47AAC7A95",
+        "5EEAFED6-DE37-4DA8-BB35-92267FFD100F",
+        "68DD82EB-8B89-4076-AEDC-924D859505E7",
+        "C5050A65-CC8D-4A21-A8AA-932ED3553810",
+        "677469AD-51AA-46EF-BDCA-94EFDDCE85AF",
+        "F14DC953-5B45-4B55-8BD4-95455FE8176E",
+        "AFBB6750-7916-4BD7-960B-95580E5807E4",
+        "DD9B85D8-CA17-4FAE-9183-963EDF16D0E7",
+        "023FBEDB-F009-4D3E-8529-971D3191F91D",
+        "5972BA62-C64C-41C3-85D7-9820EC706C6C",
+        "60BBC445-9B6E-46D4-A842-997D5F09DE67",
+        "A0827B2E-4290-4D17-99B2-9A6345A63114",
+        "FA7BB0D3-F6F4-46D8-999F-9A82BBDE5404",
+        "E1C84ADD-99FD-4F10-8DB2-9CE7E7C0964A",
+        "C351DD67-B359-4C1D-9C26-9D6079CF6E43",
+        "3D3FE941-034C-407B-8F47-9DFE38245580",
+        "77A19EDF-8489-43D1-91E0-9E1DC184A7E1",
+        "1EB6955A-7BFF-43A1-8D2A-9E9E586D520D",
+        "0364203A-E71E-4134-8923-9FB2030EEA0A",
+        "B5EBE58B-E6CE-4802-958B-9FF703C35D8A",
+        "4A8CDA51-C118-4D22-A0DB-A0779A3AEFF4",
+        "958094E7-7430-48A7-9C4C-A08619A811F9",
+        "6FC76DCC-2CFC-47AE-B27E-A0A0739916C9",
+        "CEB9C7BA-1156-4479-88D3-A0CE8473356B",
+        "5798DD7F-94B7-4D52-9AE2-A16AB69836B5",
+        "35F31163-BCBC-4D01-8D34-A1B1CBC1BC0D",
+        "7DB7B750-3C19-4ED7-88C2-A1EAAC0091EE",
+        "F6B11421-B5A9-4A79-ACF3-A22486BF62E5",
+        "01C0AAA0-461F-49D7-B15E-A2381F8D7000",
+        "38F5EB03-4118-4969-B342-A347250DAA36",
+        "74C918FD-00CE-4CE9-9DA8-A34802104A31",
+        "81C30638-DDA6-42A2-85E7-A3D72DACC716",
+        "F1F7C602-7E71-4896-833A-A42745B790F1",
+        "6EB24BA5-7C4A-4F9D-B95E-A54251829EEB",
+        "50F9C9AE-CF5F-4543-B5C1-A78BD0354AAA",
+        "455C2A79-7887-4BB6-956D-A81B254C37A5",
+        "A49E0732-298F-447A-B35D-A91338BB1840",
+        "AEEECD92-34BF-44FB-A67C-A9CE2200CD90",
+        "66CD95B2-A6F0-41F9-8932-AAE08A956503",
+        "7A421CDC-E9B8-48D7-9922-AC7FA447719F",
+        "E2CB398A-0507-48E1-B4B7-ACB1B7052E38",
+        "3C819C40-BB17-4F6E-A37F-AD31E4CD9FBF",
+        "AEB0B465-FDCB-4B15-88E7-AEAB39ECEDCA",
+        "5A59598A-DF6D-4023-899B-AF2FEAA2F206",
+        "806F6483-8513-4463-BB82-AF521C81DEF7",
+        "EF85F936-4B64-434D-A108-AFA592B1154E",
+        "26E15865-3C00-4C8B-858F-AFCD71554A5D",
+        "79E1A456-4FA5-48DF-98DC-B02B01C8D17C",
+        "39FF7EBA-A90F-4566-A008-B09CAE914DF2",
+        "8E48735B-0A7E-4925-AB6F-B0A80C19AEF9",
+        "FC5047D6-A9C8-4028-A4E1-B0DAEE4F0595",
+        "B50597ED-403B-4D1A-A26D-B141BCC6F72F",
+        "13CD9D2D-894C-48C7-B1B8-B31921B0FA68",
+        "26B9ED5B-DD59-4CE3-BD3D-B38B071B7AA1",
+        "B890DED5-9F77-46A7-A3B4-B3C5795E3B1A",
+        "5383B889-C245-494C-A1D6-B3D1415B4536",
+        "E4069D40-9AD0-452F-9259-B43EED34F08F",
+        "6A23AE5F-6358-418A-A406-B4A2D677A07D",
+        "6D21019C-19E6-480C-B454-B5D39CDEEDA1",
+        "79105994-7982-4C0F-82F3-B7668D3D97F0",
+        "BF715FE5-AB75-4DFE-95C2-B870E089A6B4",
+        "2BF8E530-1E21-4AAF-9F90-B87C4A0DD05A",
+        "D8D1EBF5-B55B-4BAD-AE6F-B8A4D481B332",
+        "95ED832C-5C65-46BA-8304-B8CACAFB1A29",
+        "08318A9F-ED1A-4CC3-96D9-B9038608A205",
+        "D5DDA2E8-07D0-4210-9FB8-B9ABFA525C94",
+        "137646E5-42EE-4A67-9DDC-B9BD61D1D634",
+        "F92A509A-6010-44AD-A295-B9D5007DC124",
+        "3DE2C507-280D-47E0-A648-BA9FA49EAB6E",
+        "E9D7753C-21BF-40DD-87D2-BAE65540CBF5",
+        "7EC081E4-4251-4819-9885-BB134AFC3265",
+        "34416CC0-CEB1-4A1F-86E3-BB4FCA3CDC13",
+        "C98B33CC-5946-4FFC-BA78-BC876176B945",
+        "67C15853-B374-4A15-ACC4-BD77DD9BC1C6",
+        "627FC47F-A4B3-4141-A342-BE080D23753B",
+        "8BE3AF9E-4311-4441-A444-BEBF09A9FAA1",
+        "33D502F1-91DD-4D12-A988-BEE28014D373",
+        "587BA2DD-3817-40CC-8009-BF2122F2D05F",
+        "A01886D7-B4BD-45B6-947C-C13CBA359285",
+        "61D9E05A-ED2B-4E31-8140-C215DE149BE6",
+        "C4AEB306-F17B-4BD2-AAF0-C28A49623E06",
+        "02889440-926C-45B0-AF0F-C2E43D537F94",
+        "64127E34-9183-4105-9A09-C309FB233817",
+        "A972F63F-20DA-451E-BB22-C459FB3D6E61",
+        "490240CD-CEEF-49EC-84CE-C57F6BF17541",
+        "D95F598A-4034-4E17-8D3D-C6C998BA5F9D",
+        "8C1D6A7F-6D0A-4A94-A1CF-C7E1A5A1FA27",
+        "27CAB07F-6508-4788-BFE9-C83D84FC6C18",
+        "F7CF6948-A75A-454B-95CB-C879419A5718",
+        "73A9FFBF-7C48-4F58-910D-C88117C71BD3",
+        "C967F2A4-D6B7-45EA-80BF-C99BCEC85EE9",
+        "EC389753-F1BF-465F-8135-CA590ABD77FF",
+        "9CFCD2B3-86EA-465F-86A3-CA8AF9EA31C3",
+        "FA24FBA0-81F2-48D2-BD58-CB7252A9D3FC",
+        "F99E4BAE-2E5F-4DBD-8FCB-CBA3399E0C08",
+        "FA67691E-9C8A-468F-9501-CCB0B2BEEDE6",
+        "4575C8ED-5222-4C17-8556-CE8183FA047E",
+        "AB0CCA85-8414-4DB9-975F-CEC35EBD35E4",
+        "CB984E74-E310-472E-9DE6-D1162C1A5E6B",
+        "1258BD32-51DB-4A84-932F-D28DE32166EF",
+        "941AD8D6-68E5-4EDA-ADE5-D385229C25AE",
+        "8C8E8CC5-7A7B-495A-9E02-D4391FFEB97B",
+        "80619C52-FD03-4023-A90C-D450AC942E9F",
+        "2A2806C9-BC23-4D83-9A54-D56AB5DA76AD",
+        "92834D79-CBD3-4131-9E81-D5AA55400F7E",
+        "3513FAEE-D197-437B-A068-D676ACA155DB",
+        "01CDA47B-FBAC-45F8-8BD5-D7241F4B9571",
+        "D592CAF6-842A-4268-BAD0-D806E527AF3E",
+        "38AE9D62-1338-43C0-93D8-D9142C915BAF",
+        "9F0079B5-DCBC-4598-9346-D96F834162C8",
+        "30469ED9-AB36-4ECB-946B-D9816E8836E7",
+        "51CEC764-2C06-4AEF-B2D0-D9881757D1DD",
+        "8083EF29-3756-4AB7-BF5F-DB27BDE115B6",
+        "32320A5D-EA42-47D3-936C-DB7DFA0D571B",
+        "339BF3B9-868C-426F-8FA1-DC616A9F1A79",
+        "E2F1CAFD-31C4-47AE-A5DD-DCC2E979A1D2",
+        "BFE89E84-EC64-415E-9236-DCE8F9385494",
+        "93AA62A2-E646-4C22-A72D-DCEA1AB1A733",
+        "3D065DE6-FB01-4FA1-810E-DE4BE1EF334A",
+        "51E5C966-4CFA-4EA6-AB13-DFA0E99A01C4",
+        "8A1D37B7-906F-4BE0-962D-E0B6594430E7",
+        "2B109511-965C-40CF-A434-E0E86177BAFD",
+        "9CD469BD-95E8-4A92-866C-E1583C501FAB",
+        "8B5355A0-708E-4117-A01F-E2F811B0DED4",
+        "B3065FAE-D948-4437-B127-E3532E7790B9",
+        "E8339872-2001-49A5-8194-E3982F606367",
+        "8E4EA9E3-C291-4CD2-AF49-E4EF8337DB9C",
+        "8803CCB6-390E-4B04-92AD-E4F11F16B4C9",
+        "8E8CBD30-37C3-499F-BFD4-E504115757C8",
+        "0DAA6284-67AF-4B5C-AE3D-E5B9AED6E085",
+        "0347D991-32C5-4FEF-BB15-E7CDE87A1CD7",
+        "389131E8-DAE0-410A-BAC5-E92189E958BF",
+        "83AC3400-389D-49C0-8095-E97C3817B666",
+        "25D6F8D9-C22B-48B3-806B-EA746A6CFF7C",
+        "1A7570C8-FD2D-4CB4-BEC6-EAF37B2F9321",
+        "107CCCC5-6277-4719-8311-EB99572F68C4",
+        "A08714DE-2024-409F-BCCD-EC55B4AAB341",
+        "884B4967-C022-488C-83BB-EE2757BFE249",
+        "3859DB2D-0049-4CDC-AF3B-EE7982CC706A",
+        "B4FAB760-0406-4964-98CA-F01CA9C6163C",
+        "EC660CEE-1533-4DFC-8CEF-F08EE229442A",
+        "B6893003-B4D5-4BE3-A2C1-F12603928F68",
+        "1902CC6A-CF30-443D-A346-F17CFEA2F1CA",
+        "01307A66-1984-45B2-B5EF-F1A6AA107D89",
+        "0C972EAF-F489-484B-BFA9-F29F1EF352B2",
+        "F0AAB378-5B55-4B3D-B16B-F2EA94B1C7B2",
+        "920278EA-5E7A-4940-9855-F50D18E34247",
+        "549659BD-D0E8-4265-A3B0-F5C4D731AEEE",
+        "5420639A-6F72-486B-B156-F5D2A0B3E113",
+        "A88BC83A-8544-4D9D-A791-F61483A90E98",
+        "70FEA54B-A755-4C19-8F7C-F63088ED6411",
+        "8E37DE7C-4511-4877-B1F1-F6C448303146",
+        "C895E999-B22D-4FD0-9F1A-F799F8F65BB9",
+        "FA782BB1-EA1F-475D-9277-F7D4F2CD2A6E",
+        "40CA2700-1230-4FE7-B6E6-F8778D02CB82",
+        "28C1A5D3-EBD8-4E53-9BDE-F894D938E4F4",
+        "FCF23D0F-ECCD-4D2E-96DC-F8F7D3511D12",
+        "B8BC6A76-76AC-4C4F-B0E4-F9042A678296",
+        "D0D93D3F-2120-4897-8D6F-F9CA7ECF940D",
+        "6EBB1114-7394-4133-8B35-F9E372908E3C",
+        "B117A371-E917-425C-B248-FAD213737AE9",
+        "1707B7F0-1393-41F8-8C68-FB5138AF099E",
+        "578BADC4-5CAE-4198-A2C9-FB5FAFF56F21",
+        "1B88E093-4B98-4AF5-BF7C-FBCDB3F450E1",
+        "B2B02F0C-0CBB-47D1-9904-FC41501E62FE",
+        "667D5FEA-A277-493E-B885-FC732A967D70",
+        "6008340D-5465-4291-B064-FCC40927523B",
+        "9B8F404C-B46A-4A2B-AF6A-FD076BF1C77A",
+        "2F1FDFEA-5A82-498B-B769-FE83513A8380",
+        "DC6E5046-DFA9-482B-B4F1-FEE4A63C9F13"
+    ],
+    "valid": [
+        "ED86925C-175F-487A-902A-ED2C94F3C68E",
+        "DFA7546E-1C7E-4F58-A6C1-42962EAA93A3",
+        "22836158-A5DC-4CE8-864D-3FCA6090D00B",
+        "918FF11C-2DFD-4386-A8AD-34EC8193AD55",
+        "87A20510-9257-47E1-8715-0C9F2C549261",
+        "C634C0CF-503B-4FFC-9733-5018969906D3",
+        "5B859D1F-4B06-4E55-8652-42FFBA94F05B",
+        "DC5BEDA5-E7FB-4F8A-9478-BD8B507E6C3D",
+        "96FA8566-E644-4A79-A40D-8DCE39F8C6F7",
+        "F9AB5AC3-8B12-4B02-9D36-001630302593",
+        "4A7E6E24-686E-4750-B854-F14B75E20B89",
+        "2F88F769-BF40-41AA-A368-197B4D865D3A",
+        "95783FD1-92CB-4CE2-B2F8-0493740BB7C2",
+        "912701BE-DF45-48A1-9BB6-AFB1A9159522",
+        "9740108B-71A9-4C02-8EC0-0AFA3383D8EC",
+        "358A6898-7973-43E4-8462-F9EA6457BC6C",
+        "47BB3615-C4A9-42FD-93DC-93346C1C98CC",
+        "D720DC29-6E65-446D-A4F1-825648C7D195",
+        "01BC301E-10B1-4933-8A94-ED4505899531",
+        "B4E14FDB-E666-4E9D-ACC2-9B518577F0C7",
+        "11C9D6C3-99DF-4561-A9E6-3ECE9D883E7F",
+        "26D30AA7-33FB-4C3F-83E0-AA360E0B3A20",
+        "242A77C3-CA9B-4BE4-B776-A3F2C1082783",
+        "5FEDCC7B-687A-498E-A509-4FCF1CD7CD0E",
+        "7E24C058-4089-4063-8DF1-7E3C4C70B548",
+        "36990F7F-8D92-4642-91D4-E6B8A724D13D",
+        "0562B6FD-36FE-4A15-9371-FEC92819B0C0",
+        "90913002-5CEB-46CA-B420-9C750CF2E49F",
+        "F8641184-A9BF-480D-B28A-87B6E2F3B0B0",
+        "1A7840C4-6B4B-4E6F-9F2E-5E64E5324E9C",
+        "4B763728-B1A4-4902-8668-B375C1FEC75D",
+        "B01E4B54-EB83-4B24-9617-0FA82668F50E",
+        "625FA286-B5A2-4CFC-A225-BA9030D284A7",
+        "535CA88C-B024-4ECD-9094-FB96F6839A3B",
+        "667C20F8-F58A-4685-A170-4FA6350709DA",
+        "8AA287B6-3560-447C-BA51-21DB94BD6F3B",
+        "F25F634B-86B0-4B07-8728-A381ECB20240",
+        "97C8A22E-BB25-46AE-9E73-B6B51C5940F2",
+        "E1CB703E-8D61-49AC-8650-98B71BA91F7A",
+        "3FD16E85-1781-4004-8663-432E81FB027B",
+        "378A8420-A813-4B19-A296-CC8D23B83A7C",
+        "217519E6-53E2-4D2C-B044-A4139BEB23F4",
+        "844C0299-35AE-42F8-8E20-1990FFD4B931",
+        "D805DC0A-E4AD-4E03-B18B-C0F5A65BE911",
+        "72AB8DE2-6E4E-431F-9D24-E1CD83856406",
+        "46C72A2B-0A8C-4F95-855C-4A77BBCAB1C6",
+        "37C20021-03D7-4B63-A8B5-219FEBFA358B",
+        "3D59B042-78B6-419B-B3BF-8DD63C6A83E5",
+        "57F8CC0C-CBB4-442E-AFBB-3F2C30ED41A9",
+        "7B378F45-6049-4C90-BB61-E4A1B85D3C40",
+        "3792900D-EC6B-452B-B04D-A043ACC888BF",
+        "AFFDF98C-4DD4-4F38-8392-619CBF56D1B3",
+        "8645BDC0-BB68-4549-90CE-884B76B5A24F",
+        "2E3ECD99-4705-4F5D-AB8B-4F35EA88EFE6",
+        "2F23AEDF-B0D6-4043-9154-D1028BCBF2D4",
+        "2189213E-B24D-4BC5-93E3-C9E00D2D443A",
+        "3152ABEB-4299-4FF8-8251-E70524A07C0F",
+        "15D5259E-8AFC-4984-BE76-77A3D56B3A10",
+        "A9311244-3D3C-45BB-A833-8936C0384DE8",
+        "40B4B46E-3B69-4E7D-8854-9135737E74D0",
+        "E9B6E27D-436E-48DD-8BA9-F3EA4D449C79",
+        "71BC5535-3E28-402E-923D-443977491E6A",
+        "4702F06F-0B91-4543-A175-193601C63DE8",
+        "58DA504E-E918-4160-908B-D24481310E4F",
+        "13BED5B1-B168-4CCE-943B-C53D9262E151"
+    ],
+    "test": [
+        "DB01E540-505A-4A0C-99F6-687D46DDA831",
+        "307F836F-13AE-4316-8A83-06A7313EDFF9",
+        "5C1FCFEE-1865-42AD-8F15-343F090D1109",
+        "38830107-2D1A-4BF4-AD10-E5EF27A2B3E5",
+        "CF610E6A-EC83-4C55-A014-2A23A9A03A87",
+        "4493F43E-A116-4BBF-A620-A43E918E4FFD",
+        "222DAA46-3086-4C10-9443-9CF322830F85",
+        "EF30B245-71A1-4213-9F08-C0B6A0BECD1C",
+        "94B14F81-9A74-4D25-8B72-E46B57567226",
+        "77C75A20-66CF-413C-9A79-B1BC14A4F6E3",
+        "7C803768-F69B-4194-86FB-C972F2E0AF42",
+        "0C4F0893-F78A-4A41-A56D-B308392FE65E",
+        "DE3FD7E3-8ABF-4344-88A5-D68755C9BCF1",
+        "3ECDC546-B253-4FE1-91B6-FAF3BFDC8CDC",
+        "7A28DB80-875A-4E49-9361-1AE67598E7D8",
+        "528F869F-21BE-407C-A1D5-E40B688682DD",
+        "D3473057-0D4F-4885-A6C0-7A39F32C682F",
+        "C45E765B-6E8B-4905-AAEE-C52E0617460E",
+        "B2A640C3-C938-4A64-AD46-99E8BFE9263A",
+        "1B92D2E8-3CF0-45A6-948C-870A73E1B9DB",
+        "0FB1EBF5-5E0B-41F8-B973-6A08709B50D7",
+        "ADCF6075-0091-4CEA-8565-D810EDCED4A8",
+        "A88750C5-F28B-4A6E-AD2F-5EC6FBBF8124",
+        "9EED4E6D-CDB0-4477-BF9D-CC6EFE4BA6EA",
+        "CC4E290E-025F-40B4-AEC1-384CE8AA44CF",
+        "26CBD0B1-2E13-41FB-AF57-071F50A471FA",
+        "00309210-23E2-4E75-90E9-B4E7EFDCB819",
+        "8A2ADB0B-7C5D-4E59-91EA-6E7BCB0AA1CD",
+        "6C754270-714F-44E6-B80E-40FBEBEB9E4B",
+        "17AD3DBB-3285-4F9D-8ED8-871935915667",
+        "A6E292A6-F4E3-4B7A-A662-7B4207FDA7F5",
+        "E3574D76-5EE7-4507-B416-1E26B296930E",
+        "71225A1A-38BC-4CC9-8CD3-058638650C91",
+        "C7A2CB26-ADD4-4443-95D6-67665B0F2D98",
+        "861F7458-C863-426E-BC3F-B88640CE77E4",
+        "C4EF56F4-9A36-48C6-AB48-A036E4204663",
+        "66B8A48D-3AAA-45C8-AE59-611E65923FD1",
+        "BB21B338-C24B-40A7-8AC6-1B954DDCEA85",
+        "143FF6ED-A38C-4B72-AAA8-DA06B9E9C93A",
+        "415651CA-C091-4CEB-B7F2-AF0F2CCAD5F0",
+        "C9D1623E-EDF2-4A4D-8458-CCC1DA11F8FC",
+        "8C1E3F67-1655-4E41-932D-67B8E0CF7E16",
+        "7DF4F107-8F2D-439D-B253-F7689057CDE0",
+        "EE676254-7181-4278-AA6F-938F3A57CDF8",
+        "D2320FC4-BE52-464F-A4AE-7CFF7B836D58",
+        "17251B2C-21AC-4D72-8EF3-4DEF1A5817CD",
+        "16DB966C-5E02-4AA9-B448-A1E8288BBA58",
+        "8E2477DE-A992-47EB-8468-E77E8D856883",
+        "EA4693CB-5BB5-4829-B438-1A887AEEE7C8",
+        "4620954A-0507-4965-9D96-44E33D5B07D6",
+        "3E3E742E-64E1-4CA7-AC9E-252E1CD19EBF",
+        "21F15E53-DD1E-4E55-A7CA-B5A4C6FEE0A1",
+        "1035AAAE-64E8-43BC-B115-EB45F9DB5C1C",
+        "ADE89EC7-5CDE-4F53-82DF-CF20037B4C02",
+        "43FA5ED1-0740-4E08-BA98-F5B61A2E128E",
+        "161B4550-3087-4B9C-A66A-47DC41CBBD95",
+        "C403350D-2451-4BA3-892A-232E30A6E205",
+        "4D75AE0A-5324-4E8B-A97F-0ACA4FFDB172",
+        "3CD9B95B-48D4-414A-9B70-B8DD33C4E216",
+        "BB56BB7D-496D-42F4-B88F-CD7B5676C9DB",
+        "FAE26B4D-5B48-43D5-BBE9-AB8547E71254",
+        "B1172ABA-3995-4908-8254-240309893D3F",
+        "C9498128-FA0F-4E5F-87DC-0882DE9655E5",
+        "36EB0995-BC3E-4C01-ABAB-A2450F48AD11",
+        "DF3366A8-2FC1-4B3A-A8C0-86E03F7D868E"
+    ]
+}

semantic_search_service_data/data_jsons_20240119.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f74bfbe62f313b6bbffe9834b49e2b409a85fbda836fa0b3feeb5515f5253a43
+size 25534952