Below are algorythms they are from git with explanations and data below I also gave you the Algorthms raw data they are the same file the git http and the raw data This post explores a few of these NLP and ML techniques and Data sets used:
Please review and revise the code in the order it appears to work together as follows there will be a few parts to this please wait until I say All DONE:
DATA SET WILL BE THE INDIVDUAL DOCUMENTS AND FOLDERS UPLOADED BY THE USER USING THE UPLOAD BUTTON THE AGENT  WILL COMPARE EACH DOCUMENT TO ITS OWN CONTENT AND COMPARE TO THE TEXT IN ALL DOCUMENTS UPLOADED TO PRODUCE A LIST OF DUPLICATE SENTANCES AND PARAGRAPHS WITH SIMILAR MEANING, UNIQUE CONTENT AND UNCLASSIFIABLE OR ERRORS. IT SHOULD DE DISPLAYED AS A LIST WITH ANY ENTRY THAT EXISTS MORE THAN 1 TIME LISTED ONCE WUTH A NUMBER OF OCCURANCES AND THE NAMES OF THE DOCUMENTS WHERE IT  IS FOUND.  USER CAN CLICK ON THE LINK TO SEE THE OCCURENCES IN CONTEXT. ALL DOCUMENTS SHOULD BE BLENDED INTO ONE FILE IN THE ORDER OF THE DUPLICATES THAT THEY CONTAIN AND ALL TEXT IN ALL DOCUMENTS  SHOULD BE COLOUR CODED ACCORDING TO HOW IT HAS BEEN CLASSED.

  Default model MODEL I WANT TO USE WITH LLAMA CPP: stsb-mpnet-base-v2
how to use

from sentence_transformers import SentenceTransformer
model = SentenceTransformer('model_name')

A
optional model that is better with paraphrasing  paraphrase-xlm-r-multilingual-v1 perhaps  later after app is developed I will revisit and have two search options to ensure maximum results.  For now i jsst need to gget this running and simpler is better for now.

algorithm for finding duplicates

https://github.com/kohjiaxuan/NLP-Model-for-Corpus-Similarity.git

#!/usr/bin/env python3
import os
import glob
from typing import List
from dotenv import load_dotenv
from multiprocessing import Pool
from tqdm import tqdm

from langchain.document_loaders import (
    CSVLoader,
    EverNoteLoader,
    PyMuPDFLoader,
    TextLoader,
    UnstructuredEmailLoader,
    UnstructuredEPubLoader,
    UnstructuredHTMLLoader,
    UnstructuredMarkdownLoader,
    UnstructuredODTLoader,
    UnstructuredPowerPointLoader,
    UnstructuredWordDocumentLoader,
)

from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.docstore.document import Document
from constants import CHROMA_SETTINGS


load_dotenv()


# Load environment variables
persist_directory = os.environ.get('PERSIST_DIRECTORY')
source_directory = os.environ.get('SOURCE_DIRECTORY', 'source_documents')
embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME')
chunk_size = 500
chunk_overlap = 50


# Custom document loaders
class MyElmLoader(UnstructuredEmailLoader):
    """Wrapper to fallback to text/plain when default does not work"""

    def load(self) -> List[Document]:
        """Wrapper adding fallback for elm without html"""
        try:
            try:
                doc = UnstructuredEmailLoader.load(self)
            except ValueError as e:
                if 'text/html content not found in email' in str(e):
                    # Try plain text
                    self.unstructured_kwargs["content_source"]="text/plain"
                    doc = UnstructuredEmailLoader.load(self)
                else:
                    raise
        except Exception as e:
            # Add file_path to exception message
            raise type(e)(f"{self.file_path}: {e}") from e

        return doc


# Map file extensions to document loaders and their arguments
LOADER_MAPPING = {
    ".csv": (CSVLoader, {}),
    # ".docx": (Docx2txtLoader, {}),
    ".doc": (UnstructuredWordDocumentLoader, {}),
    ".docx": (UnstructuredWordDocumentLoader, {}),
    ".enex": (EverNoteLoader, {}),
    ".eml": (MyElmLoader, {}),
    ".epub": (UnstructuredEPubLoader, {}),
    ".html": (UnstructuredHTMLLoader, {}),
    ".md": (UnstructuredMarkdownLoader, {}),
    ".odt": (UnstructuredODTLoader, {}),
    ".pdf": (PyMuPDFLoader, {}),
    ".ppt": (UnstructuredPowerPointLoader, {}),
    ".pptx": (UnstructuredPowerPointLoader, {}),
    ".txt": (TextLoader, {"encoding": "utf8"}),
    # Add more mappings for other file extensions and loaders as needed
}


def load_single_document(file_path: str) -> List[Document]:
    ext = "." + file_path.rsplit(".", 1)[-1]
    if ext in LOADER_MAPPING:
        loader_class, loader_args = LOADER_MAPPING[ext]
        loader = loader_class(file_path, **loader_args)
        return loader.load()

    raise ValueError(f"Unsupported file extension '{ext}'")

def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Document]:
    """
    Loads all documents from the source documents directory, ignoring specified files
    """
    all_files = []
    for ext in LOADER_MAPPING:
        all_files.extend(
            glob.glob(os.path.join(source_dir, f"**/*{ext}"), recursive=True)
        )
    filtered_files = [file_path for file_path in all_files if file_path not in ignored_files]

    with Pool(processes=os.cpu_count()) as pool:
        results = []
        with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:
            for i, docs in enumerate(pool.imap_unordered(load_single_document, filtered_files)):
                results.extend(docs)
                pbar.update()

    return results

def process_documents(ignored_files: List[str] = []) -> List[Document]:
    """
    Load documents and split in chunks
    """
    print(f"Loading documents from {source_directory}")
    documents = load_documents(source_directory, ignored_files)
    if not documents:
        print("No new documents to load")
        exit(0)
    print(f"Loaded {len(documents)} new documents from {source_directory}")
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
    texts = text_splitter.split_documents(documents)
    print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
    return texts

def does_vectorstore_exist(persist_directory: str) -> bool:
    """
    Checks if vectorstore exists
    """
    if os.path.exists(os.path.join(persist_directory, 'index')):
        if os.path.exists(os.path.join(persist_directory, 'chroma-collections.parquet')) and os.path.exists(os.path.join(persist_directory, 'chroma-embeddings.parquet')):
            list_index_files = glob.glob(os.path.join(persist_directory, 'index/*.bin'))
            list_index_files += glob.glob(os.path.join(persist_directory, 'index/*.pkl'))
            # At least 3 documents are needed in a working vectorstore
            if len(list_index_files) > 3:
                return True
    return False

def main():
    # Create embeddings
    embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)

    if does_vectorstore_exist(persist_directory):
        # Update and store locally vectorstore
        print(f"Appending to existing vectorstore at {persist_directory}")
        db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS)
        collection = db.get()
        texts = process_documents([metadata['source'] for metadata in collection['metadatas']])
        print(f"Creating embeddings. May take some minutes...")
        db.add_documents(texts)
    else:
        # Create and store locally vectorstore
        print("Creating new vectorstore")
        texts = process_documents()
        print(f"Creating embeddings. May take some minutes...")
        db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS)
    db.persist()
    db = None

    print(f"Ingestion complete! YOU MAY BEGIN TO SELECT A BLANK DOCUMENT OR TO EXPLORE MODIFY AND REFINE THE LIST. PLEASE NOTE THAT ALL MODIFICATIONS TO THE LIST WILL AFFECT THE UPLOADED DOCUMENTS AND VICE VERSA")


if __name__ == "__main__":
    main()
 

algorithms
TEXT PRE-PROCESSING

https://gist.github.com/c850d9b6d603358ba7c9bc9d5eb3ce84.git

from gensim.parsing.preprocessing import preprocess_string, strip_tags, strip_multiple_whitespaces, remove_stopwords, stem_text
custom_filters = [strip_tags, strip_multiple_whitespaces, remove_stopwords, stem_text]
def get_tokenized_questions(X):
    series = pd.Series(pd.concat([X['question1'], X['question2']]),dtype=str)
    series.dropna()
    for question in series:
        yield preprocess_string(question, custom_filters)


EMBEDDING — TEXT TO NUMBERS
Fasttext — For the first two models (logistic regression and gradient-boosted machine), we’ll use Fasttext embedding. The Fasttext model for English is pre-trained on Common Crawl and Wikipedia text.

https://gist.github.com/d084b595054eb01fa97f7828ea9b503e.git
raw:

# step 1: initialize word2vec model with the vocabulary of training data
model_w2v = gensim.models.Word2Vec(tokenized_questions, size=300) 

# step 2: intersect the initialized word2vec model with the pre-trained fasttext model
model_w2v.intersect_word2vec_format(input_folder+'GoogleNews-vectors-negative300.bin',
                                lockf=1.0,
                                binary=True)

# step 3: improve model with transfer-learning using the training data
model_w2v.train(tokenized_questions,total_examples=model_w2v.corpus_count, epochs=10)

GloVe — For the next two models (deep learning), the Spacy model for English will be used for embedding. This model is pre-trained on Common Crawl using GloVe. A provision can be made for OOV words by randomly mapping each OOV word to one of 50 randomly generated vectors (see code below).

https://gist.github.com/f6877b2cdd2f094ddbc0b781474a110b.git

## Code to embed questions using the spacy glove pre-trained model

# number of randomly generated OOV vectors
num_oov = 50

# extract all questions from the data set
questions = pd.concat([data['question1'], data['question2']]).values  

# load the spacy pre-trained glove model
nlp = spacy.load('en_vectors_web_lg')    
    
# create random vectors for OOV tokens
oov = np.random.normal(size=(num_oov, nlp.vocab.vectors_length))
oov = oov / oov.sum(axis=1, keepdims=True)
    
# append the OOV vectors to the end of the spacy glove vectors
vectors = np.zeros((num_vectors + num_oov, nlp.vocab.vectors_length), dtype='float32')
for lex in nlp.vocab:
    if lex.has_vector and lex.vector_norm > 0:
        vectors[lex.rank + 1] = lex.vector / lex.vector_norm 
vectors[num_vectors:, ] = oov   

# convert each question into a set of token ids (indices) which 
# will be used to index the vectors (built above) 
# to fetch the set of word vectors for each question 
questions_as_ids = []
for q in questions:
    q_tokens = nlp(q)
    word_ids = []

    for i, token in enumerate(q_tokens):
        # skip odd spaces from tokenizer
        if token.has_vector and token.vector_norm == 0:
            continue
            
        if token.has_vector:
            word_ids.append(token.rank + 1)
        else:
            # this is an OOV word, pick one of the randomly generated oov vectors
            word_ids.append(token.rank % num_oov + num_vectors)                

    questions_as_ids.append(word_ids)