Updated

analysis.py

@@ -0,0 +1,98 @@
+import requests
+import httpx
+import torch
+import re
+from bs4 import BeautifulSoup
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import asyncio
+from scipy.special import softmax
+from evaluate import load
+from datetime import date
+import nltk
+import fitz
+from transformers import GPT2LMHeadModel, GPT2TokenizerFast
+import nltk, spacy, subprocess, torch
+import plotly.graph_objects as go
+import torch.nn.functional as F
+import nltk
+from unidecode import unidecode
+import time
+import yaml
+import nltk
+import os
+from explainability import *
+from dotenv import load_dotenv
+import subprocess
+
+nltk.download("punkt")
+nltk.download("stopwords")
+load_dotenv()
+with open("config.yaml", "r") as file:
+    params = yaml.safe_load(file)
+device = "cuda" if torch.cuda.is_available() else "cpu"
+readability_model_id = params["READABILITY_MODEL_ID"]
+gpt2_model = GPT2LMHeadModel.from_pretrained(readability_model_id).to(device)
+gpt2_tokenizer = GPT2TokenizerFast.from_pretrained(readability_model_id)
+
+command = ["python", "-m", "spacy", "download", "en_core_web_sm"]
+subprocess.run(command)
+nlp = spacy.load("en_core_web_sm")
+
+
+def depth_analysis(input_text):
+    processed_words = preprocess_text1(input_text)
+    ttr_value = vocabulary_richness_ttr(processed_words)
+    gunning_fog = calculate_gunning_fog(input_text)
+    gunning_fog_norm = normalize(gunning_fog, min_value=0, max_value=20)
+    words, sentences = preprocess_text2(input_text)
+    average_sentence_length = calculate_average_sentence_length(sentences)
+    average_word_length = calculate_average_word_length(words)
+    average_sentence_length_norm = normalize(
+        average_sentence_length, min_value=0, max_value=40
+    )
+    average_word_length_norm = normalize(
+        average_word_length, min_value=0, max_value=8
+    )
+    average_tree_depth = calculate_syntactic_tree_depth(nlp, input_text)
+    average_tree_depth_norm = normalize(
+        average_tree_depth, min_value=0, max_value=10
+    )
+    perplexity = calculate_perplexity(
+        input_text, gpt2_model, gpt2_tokenizer, device
+    )
+    perplexity_norm = normalize(perplexity, min_value=0, max_value=30)
+
+    features = {
+        "readability": gunning_fog_norm,
+        "syntactic tree depth": average_tree_depth_norm,
+        "vocabulary richness": ttr_value,
+        "perplexity": perplexity_norm,
+        "average sentence length": average_sentence_length_norm,
+        "average word length": average_word_length_norm,
+    }
+    fig = go.Figure()
+    fig.add_trace(
+        go.Scatterpolar(
+            r=list(features.values()),
+            theta=list(features.keys()),
+            fill="toself",
+            name="Radar Plot",
+        )
+    )
+    fig.update_layout(
+        polar=dict(
+            radialaxis=dict(
+                visible=True,
+                range=[0, 100],
+            )
+        ),
+        showlegend=False,
+        margin=dict(
+            l=10,
+            r=20,
+            b=10,
+            t=10,
+        ),
+    )
+    return fig

app.py

@@ -1,366 +1,24 @@
-from utils import cosineSim, googleSearch, getSentences, parallel_scrap, matchingScore, matchingScoreWithTimeout
 import gradio as gr
-from urllib.request import urlopen, Request
-from googleapiclient.discovery import build
-import requests
-import httpx
-import torch
-import re
-from bs4 import BeautifulSoup
 import numpy as np
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import asyncio
-from scipy.special import softmax
-from evaluate import load
 from datetime import date
-import nltk
-import fitz 
-from transformers import GPT2LMHeadModel, GPT2TokenizerFast
-import nltk, spacy, subprocess, torch
-import plotly.graph_objects as go
-import torch.nn.functional as F
-import nltk
-from unidecode import unidecode
-import time
-
-nltk.download('punkt')
-
-from writing_analysis import (
-    normalize,
-    preprocess_text1,
-    preprocess_text2, 
-    vocabulary_richness_ttr,
-    calculate_gunning_fog,
-    calculate_average_sentence_length,
-    calculate_average_word_length,
-    calculate_syntactic_tree_depth,
-    calculate_perplexity,
-
-    )
+from predictors import predict_bc_scores, predict_mc_scores
+from analysis import depth_analysis
+from predictors import predict_quillbot
+from plagiarism import plagiarism_check, build_date
+from utils import extract_text_from_pdf, len_validator
 
 np.set_printoptions(suppress=True)
 
 
-def plagiarism_check(
-    plag_option,
-    input,
-    year_from,
-    month_from,
-    day_from,
-    year_to,
-    month_to,
-    day_to,
-    domains_to_skip,
-):
-    api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
-    api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
-    api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
-    # api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
-    cse_id = "851813e81162b4ed4"
-
-    time1 = time.perf_counter()
-    start = time.perf_counter()
-    sentences = getSentences(input)
-    urlCount = {}
-    ScoreArray = []
-    urlList = []
-
-    date_from = build_date(year_from, month_from, day_from)
-    date_to = build_date(year_to, month_to, day_to)
-    sort_date = f"date:r:{date_from}:{date_to}"
-
-    # get list of URLS to check
-    urlCount, ScoreArray = googleSearch(
-        plag_option,
-        sentences,
-        urlCount,
-        ScoreArray,
-        urlList,
-        sort_date,
-        domains_to_skip,
-        api_key,
-        cse_id,
-    )
-    print(f"Time for google search: {time.perf_counter()-time1}")
-    time1 = time.perf_counter()
-    
-    print("Number of URLs: ", len(urlCount))
-    print(urlList)
-
-    # Scrape URLs in list
-    formatted_tokens = []
-    soups = asyncio.run(parallel_scrap(urlList))
-
-    print(f"Time for scraping: {time.perf_counter()-time1}")
-    time1 = time.perf_counter()
-    print(len(soups))
-    print(
-        "Successful scraping: "
-        + str(len([x for x in soups if x is not None]))
-        + "out of "
-        + str(len(urlList))
-    )
-
-    # Populate matching scores for scrapped pages
-    for i, soup in enumerate(soups):
-        print(f"Analyzing {i+1} of {len(soups)} soups........................")
-        if soup:
-            page_content = soup.text
-            for j, sent in enumerate(sentences):
-                # score = matchingScore(sent, page_content)
-                score = matchingScoreWithTimeout(sent, page_content)
-                ScoreArray[i][j] = score
-
-    print(f"Time for matching score: {time.perf_counter()-time1}")
-    time1 = time.perf_counter()
-
-    # ScoreArray = asyncio.run(parallel_analyze_2(soups, sentences, ScoreArray))
-    # print("New Score Array:\n")
-    # print2D(ScoreArray)
-
-    # Gradio formatting section
-    sentencePlag = [False] * len(sentences)
-    sentenceToMaxURL = [-1] * len(sentences)
-    for j in range(len(sentences)):
-        if j > 0:
-            maxScore = ScoreArray[sentenceToMaxURL[j - 1]][j]
-            sentenceToMaxURL[j] = sentenceToMaxURL[j - 1]
-        else:
-            maxScore = -1
-        for i in range(len(ScoreArray)):
-            margin = (
-                0.1
-                if (j > 0 and sentenceToMaxURL[j] == sentenceToMaxURL[j - 1])
-                else 0
-            )
-            if ScoreArray[i][j] - maxScore > margin:
-                maxScore = ScoreArray[i][j]
-                sentenceToMaxURL[j] = i
-        if maxScore > 0.5:
-            sentencePlag[j] = True
-    
-    if (
-        (len(sentences) > 1)
-        and (sentenceToMaxURL[1] != sentenceToMaxURL[0])
-        and (
-            ScoreArray[sentenceToMaxURL[0]][0]
-            - ScoreArray[sentenceToMaxURL[1]][0]
-            < 0.1
-        )
-    ):
-        sentenceToMaxURL[0] = sentenceToMaxURL[1]
-
-    index = np.unique(sentenceToMaxURL)
-
-    urlScore = {}
-    for url in index:
-        s = [
-            ScoreArray[url][sen]
-            for sen in range(len(sentences))
-            if sentenceToMaxURL[sen] == url
-        ]
-        urlScore[url] = sum(s) / len(s)
-
-    index_descending = sorted(urlScore, key=urlScore.get, reverse=True)
-
-    urlMap = {}
-    for count, i in enumerate(index_descending):
-        urlMap[i] = count + 1
-    for i, sent in enumerate(sentences):
-        formatted_tokens.append(
-            (sent, "[" + str(urlMap[sentenceToMaxURL[i]]) + "]")
-        )
-
-    formatted_tokens.append(("\n", None))
-    formatted_tokens.append(("\n", None))
-    formatted_tokens.append(("\n", None))
-
-    print(formatted_tokens)
-    print(index_descending)
-    
-    for ind in index_descending:
-        formatted_tokens.append(
-            (
-                urlList[ind] + " --- Matching Score: " + f"{str(round(urlScore[ind] * 100, 2))}%",
-                "[" + str(urlMap[ind]) + "]",
-            )
+def ai_generated_test(option, input):
+    if option == "Human vs AI":
+        return predict_bc_scores(input), None
+    else:
+        return (
+            predict_bc_scores(input),
+            predict_mc_scores(input),
         )
-        formatted_tokens.append(("\n", None))
-
-    print(f"Formatted Tokens: {formatted_tokens}")
-
-    print(f"Time for plagiarism check: {time.perf_counter()-start}")
 
-    return formatted_tokens
-
-  
-"""
-AI DETECTION SECTION
-"""
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-text_bc_model_path = "polygraf-ai/text-detect-bc-v11-4m"
-text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
-text_bc_model = AutoModelForSequenceClassification.from_pretrained(text_bc_model_path).to(device)
-
-text_mc_model_path = "polygraf-ai/ai-text-detection-mc-robert-open-ai-detector-v4"
-text_mc_tokenizer = AutoTokenizer.from_pretrained(text_mc_model_path)
-text_mc_model = AutoModelForSequenceClassification.from_pretrained(text_mc_model_path).to(device)
-
-quillbot_labels = ["Original", "QuillBot"]
-quillbot_tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base")
-quillbot_model = AutoModelForSequenceClassification.from_pretrained("polygraf-ai/quillbot-detector-28k").to(device)
-
-def remove_accents(input_str):
-    text_no_accents = unidecode(input_str)
-    return text_no_accents
-
-def remove_special_characters(text):
-    text = remove_accents(text)
-    pattern = r'[^\w\s\d.,!?\'"()-;]+'
-    text = re.sub(pattern, '', text)
-    return text
-
-def remove_special_characters_2(text):
-    pattern = r'[^a-zA-Z0-9 ]+'
-    text = re.sub(pattern, '', text)
-    return text
-
-def update_character_count(text):
-    return f"{len(text)} characters"
-
-
-def split_text_allow_complete_sentences_nltk(text, max_length=256, tolerance=30, min_last_segment_length=100, type_det='bc'):
-    sentences = nltk.sent_tokenize(text)
-    segments = []
-    current_segment = []
-    current_length = 0 
-    
-    if type_det == 'bc':
-        tokenizer = text_bc_tokenizer
-        max_length = 333
-        
-    elif type_det == 'mc':
-        tokenizer = text_mc_tokenizer
-        max_length = 256
-    
-    for sentence in sentences:
-        tokens = tokenizer.tokenize(sentence)
-        sentence_length = len(tokens)
-        
-        if current_length + sentence_length <= max_length + tolerance - 2: 
-            current_segment.append(sentence)
-            current_length += sentence_length
-        else:
-            if current_segment:
-                encoded_segment = tokenizer.encode(' '.join(current_segment), add_special_tokens=True, max_length=max_length+tolerance, truncation=True)
-                segments.append((current_segment, len(encoded_segment)))
-            current_segment = [sentence]
-            current_length = sentence_length
-    
-    if current_segment:
-        encoded_segment = tokenizer.encode(' '.join(current_segment), add_special_tokens=True, max_length=max_length+tolerance, truncation=True)
-        segments.append((current_segment, len(encoded_segment)))
-
-    final_segments = []
-    for i, (seg, length) in enumerate(segments):
-        if i == len(segments) - 1:  
-            if length < min_last_segment_length and len(final_segments) > 0:
-                prev_seg, prev_length = final_segments[-1]
-                combined_encoded = tokenizer.encode(' '.join(prev_seg + seg), add_special_tokens=True, max_length=max_length+tolerance, truncation=True)
-                if len(combined_encoded) <= max_length + tolerance:
-                    final_segments[-1] = (prev_seg + seg, len(combined_encoded))
-                else:
-                    final_segments.append((seg, length))
-            else:
-                final_segments.append((seg, length))
-        else:
-            final_segments.append((seg, length))
-
-    decoded_segments = []
-    encoded_segments = []
-    for seg, _ in final_segments:
-        encoded_segment = tokenizer.encode(' '.join(seg), add_special_tokens=True, max_length=max_length+tolerance, truncation=True)
-        decoded_segment = tokenizer.decode(encoded_segment)
-        decoded_segments.append(decoded_segment)
-    return decoded_segments
-
-def predict_quillbot(text):
-    with torch.no_grad():
-        quillbot_model.eval()
-        tokenized_text = quillbot_tokenizer(text, padding="max_length", truncation=True, max_length=256, return_tensors="pt").to(device)
-        output = quillbot_model(**tokenized_text)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        q_score = {"QuillBot": output_norm[1].item(), "Original": output_norm[0].item()}
-        return q_score
-
-def predict_bc(model, tokenizer, text):
-    with torch.no_grad():
-        model.eval()
-        tokens = text_bc_tokenizer(
-            text, padding='max_length', truncation=True, max_length=333, return_tensors="pt"
-        ).to(device)
-        output = model(**tokens)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        print("BC Score: ", output_norm)
-        return output_norm
-
-def predict_mc(model, tokenizer, text):
-    with torch.no_grad():
-        model.eval()
-        tokens = text_mc_tokenizer(
-            text, padding='max_length', truncation=True, return_tensors="pt", max_length=256
-        ).to(device)
-        output = model(**tokens)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        print("MC Score: ", output_norm)
-        return output_norm
-
-def ai_generated_test(ai_option, input):
-    
-    bc_scores = []
-    mc_scores = []
-    samples_len_bc = len(split_text_allow_complete_sentences_nltk(input, type_det = 'bc'))
-    samples_len_mc = len(split_text_allow_complete_sentences_nltk(input, type_det = 'mc'))
-    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det = 'bc')
-    segments_mc = split_text_allow_complete_sentences_nltk(input, type_det = 'bc')
-    
-    for i in range(samples_len_bc):
-        cleaned_text_bc = remove_special_characters(segments_bc[i])
-        bc_score = predict_bc(text_bc_model, text_bc_tokenizer,cleaned_text_bc )
-        bc_scores.append(bc_score)
-        
-    for i in range(samples_len_mc):
-        cleaned_text_mc = remove_special_characters(segments_mc[i])
-        mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
-        mc_scores.append(mc_score)
-
-    bc_scores_array = np.array(bc_scores)
-    mc_scores_array = np.array(mc_scores)
-    average_bc_scores = np.mean(bc_scores_array, axis=0)
-    average_mc_scores = np.mean(mc_scores_array, axis=0)
-    bc_score_list = average_bc_scores.tolist()
-    mc_score_list = average_mc_scores.tolist()
-    
-    bc_score = {"AI": bc_score[1].item(), "HUMAN": bc_score[0].item()}
-    mc_score = {}
-    label_map = ["OpenAI GPT", "Mistral", "CLAUDE", "Gemini", "LLAMA 2"]
-    
-    for score, label in zip(mc_score_list, label_map):
-        mc_score[label.upper()] = score
-    
-    sum_prob = 1 - bc_score["HUMAN"]
-    for key, value in mc_score.items():
-        mc_score[key] = value * sum_prob
-    
-    if ai_option == "Human vs AI":
-        mc_score = {}
-
-    if sum_prob < 0.01  :
-        mc_score = {}
-        return bc_score, mc_score
-    else:
-        return bc_score, mc_score
 
 # COMBINED
 def main(
@@ -389,117 +47,18 @@ def main(
         domains_to_skip,
     )
     depth_analysis_plot = depth_analysis(input)
-    bc_score, mc_score = ai_generated_test(ai_option,input)
+    bc_score = predict_bc_scores(input)
+    mc_score = predict_mc_scores(input)
     quilscore = predict_quillbot(input)
-    
-    return (
-    bc_score,
-    mc_score,
-    formatted_tokens,
-    depth_analysis_plot,
-    quilscore
-            )
 
-
-def build_date(year, month, day):
-    return f"{year}{months[month]}{day}"
-
-def len_validator(text):
-    min_tokens = 200  
-    lengt = len(text_bc_tokenizer.tokenize(text = text, return_tensors="pt"))
-    if  lengt < min_tokens:
-        return f"Warning! Input length is {lengt}. Please input a text that is greater than {min_tokens} tokens long. Recommended length {min_tokens*2} tokens." 
-    else :
-        return f"Input length ({lengt}) is satisified." 
-
-def extract_text_from_pdf(pdf_path):
-    doc = fitz.open(pdf_path)
-    text = ""
-    for page in doc:
-        text += page.get_text()
-    return text
-
-
-# DEPTH ANALYSIS
-print("loading depth analysis")
-nltk.download('stopwords')
-nltk.download('punkt')
-command = ['python3', '-m', 'spacy', 'download', 'en_core_web_sm']
-# Execute the command
-subprocess.run(command)
-nlp = spacy.load("en_core_web_sm")
-
-# for perplexity
-model_id = "gpt2"
-gpt2_model = GPT2LMHeadModel.from_pretrained(model_id).to(device)
-gpt2_tokenizer = GPT2TokenizerFast.from_pretrained(model_id)
-
-def depth_analysis(input_text):
-
-    # vocanulary richness
-    processed_words = preprocess_text1(input_text)
-    ttr_value = vocabulary_richness_ttr(processed_words)
-
-    # readability
-    gunning_fog = calculate_gunning_fog(input_text)
-    gunning_fog_norm = normalize(gunning_fog, min_value=0, max_value=20)
-    
-    # average sentence length and average word length
-    words, sentences = preprocess_text2(input_text)
-    average_sentence_length = calculate_average_sentence_length(sentences)
-    average_word_length = calculate_average_word_length(words)
-    average_sentence_length_norm = normalize(average_sentence_length, min_value=0, max_value=40)
-    average_word_length_norm = normalize(average_word_length, min_value=0, max_value=8)
-
-    # syntactic_tree_depth
-    average_tree_depth = calculate_syntactic_tree_depth(nlp, input_text)
-    average_tree_depth_norm = normalize(average_tree_depth, min_value=0, max_value=10)
-
-    # perplexity
-    perplexity = calculate_perplexity(input_text, gpt2_model, gpt2_tokenizer, device)
-    perplexity_norm = normalize(perplexity, min_value=0, max_value=30)
-
-    features = {
-        "readability": gunning_fog_norm, 
-        "syntactic tree depth": average_tree_depth_norm,
-        "vocabulary richness": ttr_value,
-        "perplexity": perplexity_norm,
-        "average sentence length": average_sentence_length_norm,
-        "average word length": average_word_length_norm, 
-    }
-
-    print(features)
-
-    fig = go.Figure()
-
-    fig.add_trace(go.Scatterpolar(
-        r=list(features.values()),
-        theta=list(features.keys()),
-        fill='toself',
-        name='Radar Plot'
-    ))
-
-    fig.update_layout(
-        polar=dict(
-            radialaxis=dict(
-                visible=True,
-                range=[0, 100],
-            )),
-        showlegend=False,
-        # autosize=False,
-        # width=600,
-        # height=600,
-        margin=dict(
-            l=10,
-            r=20,
-            b=10,
-            t=10,
-            # pad=100
-        ),
+    return (
+        bc_score,
+        mc_score,
+        formatted_tokens,
+        depth_analysis_plot,
+        quilscore,
     )
 
-    return fig
-
 
 # START OF GRADIO
 
@@ -536,16 +95,23 @@ with gr.Blocks() as demo:
     with gr.Row():
         input_text = gr.Textbox(label="Input text", lines=6, placeholder="")
         file_input = gr.File(label="Upload PDF")
-        file_input.change(fn=extract_text_from_pdf, inputs=file_input, outputs=input_text)
+        file_input.change(
+            fn=extract_text_from_pdf, inputs=file_input, outputs=input_text
+        )
 
-    char_count = gr.Textbox(label="Minumum Character Limit Check") 
+    char_count = gr.Textbox(label="Minumum Character Limit Check")
     input_text.change(fn=len_validator, inputs=input_text, outputs=char_count)
 
     with gr.Row():
         with gr.Column():
-            ai_option = gr.Radio(["Human vs AI", "Human vs AI Source Models"], label="Choose an option please.")
+            ai_option = gr.Radio(
+                ["Human vs AI", "Human vs AI Source Models"],
+                label="Choose an option please.",
+            )
         with gr.Column():
-            plag_option = gr.Radio(["Standard", "Advanced"], label="Choose an option please.")
+            plag_option = gr.Radio(
+                ["Standard", "Advanced"], label="Choose an option please."
+            )
 
     with gr.Row():
         with gr.Column():
@@ -555,7 +121,7 @@ with gr.Blocks() as demo:
             only_plagiarism_btn = gr.Button("Source Check")
 
     with gr.Row():
-        quillbot_check = gr.Button("Humanized Text Check (Quillbot)")
+        quillbot_check = gr.Button("Humanized Text Check")
 
     with gr.Row():
         depth_analysis_btn = gr.Button("Detailed Writing Analysis")
@@ -568,14 +134,14 @@ with gr.Blocks() as demo:
         ## Output
         """
     )
-    
+
     # models = gr.Dropdown(
-            # model_list,
-            # value=model_list,
-            # multiselect=True,
-            # label="Models to test against",
-        # )
-    
+    # model_list,
+    # value=model_list,
+    # multiselect=True,
+    # label="Models to test against",
+    # )
+
     with gr.Row():
         with gr.Column():
             bcLabel = gr.Label(label="Source")
@@ -627,9 +193,7 @@ with gr.Blocks() as demo:
 
     with gr.Row():
         with gr.Column():
-            writing_analysis_plot = gr.Plot(
-                label="Writing Analysis Plot"
-            )
+            writing_analysis_plot = gr.Plot(label="Writing Analysis Plot")
 
     full_check_btn.click(
         fn=main,
@@ -651,7 +215,7 @@ with gr.Blocks() as demo:
             mcLabel,
             sentenceBreakdown,
             writing_analysis_plot,
-            QLabel
+            QLabel,
         ],
         api_name="main",
     )
@@ -702,4 +266,4 @@ with gr.Blocks() as demo:
     date_from = ""
     date_to = ""
   
-demo.launch(share=True, server_name="0.0.0.0", server_port =80, auth=("polygraf-admin", "test@aisd"))
+demo.launch(share=True, server_name="0.0.0.0", server_port =80, auth=("polygraf-admin", "test@aisd"))

explainability.py

@@ -0,0 +1,119 @@
+import re, textstat
+from nltk import FreqDist
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize, sent_tokenize
+import torch
+import nltk
+from tqdm import tqdm
+
+nltk.download("punkt")
+
+
+def normalize(value, min_value, max_value):
+    normalized_value = ((value - min_value) * 100) / (max_value - min_value)
+    return max(0, min(100, normalized_value))
+
+
+def preprocess_text1(text):
+    text = text.lower()
+    text = re.sub(r"[^\w\s]", "", text)  # remove punctuation
+    stop_words = set(stopwords.words("english"))  # remove stopwords
+    words = [word for word in text.split() if word not in stop_words]
+    words = [word for word in words if not word.isdigit()]  # remove numbers
+    return words
+
+
+def vocabulary_richness_ttr(words):
+    unique_words = set(words)
+    ttr = len(unique_words) / len(words) * 100
+    return ttr
+
+
+def calculate_gunning_fog(text):
+    """range 0-20"""
+    gunning_fog = textstat.gunning_fog(text)
+    return gunning_fog
+
+
+def calculate_automated_readability_index(text):
+    """range 1-20"""
+    ari = textstat.automated_readability_index(text)
+    return ari
+
+
+def calculate_flesch_reading_ease(text):
+    """range 0-100"""
+    fre = textstat.flesch_reading_ease(text)
+    return fre
+
+
+def preprocess_text2(text):
+    sentences = sent_tokenize(text)
+    words = [
+        word.lower()
+        for sent in sentences
+        for word in word_tokenize(sent)
+        if word.isalnum()
+    ]
+    stop_words = set(stopwords.words("english"))
+    words = [word for word in words if word not in stop_words]
+    return words, sentences
+
+
+def calculate_average_sentence_length(sentences):
+    """range 0-40 or 50 based on the histogram"""
+    total_words = sum(len(word_tokenize(sent)) for sent in sentences)
+    average_sentence_length = total_words / (len(sentences) + 0.0000001)
+    return average_sentence_length
+
+
+def calculate_average_word_length(words):
+    """range 0-8 based on the histogram"""
+    total_characters = sum(len(word) for word in words)
+    average_word_length = total_characters / (len(words) + 0.0000001)
+    return average_word_length
+
+
+def calculate_max_depth(sent):
+    return max(len(list(token.ancestors)) for token in sent)
+
+
+def calculate_syntactic_tree_depth(nlp, text):
+    """0-10 based on the histogram"""
+    doc = nlp(text)
+    sentence_depths = [calculate_max_depth(sent) for sent in doc.sents]
+    average_depth = (
+        sum(sentence_depths) / len(sentence_depths) if sentence_depths else 0
+    )
+    return average_depth
+
+
+def calculate_perplexity(text, model, tokenizer, device, stride=512):
+    """range 0-30 based on the histogram"""
+    encodings = tokenizer(text, return_tensors="pt")
+    max_length = model.config.n_positions
+    seq_len = encodings.input_ids.size(1)
+
+    nlls = []
+    prev_end_loc = 0
+    for begin_loc in tqdm(range(0, seq_len, stride)):
+        end_loc = min(begin_loc + max_length, seq_len)
+        trg_len = (
+            end_loc - prev_end_loc
+        )  # may be different from stride on last loop
+        input_ids = encodings.input_ids[:, begin_loc:end_loc].to(device)
+        target_ids = input_ids.clone()
+        target_ids[:, :-trg_len] = -100
+
+        with torch.no_grad():
+            outputs = model(input_ids, labels=target_ids)
+            neg_log_likelihood = outputs.loss
+
+        nlls.append(neg_log_likelihood)
+
+        prev_end_loc = end_loc
+        if end_loc == seq_len:
+            break
+
+    ppl = torch.exp(torch.stack(nlls).mean())
+    return ppl.item()

plagiarism.py

@@ -0,0 +1,344 @@
+import time
+from nltk.tokenize import sent_tokenize
+from googleapiclient.discovery import build
+from collections import Counter
+import re, math
+from sentence_transformers import SentenceTransformer, util
+import asyncio
+import httpx
+from bs4 import BeautifulSoup
+import numpy as np
+import concurrent
+
+
+WORD = re.compile(r"\w+")
+model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+
+
+# returns cosine similarity of two vectors
+# input: two vectors
+# output: integer between 0 and 1.
+def get_cosine(vec1, vec2):
+    intersection = set(vec1.keys()) & set(vec2.keys())
+
+    # calculating numerator
+    numerator = sum([vec1[x] * vec2[x] for x in intersection])
+
+    # calculating denominator
+    sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
+    sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
+    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+
+    # checking for divide by zero
+    if denominator == 0:
+        return 0.0
+    else:
+        return float(numerator) / denominator
+
+
+# converts given text into a vector
+def text_to_vector(text):
+    # uses the Regular expression above and gets all words
+    words = WORD.findall(text)
+    # returns a counter of all the words (count of number of occurences)
+    return Counter(words)
+
+
+# returns cosine similarity of two words
+# uses: text_to_vector(text) and get_cosine(v1,v2)
+def cosineSim(text1, text2):
+    vector1 = text_to_vector(text1)
+    vector2 = text_to_vector(text2)
+    # print vector1,vector2
+    cosine = get_cosine(vector1, vector2)
+    return cosine
+
+
+def cos_sim_torch(embedding_1, embedding_2):
+    return util.pytorch_cos_sim(embedding_1, embedding_2).item()
+
+
+def embed_text(text):
+    return model.encode(text, convert_to_tensor=True)
+
+
+def sentence_similarity(text1, text2):
+    embedding_1 = model.encode(text1, convert_to_tensor=True)
+    embedding_2 = model.encode(text2, convert_to_tensor=True)
+
+    o = util.pytorch_cos_sim(embedding_1, embedding_2)
+    return o.item()
+
+
+def google_search(
+    plag_option,
+    sentences,
+    urlCount,
+    scoreArray,
+    urlList,
+    sorted_date,
+    domains_to_skip,
+    api_key,
+    cse_id,
+    **kwargs,
+):
+    service = build("customsearch", "v1", developerKey=api_key)
+    for i, sentence in enumerate(sentences):
+        results = (
+            service.cse()
+            .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
+            .execute()
+        )
+        if "items" in results and len(results["items"]) > 0:
+            for count, link in enumerate(results["items"]):
+                # stop after 3 pages
+                if count >= 3:
+                    break
+                # skip user selected domains
+                if any(
+                    ("." + domain) in link["link"] for domain in domains_to_skip
+                ):
+                    continue
+                # clean up snippet of '...'
+                snippet = link["snippet"]
+                ind = snippet.find("...")
+                if ind < 20 and ind > 9:
+                    snippet = snippet[ind + len("... ") :]
+                ind = snippet.find("...")
+                if ind > len(snippet) - 5:
+                    snippet = snippet[:ind]
+
+                # update cosine similarity between snippet and given text
+                url = link["link"]
+                if url not in urlList:
+                    urlList.append(url)
+                    scoreArray.append([0] * len(sentences))
+                urlCount[url] = urlCount[url] + 1 if url in urlCount else 1
+                if plag_option == "Standard":
+                    scoreArray[urlList.index(url)][i] = cosineSim(
+                        sentence, snippet
+                    )
+                else:
+                    scoreArray[urlList.index(url)][i] = sentence_similarity(
+                        sentence, snippet
+                    )
+    return urlCount, scoreArray
+
+
+def split_sentence_blocks(text):
+
+    sents = sent_tokenize(text)
+    two_sents = []
+    for i in range(len(sents)):
+        if (i % 4) == 0:
+            two_sents.append(sents[i])
+        else:
+            two_sents[len(two_sents) - 1] += " " + sents[i]
+    return two_sents
+
+
+months = {
+    "January": "01",
+    "February": "02",
+    "March": "03",
+    "April": "04",
+    "May": "05",
+    "June": "06",
+    "July": "07",
+    "August": "08",
+    "September": "09",
+    "October": "10",
+    "November": "11",
+    "December": "12",
+}
+
+
+def build_date(year=2024, month="March", day=1):
+    return f"{year}{months[month]}{day}"
+
+
+async def get_url_data(url, client):
+    try:
+        r = await client.get(url)
+        # print(r.status_code)
+        if r.status_code == 200:
+            # print("in")
+            soup = BeautifulSoup(r.content, "html.parser")
+            return soup
+    except Exception:
+        return None
+
+
+def remove_punc(text):
+    res = re.sub(r"[^\w\s]", "", text)
+    return res
+
+
+def split_ngrams(text, n):
+    # return n-grams of size n
+    words = text.split()
+    return [words[i : i + n] for i in range(len(words) - n + 1)]
+
+
+async def parallel_scrap(urls):
+    async with httpx.AsyncClient(timeout=30) as client:
+        tasks = []
+        for url in urls:
+            tasks.append(get_url_data(url=url, client=client))
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+    return results
+
+
+def matching_score(args_list):
+    sentence = remove_punc(args_list[0])
+    content = remove_punc(args_list[1])
+    if sentence in content:
+        return 1
+    else:
+        n = 5
+        ngrams = split_ngrams(sentence, n)
+        if len(ngrams) == 0:
+            return 0
+        matched = [x for x in ngrams if " ".join(x) in content]
+    return len(matched) / len(ngrams)
+
+
+def plagiarism_check(
+    plag_option,
+    input,
+    year_from,
+    month_from,
+    day_from,
+    year_to,
+    month_to,
+    day_to,
+    domains_to_skip,
+):
+    api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
+    api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
+    api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
+    # api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
+    api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
+    cse_id = "851813e81162b4ed4"
+
+    sentences = split_sentence_blocks(input)
+    urlCount = {}
+    ScoreArray = []
+    urlList = []
+    date_from = build_date(year_from, month_from, day_from)
+    date_to = build_date(year_to, month_to, day_to)
+    sort_date = f"date:r:{date_from}:{date_to}"
+    # get list of URLS to check
+    urlCount, ScoreArray = google_search(
+        plag_option,
+        sentences,
+        urlCount,
+        ScoreArray,
+        urlList,
+        sort_date,
+        domains_to_skip,
+        api_key,
+        cse_id,
+    )
+
+    # Scrape URLs in list
+    formatted_tokens = []
+    soups = asyncio.run(parallel_scrap(urlList))
+
+    # Populate matching scores for scrapped pages
+    for i, soup in enumerate(soups):
+        print(f"Analyzing {i+1} of {len(soups)} soups........................")
+        if soup:
+            page_content = soup.text
+            for j, sent in enumerate(sentences):
+                args_list = (sent, page_content)
+                score = matching_score(args_list)
+                # score = cos_sim_torch(embed_text(sent), source_embeddings[i])
+                ScoreArray[i][j] = score
+
+    # with concurrent.futures.ProcessPoolExecutor() as executor:
+    #     results = executor.map(matching_score, args_list)
+
+    # *****IF THIS IS TO BE USED, PLEASE PROVIDE "preprocess()" FUNCTION IN LINE 248**************
+    # source_embeddings = []
+    # for i, soup in enumerate(soups):
+    #     if soup:
+    #         page_content = soup.text
+    #         source_embeddings.append(embed_text(page_content))
+    #     else:
+    #        source_embeddings.append(None)
+
+    # def compute_cosine_similarity(args):
+    #     sent, source_embedding, i, j = args
+    #     score = cos_sim_torch(embed_text(sent), source_embedding)
+    #     return i, j, score
+
+    # def main(soups, sentences):
+    #     source_embeddings = [preprocess(soup) for soup in soups]
+    #     ScoreArray = [[0 for _ in sentences] for _ in soups]
+    #     args_list = []
+    #     for i, soup in enumerate(soups):
+    #         if soup:
+    #             for j, sent in enumerate(sentences):
+    #                 args_list.append((sent, source_embeddings[i], i, j))
+    #     with concurrent.futures.ProcessPoolExecutor() as executor:
+    #         results = executor.map(compute_cosine_similarity, args_list)
+    #         for i, j, score in results:
+    #             ScoreArray[i][j] = score
+    #     return ScoreArray
+
+    # # Populate matching scores for scrapped pages
+    # ScoreArray = main(soups, sentences)
+    # *******************************************************************************************
+
+    # Calculate URL of max matching score for each sentence chunk
+    sentenceToMaxURL = [-1] * len(sentences)
+    for j in range(len(sentences)):
+        if j > 0:
+            maxScore = ScoreArray[sentenceToMaxURL[j - 1]][j]
+            sentenceToMaxURL[j] = sentenceToMaxURL[j - 1]
+        else:
+            maxScore = -1
+
+        for i in range(len(ScoreArray)):
+            margin = (
+                0.1
+                if (j > 0 and sentenceToMaxURL[j] == sentenceToMaxURL[j - 1])
+                else 0
+            )
+            if ScoreArray[i][j] - maxScore > margin:
+                maxScore = ScoreArray[i][j]
+                sentenceToMaxURL[j] = i
+
+    index = np.unique(sentenceToMaxURL)
+
+    urlScore = {}
+    for url in index:
+        s = [
+            ScoreArray[url][sen]
+            for sen in range(len(sentences))
+            if sentenceToMaxURL[sen] == url
+        ]
+        urlScore[url] = sum(s) / len(s)
+
+    index_descending = sorted(urlScore, key=urlScore.get, reverse=True)
+
+    urlMap = {}
+    for count, i in enumerate(index_descending):
+        urlMap[i] = count + 1
+    for i, sent in enumerate(sentences):
+        formatted_tokens.append(
+            (sent, "[" + str(urlMap[sentenceToMaxURL[i]]) + "]")
+        )
+    for ind in index_descending:
+        formatted_tokens.append(
+            (
+                urlList[ind]
+                + " --- Matching Score: "
+                + f"{str(round(urlScore[ind] * 100, 2))}%",
+                "[" + str(urlMap[ind]) + "]",
+            )
+        )
+        formatted_tokens.append(("\n", None))
+
+    return formatted_tokens

predictors.py

@@ -0,0 +1,246 @@
+import requests
+import httpx
+import torch
+import re
+from bs4 import BeautifulSoup
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import asyncio
+from evaluate import load
+from datetime import date
+import nltk
+from transformers import GPT2LMHeadModel, GPT2TokenizerFast
+import plotly.graph_objects as go
+import torch.nn.functional as F
+import nltk
+from unidecode import unidecode
+import time
+from scipy.special import softmax
+import yaml
+import os
+from utils import *
+from dotenv import load_dotenv
+
+with open("config.yaml", "r") as file:
+    params = yaml.safe_load(file)
+nltk.download("punkt")
+nltk.download("stopwords")
+load_dotenv()
+device = "cuda" if torch.cuda.is_available() else "cpu"
+text_bc_model_path = params["TEXT_BC_MODEL_PATH"]
+text_mc_model_path = params["TEXT_MC_MODEL_PATH"]
+text_quillbot_model_path = params["TEXT_QUILLBOT_MODEL_PATH"]
+quillbot_labels = params["QUILLBOT_LABELS"]
+mc_label_map = params["MC_OUTPUT_LABELS"]
+mc_token_size = int(params["MC_TOKEN_SIZE"])
+bc_token_size = int(params["BC_TOKEN_SIZE"])
+text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
+text_bc_model = AutoModelForSequenceClassification.from_pretrained(
+    text_bc_model_path
+).to(device)
+text_mc_tokenizer = AutoTokenizer.from_pretrained(text_mc_model_path)
+text_mc_model = AutoModelForSequenceClassification.from_pretrained(
+    text_mc_model_path
+).to(device)
+quillbot_tokenizer = AutoTokenizer.from_pretrained(text_quillbot_model_path)
+quillbot_model = AutoModelForSequenceClassification.from_pretrained(
+    text_quillbot_model_path
+).to(device)
+
+
+def split_text_allow_complete_sentences_nltk(
+    text,
+    max_length=256,
+    tolerance=30,
+    min_last_segment_length=100,
+    type_det="bc",
+):
+    sentences = nltk.sent_tokenize(text)
+    segments = []
+    current_segment = []
+    current_length = 0
+    if type_det == "bc":
+        tokenizer = text_bc_tokenizer
+        max_length = bc_token_size
+    elif type_det == "mc":
+        tokenizer = text_mc_tokenizer
+        max_length = mc_token_size
+    for sentence in sentences:
+        tokens = tokenizer.tokenize(sentence)
+        sentence_length = len(tokens)
+
+        if current_length + sentence_length <= max_length + tolerance - 2:
+            current_segment.append(sentence)
+            current_length += sentence_length
+        else:
+            if current_segment:
+                encoded_segment = tokenizer.encode(
+                    " ".join(current_segment),
+                    add_special_tokens=True,
+                    max_length=max_length + tolerance,
+                    truncation=True,
+                )
+                segments.append((current_segment, len(encoded_segment)))
+            current_segment = [sentence]
+            current_length = sentence_length
+
+    if current_segment:
+        encoded_segment = tokenizer.encode(
+            " ".join(current_segment),
+            add_special_tokens=True,
+            max_length=max_length + tolerance,
+            truncation=True,
+        )
+        segments.append((current_segment, len(encoded_segment)))
+
+    final_segments = []
+    for i, (seg, length) in enumerate(segments):
+        if i == len(segments) - 1:
+            if length < min_last_segment_length and len(final_segments) > 0:
+                prev_seg, prev_length = final_segments[-1]
+                combined_encoded = tokenizer.encode(
+                    " ".join(prev_seg + seg),
+                    add_special_tokens=True,
+                    max_length=max_length + tolerance,
+                    truncation=True,
+                )
+                if len(combined_encoded) <= max_length + tolerance:
+                    final_segments[-1] = (prev_seg + seg, len(combined_encoded))
+                else:
+                    final_segments.append((seg, length))
+            else:
+                final_segments.append((seg, length))
+        else:
+            final_segments.append((seg, length))
+
+    decoded_segments = []
+    encoded_segments = []
+    for seg, _ in final_segments:
+        encoded_segment = tokenizer.encode(
+            " ".join(seg),
+            add_special_tokens=True,
+            max_length=max_length + tolerance,
+            truncation=True,
+        )
+        decoded_segment = tokenizer.decode(encoded_segment)
+        decoded_segments.append(decoded_segment)
+    return decoded_segments
+
+
+def predict_quillbot(text):
+    with torch.no_grad():
+        quillbot_model.eval()
+        tokenized_text = quillbot_tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            max_length=256,
+            return_tensors="pt",
+        ).to(device)
+        output = quillbot_model(**tokenized_text)
+        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
+        q_score = {
+            "Humanized": output_norm[1].item(),
+            "Original": output_norm[0].item(),
+        }
+        return q_score
+
+
+def predict_bc(model, tokenizer, text):
+    with torch.no_grad():
+        model.eval()
+        tokens = text_bc_tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            max_length=bc_token_size,
+            return_tensors="pt",
+        ).to(device)
+        output = model(**tokens)
+        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
+        return output_norm
+
+
+def predict_mc(model, tokenizer, text):
+    with torch.no_grad():
+        model.eval()
+        tokens = text_mc_tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            return_tensors="pt",
+            max_length=mc_token_size,
+        ).to(device)
+        output = model(**tokens)
+        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
+        return output_norm
+
+
+def predict_mc_scores(input):
+    bc_scores = []
+    mc_scores = []
+
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
+    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    for i in range(samples_len_bc):
+        cleaned_text_bc = remove_special_characters(segments_bc[i])
+        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
+        bc_scores.append(bc_score)
+    bc_scores_array = np.array(bc_scores)
+    average_bc_scores = np.mean(bc_scores_array, axis=0)
+    bc_score_list = average_bc_scores.tolist()
+    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
+    segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    samples_len_mc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    )
+    for i in range(samples_len_mc):
+        cleaned_text_mc = remove_special_characters(segments_mc[i])
+        mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
+        mc_scores.append(mc_score)
+    mc_scores_array = np.array(mc_scores)
+    average_mc_scores = np.mean(mc_scores_array, axis=0)
+    mc_score_list = average_mc_scores.tolist()
+    mc_score = {}
+    for score, label in zip(mc_score_list, mc_label_map):
+        mc_score[label.upper()] = score
+
+    sum_prob = 1 - bc_score["HUMAN"]
+    for key, value in mc_score.items():
+        mc_score[key] = value * sum_prob
+    if sum_prob < 0.01:
+        mc_score = {}
+
+    return mc_score
+
+
+def predict_bc_scores(input):
+    bc_scores = []
+    mc_scores = []
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
+    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    for i in range(samples_len_bc):
+        cleaned_text_bc = remove_special_characters(segments_bc[i])
+        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
+        bc_scores.append(bc_score)
+    bc_scores_array = np.array(bc_scores)
+    average_bc_scores = np.mean(bc_scores_array, axis=0)
+    bc_score_list = average_bc_scores.tolist()
+    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
+    return bc_score
+
+
+# def predict_1on1(input):
+#     models = ['bard', 'claude', 'gpt4', 'mistral_ai', 'llama2']
+#     text = str(row["text"])
+#     predictions = {}
+#     prediction = predict(text, bard_model, bard_tokenizer) predictions['bard'] = prediction[1]
+#     prediction = predict(text, claude_model, claude_tokenizer) predictions['claude'] = prediction[1]
+#     prediction = predict(text, gpt4_model, gpt4_tokenizer) predictions['gpt4'] = prediction[1]
+#     prediction = predict(text, mistral_ai_model, mistral_ai_tokenizer) predictions['mistral_ai'] = prediction[1]
+#     prediction = predict(text, llama2_model, llama2_tokenizer) predictions['llama2'] = prediction[1]
+#     max_key = max(predictions, key=predictions.get)

requirements.txt

@@ -6,8 +6,8 @@ BeautifulSoup4
 scrapingbee
 requests
 numpy
-torch==1.13.0
-transformers==4.25.1
+torch
+transformers
 transformers-interpret 
 textstat
 scipy
@@ -22,4 +22,5 @@ plotly
 tqdm
 pymupdf
 sentence-transformers
-Unidecode
+Unidecode
+python-dotenv

utils.py

@@ -11,268 +11,354 @@ import asyncio
 import nltk
 from sentence_transformers import SentenceTransformer, util
 import threading
+import torch
+import re
+import numpy as np
+import asyncio
+from datetime import date
+import nltk
+from unidecode import unidecode
+from scipy.special import softmax
+from transformers import AutoTokenizer
+import yaml
+import fitz
+import os
 
-nltk.download('punkt')
 
-WORD = re.compile(r"\w+")
-model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+def remove_accents(input_str):
+    text_no_accents = unidecode(input_str)
+    return text_no_accents
 
 
-# returns cosine similarity of two vectors
-# input: two vectors
-# output: integer between 0 and 1.
-def get_cosine(vec1, vec2):
-    intersection = set(vec1.keys()) & set(vec2.keys())
+def remove_special_characters(text):
+    text = remove_accents(text)
+    pattern = r'[^\w\s\d.,!?\'"()-;]+'
+    text = re.sub(pattern, "", text)
+    return text
 
-    # calculating numerator
-    numerator = sum([vec1[x] * vec2[x] for x in intersection])
 
-    # calculating denominator
-    sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
-    sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
-    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+def remove_special_characters_2(text):
+    pattern = r"[^a-zA-Z0-9 ]+"
+    text = re.sub(pattern, "", text)
+    return text
 
-    # checking for divide by zero
-    if denominator == 0:
-        return 0.0
-    else:
-        return float(numerator) / denominator
-
-
-# converts given text into a vector
-def text_to_vector(text):
-    # uses the Regular expression above and gets all words
-    words = WORD.findall(text)
-    # returns a counter of all the words (count of number of occurences)
-    return Counter(words)
-
-
-# returns cosine similarity of two words
-# uses: text_to_vector(text) and get_cosine(v1,v2)
-def cosineSim(text1, text2):
-    vector1 = text_to_vector(text1)
-    vector2 = text_to_vector(text2)
-    # print vector1,vector2
-    cosine = get_cosine(vector1, vector2)
-    return cosine
-
-def sentence_similarity(text1, text2):
-    embedding_1= model.encode(text1, convert_to_tensor=True)
-    embedding_2 = model.encode(text2, convert_to_tensor=True)
-
-    o = util.pytorch_cos_sim(embedding_1, embedding_2)
-    return o.item()
-
-def get_soup_requests(url):
-    page = requests.get(url)
-    if page.status_code == 200:
-        soup = BeautifulSoup(page.content, "html.parser")
-        return soup
-    print("HTML soup failed")
-    return None
-
-
-def get_soup_httpx(url):
-    client = httpx.Client(timeout=30)
-    try:
-        page = client.get(url)
-        if page.status_code == httpx.codes.OK:
-            soup = BeautifulSoup(page.content, "html.parser")
-            return soup
-    except:
-        print("HTTPx soup failed")
-        return None
-
-def getSentences(text):
-    from nltk.tokenize import sent_tokenize
-
-    sents = sent_tokenize(text)
-    two_sents = []
-    for i in range(len(sents)):
-        if (i % 2) == 0:
-            two_sents.append(sents[i])
-        else:
-            two_sents[len(two_sents) - 1] += " " + sents[i]
-    return two_sents
-
-
-def googleSearch(
-    plag_option,
-    sentences,
-    urlCount,
-    scoreArray,
-    urlList,
-    sorted_date,
-    domains_to_skip,
-    api_key,
-    cse_id,
-    **kwargs,
-):
-    service = build("customsearch", "v1", developerKey=api_key)
-    for i, sentence in enumerate(sentences):
-        results = (
-            service.cse()
-            .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
-            .execute()
-        )
-        if "items" in results and len(results["items"]) > 0:
-            for count, link in enumerate(results["items"]):
-                # stop after 3 pages
-                if count >= 3:
-                    break
-                # skip user selected domains
-                if any(
-                    ("." + domain) in link["link"]
-                    for domain in domains_to_skip
-                ):
-                    continue
-                # clean up snippet of '...'
-                snippet = link["snippet"]
-                ind = snippet.find("...")
-                if ind < 20 and ind > 9:
-                    snippet = snippet[ind + len("... ") :]
-                ind = snippet.find("...")
-                if ind > len(snippet) - 5:
-                    snippet = snippet[:ind]
-
-                # update cosine similarity between snippet and given text
-                url = link["link"]
-                if url not in urlList:
-                    urlList.append(url)
-                    scoreArray.append([0] * len(sentences))
-                urlCount[url] = urlCount[url] + 1 if url in urlCount else 1
-                if plag_option == 'Standard':
-                    scoreArray[urlList.index(url)][i] = cosineSim(
-                        sentence, snippet)
-                else :     
-                    scoreArray[urlList.index(url)][i] = sentence_similarity(
-                        sentence, snippet
-                )
-        else:
-            print("Google Search failed")
-    return urlCount, scoreArray
-
-
-def getQueries(text, n):
-    # return n-grams of size n
-    words = text.split()
-    return [words[i : i + n] for i in range(len(words) - n + 1)]
-
-
-def print2D(array):
-    print(np.array(array))
-
-
-def removePunc(text):
-    res = re.sub(r"[^\w\s]", "", text)
-    return res
-
-
-async def get_url_data(url, client):
-    try:
-        r = await client.get(url)
-        # print(r.status_code)
-        if r.status_code == 200:
-            # print("in")
-            soup = BeautifulSoup(r.content, "html.parser")
-            return soup
-    except Exception:
-        print("HTTPx parallel soup failed")
-        return None
-
-
-async def parallel_scrap(urls):
-    async with httpx.AsyncClient(timeout=30) as client:
-        tasks = []
-        for url in urls:
-            tasks.append(get_url_data(url=url, client=client))
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-    return results
-
-
-class TimeoutError(Exception):
-    pass
-
-    
-
-def matchingScore(sentence, content):
-    if sentence in content:
-        return 1
-    sentence = removePunc(sentence)
-    content = removePunc(content)
-    if sentence in content:
-        return 1
+
+def update_character_count(text):
+    return f"{len(text)} characters"
+
+
+nltk.download("punkt")
+
+
+with open("config.yaml", "r") as file:
+    params = yaml.safe_load(file)
+
+text_bc_model_path = params["TEXT_BC_MODEL_PATH"]
+
+text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
+
+
+def len_validator(text):
+    min_tokens = 200
+    lengt = len(text_bc_tokenizer.tokenize(text=text, return_tensors="pt"))
+    if lengt < min_tokens:
+        return f"Warning! Input length is {lengt}. Please input a text that is greater than {min_tokens} tokens long. Recommended length {min_tokens*2} tokens."
     else:
-        n = 5
-        ngrams = getQueries(sentence, n)
-        if len(ngrams) == 0:
-            return 0
-        matched = [x for x in ngrams if " ".join(x) in content]
-    return len(matched) / len(ngrams)
-
-
-def matchingScoreWithTimeout(sentence, content):
-    def timeout_handler():
-        raise TimeoutError("Function timed out")
-
-    timer = threading.Timer(2, timeout_handler)  # Set a timer for 2 seconds
-    timer.start()
-    try:
-        score = matchingScore(sentence, content)
-        timer.cancel()  # Cancel the timer if calculation completes before timeout
-        return score
-    except TimeoutError:
-        return 0    
-
-
-async def matchingScoreAsync(sentences, content, content_idx, ScoreArray):
-    content = removePunc(content)
-    for j, sentence in enumerate(sentences):
-        sentence = removePunc(sentence)
-        if sentence in content:
-            ScoreArray[content_idx][j] = 1
-        else:
-            n = 5
-            ngrams = getQueries(sentence, n)
-            if len(ngrams) == 0:
-                return 0
-            matched = [x for x in ngrams if " ".join(x) in content]
-            ScoreArray[content_idx][j] = len(matched) / len(ngrams)
-    print(
-        f"Analyzed {content_idx+1} of soups (SOUP SUCCEEDED)........................"
-    )
-    return ScoreArray
-
-
-async def parallel_analyze(soups, sentences, ScoreArray):
-    tasks = []
-    for i, soup in enumerate(soups):
-        if soup:
-            page_content = soup.text
-            tasks.append(
-                matchingScoreAsync(sentences, page_content, i, ScoreArray)
-            )
-        else:
-            print(
-                f"Analyzed {i+1} of soups (SOUP FAILED)........................"
-            )
-    ScoreArray = await asyncio.gather(*tasks, return_exceptions=True)
-    return ScoreArray
-
-
-async def parallel_analyze_2(soups, sentences, ScoreArray):
-    tasks = [[0] * len(ScoreArray[0]) for i in range(len(ScoreArray))]
-    for i, soup in enumerate(soups):
-        if soup:
-            page_content = soup.text
-            for j, sent in enumerate(sentences):
-                print(
-                    f"Analyzing {i+1} of {len(soups)} soups with {j+1} of {len(sentences)} sentences........................"
-                )
-                tasks[i][j] = matchingScore(sent, page_content)
-        else:
-            print(
-                f"Analyzed {i+1} of soups (SOUP FAILED)........................"
-            )
-    ScoreArray = await asyncio.gather(*tasks, return_exceptions=True)
-    return ScoreArray
+        return f"Input length ({lengt}) is satisified."
+
+
+def extract_text_from_pdf(pdf_path):
+    doc = fitz.open(pdf_path)
+    text = ""
+    for page in doc:
+        text += page.get_text()
+    return text
+
+
+WORD = re.compile(r"\w+")
+model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+
+
+# returns cosine similarity of two vectors
+# input: two vectors
+# output: integer between 0 and 1.
+# def get_cosine(vec1, vec2):
+#     intersection = set(vec1.keys()) & set(vec2.keys())
+
+#     # calculating numerator
+#     numerator = sum([vec1[x] * vec2[x] for x in intersection])
+
+#     # calculating denominator
+#     sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
+#     sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
+#     denominator = math.sqrt(sum1) * math.sqrt(sum2)
+
+#     # checking for divide by zero
+#     if denominator == 0:
+#         return 0.0
+#     else:
+#         return float(numerator) / denominator
+
+
+# # converts given text into a vector
+# def text_to_vector(text):
+#     # uses the Regular expression above and gets all words
+#     words = WORD.findall(text)
+#     # returns a counter of all the words (count of number of occurences)
+#     return Counter(words)
+
+
+# # returns cosine similarity of two words
+# # uses: text_to_vector(text) and get_cosine(v1,v2)
+# def cosineSim(text1, text2):
+#     vector1 = text_to_vector(text1)
+#     vector2 = text_to_vector(text2)
+#     # print vector1,vector2
+#     cosine = get_cosine(vector1, vector2)
+#     return cosine
+
+
+# def cos_sim_torch(embedding_1, embedding_2):
+#     return util.pytorch_cos_sim(embedding_1, embedding_2).item()
+
+
+# def embed_text(text):
+#     return model.encode(text, convert_to_tensor=True)
+
+
+# def sentence_similarity(text1, text2):
+#     embedding_1 = model.encode(text1, convert_to_tensor=True)
+#     embedding_2 = model.encode(text2, convert_to_tensor=True)
+
+#     o = util.pytorch_cos_sim(embedding_1, embedding_2)
+#     return o.item()
+
+
+# def get_soup_requests(url):
+#     page = requests.get(url)
+#     if page.status_code == 200:
+#         soup = BeautifulSoup(page.content, "html.parser")
+#         return soup
+#     print("HTML soup failed")
+#     return None
+
+
+# def get_soup_httpx(url):
+#     client = httpx.Client(timeout=30)
+#     try:
+#         page = client.get(url)
+#         if page.status_code == httpx.codes.OK:
+#             soup = BeautifulSoup(page.content, "html.parser")
+#             return soup
+#     except:
+#         print("HTTPx soup failed")
+#         return None
+
+
+# def getSentences(text):
+#     from nltk.tokenize import sent_tokenize
+
+#     sents = sent_tokenize(text)
+#     two_sents = []
+#     for i in range(len(sents)):
+#         if (i % 2) == 0:
+#             two_sents.append(sents[i])
+#         else:
+#             two_sents[len(two_sents) - 1] += " " + sents[i]
+#     return two_sents
+
+
+# def googleSearch(
+#     plag_option,
+#     sentences,
+#     urlCount,
+#     scoreArray,
+#     urlList,
+#     sorted_date,
+#     domains_to_skip,
+#     api_key,
+#     cse_id,
+#     **kwargs,
+# ):
+#     service = build("customsearch", "v1", developerKey=api_key)
+#     for i, sentence in enumerate(sentences):
+#         results = (
+#             service.cse()
+#             .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
+#             .execute()
+#         )
+#         if "items" in results and len(results["items"]) > 0:
+#             for count, link in enumerate(results["items"]):
+#                 # stop after 3 pages
+#                 if count >= 3:
+#                     break
+#                 # skip user selected domains
+#                 if any(
+#                     ("." + domain) in link["link"] for domain in domains_to_skip
+#                 ):
+#                     continue
+#                 # clean up snippet of '...'
+#                 snippet = link["snippet"]
+#                 ind = snippet.find("...")
+#                 if ind < 20 and ind > 9:
+#                     snippet = snippet[ind + len("... ") :]
+#                 ind = snippet.find("...")
+#                 if ind > len(snippet) - 5:
+#                     snippet = snippet[:ind]
+
+#                 # update cosine similarity between snippet and given text
+#                 url = link["link"]
+#                 if url not in urlList:
+#                     urlList.append(url)
+#                     scoreArray.append([0] * len(sentences))
+#                 urlCount[url] = urlCount[url] + 1 if url in urlCount else 1
+#                 if plag_option == "Standard":
+#                     scoreArray[urlList.index(url)][i] = cosineSim(
+#                         sentence, snippet
+#                     )
+#                 else:
+#                     scoreArray[urlList.index(url)][i] = sentence_similarity(
+#                         sentence, snippet
+#                     )
+#         else:
+#             print("Google Search failed")
+#     return urlCount, scoreArray
+
+
+# def getQueries(text, n):
+#     # return n-grams of size n
+#     words = text.split()
+#     return [words[i : i + n] for i in range(len(words) - n + 1)]
+
+
+# def print2D(array):
+#     print(np.array(array))
+
+
+# def removePunc(text):
+#     res = re.sub(r"[^\w\s]", "", text)
+#     return res
+
+
+# async def get_url_data(url, client):
+#     try:
+#         r = await client.get(url)
+#         # print(r.status_code)
+#         if r.status_code == 200:
+#             # print("in")
+#             soup = BeautifulSoup(r.content, "html.parser")
+#             return soup
+#     except Exception:
+#         print("HTTPx parallel soup failed")
+#         return None
+
+
+# async def parallel_scrap(urls):
+#     async with httpx.AsyncClient(timeout=30) as client:
+#         tasks = []
+#         for url in urls:
+#             tasks.append(get_url_data(url=url, client=client))
+#         results = await asyncio.gather(*tasks, return_exceptions=True)
+#     return results
+
+
+# class TimeoutError(Exception):
+#     pass
+
+
+# def matchingScore(sentence, content):
+#     if sentence in content:
+#         return 1
+#     sentence = removePunc(sentence)
+#     content = removePunc(content)
+#     if sentence in content:
+#         return 1
+#     else:
+#         n = 5
+#         ngrams = getQueries(sentence, n)
+#         if len(ngrams) == 0:
+#             return 0
+#         matched = [x for x in ngrams if " ".join(x) in content]
+#     return len(matched) / len(ngrams)
+
+
+# # def matchingScoreWithTimeout(sentence, content):
+# #     def timeout_handler():
+# #         raise TimeoutError("Function timed out")
+
+# #     timer = threading.Timer(10, timeout_handler)  # Set a timer for 2 seconds
+# #     timer.start()
+# #     try:
+# #         score = sentence_similarity(sentence, content)
+# #         # score = matchingScore(sentence, content)
+# #         timer.cancel()  # Cancel the timer if calculation completes before timeout
+# #         return score
+# #     except TimeoutError:
+# #         return 0
+
+
+# # async def matchingScoreAsync(sentences, content, content_idx, ScoreArray):
+# #     content = removePunc(content)
+# #     for j, sentence in enumerate(sentences):
+# #         sentence = removePunc(sentence)
+# #         if sentence in content:
+# #             ScoreArray[content_idx][j] = 1
+# #         else:
+# #             n = 5
+# #             ngrams = getQueries(sentence, n)
+# #             if len(ngrams) == 0:
+# #                 return 0
+# #             matched = [x for x in ngrams if " ".join(x) in content]
+# #             ScoreArray[content_idx][j] = len(matched) / len(ngrams)
+# #     print(
+# #         f"Analyzed {content_idx+1} of soups (SOUP SUCCEEDED)........................"
+# #     )
+# #     return ScoreArray
+
+
+# async def matchingScoreAsync(
+#     sentences, content, content_idx, ScoreArray, model, util
+# ):
+#     content = removePunc(content)
+#     for j, sentence in enumerate(sentences):
+#         sentence = removePunc(sentence)
+#         similarity_score = sentence_similarity(sentence, content, model, util)
+#         ScoreArray[content_idx][j] = similarity_score
+#     print(
+#         f"Analyzed {content_idx+1} of contents (CONTENT ANALYZED)........................"
+#     )
+#     return ScoreArray
+
+
+# async def parallel_analyze(soups, sentences, ScoreArray):
+#     tasks = []
+#     for i, soup in enumerate(soups):
+#         if soup:
+#             page_content = soup.text
+#             tasks.append(
+#                 matchingScoreAsync(sentences, page_content, i, ScoreArray)
+#             )
+#         else:
+#             print(
+#                 f"Analyzed {i+1} of soups (SOUP FAILED)........................"
+#             )
+#     ScoreArray = await asyncio.gather(*tasks, return_exceptions=True)
+#     return ScoreArray
+
+
+# async def parallel_analyze_2(soups, sentences, ScoreArray):
+#     tasks = [[0] * len(ScoreArray[0]) for i in range(len(ScoreArray))]
+#     for i, soup in enumerate(soups):
+#         if soup:
+#             page_content = soup.text
+#             for j, sent in enumerate(sentences):
+#                 print(
+#                     f"Analyzing {i+1} of {len(soups)} soups with {j+1} of {len(sentences)} sentences........................"
+#                 )
+#                 tasks[i][j] = sentence_similarity(sent, page_content)
+#         else:
+#             print(
+#                 f"Analyzed {i+1} of soups (SOUP FAILED)........................"
+#             )
+#     ScoreArray = await asyncio.gather(*tasks, return_exceptions=True)
+#     return ScoreArray