Merge branch 'main' into demo

analysis.py

@@ -62,7 +62,10 @@ def depth_analysis(input_text):
         "punctuation_diversity": (-0.21875, 0.53125),
         "type_token_ratio": (0.33002482852189063, 1.0894414982357028),
         "calculate_perplexity": (-25.110544681549072, 82.4620680809021),
-        "calculate_syntactic_tree_depth": (1.8380681818181812, 10.997159090909092),
+        "calculate_syntactic_tree_depth": (
+            1.8380681818181812,
+            10.997159090909092,
+        ),
         "hapax_legomena_ratio": (0.0830971690138207, 1.0302715687215778),
         "mtld": (-84.03125000000001, 248.81875000000002),
     }
@@ -72,14 +75,17 @@ def depth_analysis(input_text):
     determiner_use = determiners_frequency(input_text, nlp)
     punctuation_variety = punctuation_diversity(input_text)
     sentence_depth = calculate_syntactic_tree_depth(input_text, nlp)
-    perplexity = calculate_perplexity(input_text, gpt2_model, gpt2_tokenizer, device)
+    perplexity = calculate_perplexity(
+        input_text, gpt2_model, gpt2_tokenizer, device
+    )
     lexical_diversity = type_token_ratio(input_text)
     unique_words = hapax_legomena_ratio(input_text)
     vocabulary_stability = mtld(input_text)
 
     # normalize between 0 and 100
     vocabulary_level_norm = normalize(
-        vocabulary_level, *usual_ranges["estimated_slightly_difficult_words_ratio"]
+        vocabulary_level,
+        *usual_ranges["estimated_slightly_difficult_words_ratio"],
     )
     entity_ratio_norm = normalize(entity_ratio, *usual_ranges["entity_density"])
     determiner_use_norm = normalize(
@@ -91,12 +97,18 @@ def depth_analysis(input_text):
     lexical_diversity_norm = normalize(
         lexical_diversity, *usual_ranges["type_token_ratio"]
     )
-    unique_words_norm = normalize(unique_words, *usual_ranges["hapax_legomena_ratio"])
-    vocabulary_stability_norm = normalize(vocabulary_stability, *usual_ranges["mtld"])
+    unique_words_norm = normalize(
+        unique_words, *usual_ranges["hapax_legomena_ratio"]
+    )
+    vocabulary_stability_norm = normalize(
+        vocabulary_stability, *usual_ranges["mtld"]
+    )
     sentence_depth_norm = normalize(
         sentence_depth, *usual_ranges["calculate_syntactic_tree_depth"]
     )
-    perplexity_norm = normalize(perplexity, *usual_ranges["calculate_perplexity"])
+    perplexity_norm = normalize(
+        perplexity, *usual_ranges["calculate_perplexity"]
+    )
 
     features = {
         "Lexical Diversity": lexical_diversity_norm,
@@ -161,7 +173,8 @@ def depth_analysis(input_text):
                         path=Path.unit_regular_polygon(num_vars),
                     )
                     spine.set_transform(
-                        Affine2D().scale(0.5).translate(0.5, 0.5) + self.transAxes
+                        Affine2D().scale(0.5).translate(0.5, 0.5)
+                        + self.transAxes
                     )
                     return {"polar": spine}
 
@@ -172,14 +185,19 @@ def depth_analysis(input_text):
     theta = radar_factory(N, frame="polygon")
     data = features.values()
     labels = features.keys()
-    fig, ax = plt.subplots(subplot_kw=dict(projection="radar"), figsize=(7.5, 5))
+    fig, ax = plt.subplots(
+        subplot_kw=dict(projection="radar"), figsize=(7.5, 5)
+    )
     ax.plot(theta, data)
     ax.fill(theta, data, alpha=0.4)
     ax.set_varlabels(labels)
 
     rgrids = np.linspace(0, 100, num=6)
     ax.set_rgrids(
-        rgrids, labels=[f"{round(r)}%" for r in rgrids], fontsize=8, color="black"
+        rgrids,
+        labels=[f"{round(r)}%" for r in rgrids],
+        fontsize=8,
+        color="black",
     )
     ax.grid(True, color="black", linestyle="-", linewidth=0.5, alpha=0.5)
 

app.py

@@ -47,6 +47,7 @@ def main(
     month_to,
     day_to,
     domains_to_skip,
+    source_block_size,
 ):
 
     # formatted_tokens = plagiarism_check(
@@ -70,6 +71,7 @@ def main(
         month_to,
         day_to,
         domains_to_skip,
+        source_block_size,
     )
     depth_analysis_plot = depth_analysis(input)
     bc_score = predict_bc_scores(input)
@@ -153,6 +155,13 @@ with gr.Blocks() as demo:
             plag_option = gr.Radio(
                 ["Standard", "Advanced"], label="Choose an option please."
             )
+    with gr.Row():
+        source_block_size = gr.Dropdown(
+            choices=["1", "2", "3", "Paragraph"],
+            label="Source Check Granularity",
+            value="2",
+            interactive=True,
+        )
 
     with gr.Row():
         with gr.Column():
@@ -307,6 +316,7 @@ with gr.Blocks() as demo:
             month_to,
             day_to,
             domains_to_skip,
+            source_block_size,
         ],
         outputs=[
             bcLabel,
@@ -347,6 +357,7 @@ with gr.Blocks() as demo:
             month_to,
             day_to,
             domains_to_skip,
+            source_block_size,
         ],
         outputs=[
             sentenceBreakdown,

plagiarism.py

@@ -16,37 +16,36 @@ WORD = re.compile(r"\w+")
 model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 
 
-# returns cosine similarity of two vectors
-# input: two vectors
-# output: integer between 0 and 1.
-def get_cosine(vec1, vec2):
-    intersection = set(vec1.keys()) & set(vec2.keys())
-
-    # calculating numerator
-    numerator = sum([vec1[x] * vec2[x] for x in intersection])
-
-    # calculating denominator
-    sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
-    sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
-    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+months = {
+    "January": "01",
+    "February": "02",
+    "March": "03",
+    "April": "04",
+    "May": "05",
+    "June": "06",
+    "July": "07",
+    "August": "08",
+    "September": "09",
+    "October": "10",
+    "November": "11",
+    "December": "12",
+}
 
-    # checking for divide by zero
-    if denominator == 0:
-        return 0.0
-    else:
-        return float(numerator) / denominator
+color_map = [
+    "#cf2323",
+    "#eb9d59",
+    "#c2ad36",
+    "#e1ed72",
+    "#c2db76",
+    "#a2db76",
+]
 
 
-# converts given text into a vector
 def text_to_vector(text):
-    # uses the Regular expression above and gets all words
     words = WORD.findall(text)
-    # returns a counter of all the words (count of number of occurences)
     return Counter(words)
 
 
-# returns cosine similarity of two words
-# uses: text_to_vector(text) and get_cosine(v1,v2)
 def cosineSim(text1, text2):
     vector1 = text_to_vector(text1)
     vector2 = text_to_vector(text2)
@@ -55,132 +54,61 @@ def cosineSim(text1, text2):
     return cosine
 
 
-def cos_sim_torch(embedding_1, embedding_2):
-    return util.pytorch_cos_sim(embedding_1, embedding_2).item()
-
-
-def embed_text(text):
-    return model.encode(text, convert_to_tensor=True)
-
-
-def sentence_similarity(text1, text2):
-    embedding_1 = model.encode(text1, convert_to_tensor=True)
-    embedding_2 = model.encode(text2, convert_to_tensor=True)
-
-    o = util.pytorch_cos_sim(embedding_1, embedding_2)
-    return o.item()
+def get_cosine(vec1, vec2):
+    intersection = set(vec1.keys()) & set(vec2.keys())
+    numerator = sum([vec1[x] * vec2[x] for x in intersection])
+    sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
+    sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
+    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+    if denominator == 0:
+        return 0.0
+    else:
+        return float(numerator) / denominator
 
 
-def google_search(
-    plag_option,
-    sentences,
-    url_count,
-    score_array,
-    url_list,
-    sorted_date,
-    domains_to_skip,
-    api_key,
-    cse_id,
-    **kwargs,
-):
-    service = build("customsearch", "v1", developerKey=api_key)
-    for i, sentence in enumerate(sentences):
-        results = (
-            service.cse()
-            .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
-            .execute()
-        )
-        if "items" in results and len(results["items"]) > 0:
-            for count, link in enumerate(results["items"]):
-                # stop after 3 pages
-                if count >= 3:
-                    break
-                # # skip user selected domains
-                # if any(
-                #     ("." + domain) in link["link"] for domain in domains_to_skip
-                # ):
-                #     continue
-                # clean up snippet of '...'
-                snippet = link["snippet"]
-                ind = snippet.find("...")
-                if ind < 20 and ind > 9:
-                    snippet = snippet[ind + len("... ") :]
-                ind = snippet.find("...")
-                if ind > len(snippet) - 5:
-                    snippet = snippet[:ind]
-
-                # update cosine similarity between snippet and given text
-                url = link["link"]
-                if url not in url_list:
-                    url_list.append(url)
-                    score_array.append([0] * len(sentences))
-                url_count[url] = url_count[url] + 1 if url in url_count else 1
-                if plag_option == "Standard":
-                    score_array[url_list.index(url)][i] = cosineSim(
-                        sentence, snippet
-                    )
+def split_sentence_blocks(text, size):
+    if size == "Paragraph":
+        blocks = text.split("\n")
+        return blocks
+    else:
+        blocks = []
+        size = int(size)
+        for para in text.split("\n\n"):
+            sents = sent_tokenize(para)
+            for i in range(len(sents)):
+                if (i % size) == 0:
+                    blocks.append(sents[i])
                 else:
-                    score_array[url_list.index(url)][i] = sentence_similarity(
-                        sentence, snippet
-                    )
-    return url_count, score_array
+                    blocks[int(i / size)] += " " + sents[i]
+        return blocks
 
 
-def split_sentence_blocks(text):
-    two_sents = []
-    for para in text.split("\n\n"):
-        sents = sent_tokenize(para)
-        for i in range(len(sents)):
-            if (i % 2) == 0:
-                two_sents.append(sents[i])
-            else:
-                two_sents[len(two_sents) - 1] += " " + sents[i]
-    return two_sents
+def build_date(year=2024, month="March", day=1):
+    return f"{year}{months[month]}{day}"
 
 
-months = {
-    "January": "01",
-    "February": "02",
-    "March": "03",
-    "April": "04",
-    "May": "05",
-    "June": "06",
-    "July": "07",
-    "August": "08",
-    "September": "09",
-    "October": "10",
-    "November": "11",
-    "December": "12",
-}
+def split_ngrams(text, n):
+    words = text.split()
+    return [words[i : i + n] for i in range(len(words) - n + 1)]
 
 
-def build_date(year=2024, month="March", day=1):
-    return f"{year}{months[month]}{day}"
+def sentence_similarity(text1, text2):
+    embedding_1 = model.encode(text1, convert_to_tensor=True)
+    embedding_2 = model.encode(text2, convert_to_tensor=True)
+    o = util.pytorch_cos_sim(embedding_1, embedding_2)
+    return o.item()
 
 
 async def get_url_data(url, client):
     try:
         r = await client.get(url)
-        # print(r.status_code)
         if r.status_code == 200:
-            # print("in")
             soup = BeautifulSoup(r.content, "html.parser")
             return soup
     except Exception:
         return None
 
 
-def remove_punc(text):
-    res = re.sub(r"[^\w\s]", "", text)
-    return res
-
-
-def split_ngrams(text, n):
-    # return n-grams of size n
-    words = text.split()
-    return [words[i : i + n] for i in range(len(words) - n + 1)]
-
-
 async def parallel_scrap(urls):
     async with httpx.AsyncClient(timeout=30) as client:
         tasks = []
@@ -209,11 +137,6 @@ def process_with_multiprocessing(input_data):
     return scores
 
 
-def print2d(array):
-    for row in array:
-        print(row)
-
-
 def map_sentence_url(sentences, score_array):
     sentenceToMaxURL = [-1] * len(sentences)
     for j in range(len(sentences)):
@@ -234,65 +157,59 @@ def map_sentence_url(sentences, score_array):
     return sentenceToMaxURL
 
 
-def html_highlight(
+def google_search(
     plag_option,
-    input,
-    year_from,
-    month_from,
-    day_from,
-    year_to,
-    month_to,
-    day_to,
+    sentences,
+    url_count,
+    score_array,
+    url_list,
+    sorted_date,
     domains_to_skip,
+    api_key,
+    cse_id,
+    **kwargs,
 ):
-    sentence_scores, url_scores = plagiarism_check(
-        plag_option,
-        input,
-        year_from,
-        month_from,
-        day_from,
-        year_to,
-        month_to,
-        day_to,
-        domains_to_skip,
-    )
-    color_map = [
-        "#cf2323",
-        "#eb9d59",
-        "#c2ad36",
-        "#e1ed72",
-        "#c2db76",
-        "#a2db76",
-    ]
-    font = "Roboto"
-    html_content = "<link href='https://fonts.googleapis.com/css?family=Roboto' rel='stylesheet'>\n<div style='font-family: {font}; border: 2px solid black; background-color: #333333; padding: 10px; color: #FFFFFF;'>"
-    prev_idx = None
-    combined_sentence = ""
-    for sentence, _, _, idx in sentence_scores:
-        if idx != prev_idx and prev_idx is not None:
-            color = color_map[prev_idx - 1]
-            index_part = f'<span style="background-color: {color}; padding: 2px;">[{prev_idx}]</span>'
-            formatted_sentence = f"<p>{combined_sentence} {index_part}</p>"
-            html_content += formatted_sentence
-            combined_sentence = ""
-        combined_sentence += " " + sentence
-        prev_idx = idx
-
-    if combined_sentence:
-        color = color_map[prev_idx - 1]
-        index_part = f'<span style="background-color: {color}; padding: 2px;">[{prev_idx}]</span>'
-        formatted_sentence = f"<p>{combined_sentence} {index_part}</p>"
-        html_content += formatted_sentence
-
-    html_content += "<hr>"
-    for url, score, idx in url_scores:
-        color = color_map[idx - 1]
-        formatted_url = f'<p style="background-color: {color}; padding: 5px;">({idx}) <b>{url}</b></p><p> --- Matching Score: {score}%</p>'
-        html_content += formatted_url
-
-    html_content += "</div>"
+    service = build("customsearch", "v1", developerKey=api_key)
+    for i, sentence in enumerate(sentences):
+        results = (
+            service.cse()
+            .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
+            .execute()
+        )
+        if "items" in results and len(results["items"]) > 0:
+            for count, link in enumerate(results["items"]):
+                # stop after 3 pages
+                if count >= 3:
+                    break
+                # skip user selected domains
+                if (domains_to_skip is not None) and any(
+                    ("." + domain) in link["link"] for domain in domains_to_skip
+                ):
+                    continue
+                # clean up snippet of '...'
+                snippet = link["snippet"]
+                ind = snippet.find("...")
+                if ind < 20 and ind > 9:
+                    snippet = snippet[ind + len("... ") :]
+                ind = snippet.find("...")
+                if ind > len(snippet) - 5:
+                    snippet = snippet[:ind]
 
-    return html_content
+                # update cosine similarity between snippet and given text
+                url = link["link"]
+                if url not in url_list:
+                    url_list.append(url)
+                    score_array.append([0] * len(sentences))
+                url_count[url] = url_count[url] + 1 if url in url_count else 1
+                if plag_option == "Standard":
+                    score_array[url_list.index(url)][i] = cosineSim(
+                        sentence, snippet
+                    )
+                else:
+                    score_array[url_list.index(url)][i] = sentence_similarity(
+                        sentence, snippet
+                    )
+    return url_count, score_array
 
 
 def plagiarism_check(
@@ -305,17 +222,20 @@ def plagiarism_check(
     month_to,
     day_to,
     domains_to_skip,
+    source_block_size,
 ):
     api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
     api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
+    # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
+    # api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
     # api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
     # api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
-    # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
+    api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
     cse_id = "851813e81162b4ed4"
 
     url_scores = []
     sentence_scores = []
-    sentences = split_sentence_blocks(input)
+    sentences = split_sentence_blocks(input, source_block_size)
     url_count = {}
     score_array = []
     url_list = []
@@ -384,3 +304,58 @@ def plagiarism_check(
             )
 
     return sentence_scores, url_scores
+
+
+def html_highlight(
+    plag_option,
+    input,
+    year_from,
+    month_from,
+    day_from,
+    year_to,
+    month_to,
+    day_to,
+    domains_to_skip,
+    source_block_size,
+):
+    sentence_scores, url_scores = plagiarism_check(
+        plag_option,
+        input,
+        year_from,
+        month_from,
+        day_from,
+        year_to,
+        month_to,
+        day_to,
+        domains_to_skip,
+        source_block_size,
+    )
+
+    html_content = "<link href='https://fonts.googleapis.com/css?family=Roboto' rel='stylesheet'>\n<div style='font-family: {font}; border: 2px solid black; background-color: #333333; padding: 10px; color: #FFFFFF;'>"
+    prev_idx = None
+    combined_sentence = ""
+    for sentence, _, _, idx in sentence_scores:
+        if idx != prev_idx and prev_idx is not None:
+            color = color_map[prev_idx - 1]
+            index_part = f'<span style="background-color: {color}; padding: 2px;">[{prev_idx}]</span>'
+            formatted_sentence = f"<p>{combined_sentence} {index_part}</p>"
+            html_content += formatted_sentence
+            combined_sentence = ""
+        combined_sentence += " " + sentence
+        prev_idx = idx
+
+    if combined_sentence:
+        color = color_map[prev_idx - 1]
+        index_part = f'<span style="background-color: {color}; padding: 2px;">[{prev_idx}]</span>'
+        formatted_sentence = f"<p>{combined_sentence} {index_part}</p>"
+        html_content += formatted_sentence
+
+    html_content += "<hr>"
+    for url, score, idx in url_scores:
+        color = color_map[idx - 1]
+        formatted_url = f'<p style="background-color: {color}; padding: 5px;">({idx}) <b>{url}</b></p><p> --- Matching Score: {score}%</p>'
+        html_content += formatted_url
+
+    html_content += "</div>"
+
+    return html_content

predictors.py

@@ -1,23 +1,11 @@
-import requests
-import httpx
 import torch
-import re
-from bs4 import BeautifulSoup
 import numpy as np
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import asyncio
-from evaluate import load
-from datetime import date
 import nltk
-from transformers import GPT2LMHeadModel, GPT2TokenizerFast
-import plotly.graph_objects as go
 import torch.nn.functional as F
 import nltk
-from unidecode import unidecode
-import time
 from scipy.special import softmax
 import yaml
-import os
 from utils import *
 import joblib
 from optimum.bettertransformer import BetterTransformer
@@ -64,24 +52,9 @@ tokenizers_1on1 = {}
 models_1on1 = {}
 for model_name, model in zip(mc_label_map, text_1on1_models):
     tokenizers_1on1[model_name] = AutoTokenizer.from_pretrained(model)
-    models_1on1[model_name] = AutoModelForSequenceClassification.from_pretrained(
-        model
-    ).to(device)
-
-
-bias_model_checker = AutoModelForSequenceClassification.from_pretrained(bias_checker_model_name)
-tokenizer = AutoTokenizer.from_pretrained(bias_checker_model_name)
-bias_model_checker = BetterTransformer.transform(bias_model_checker, keep_original_model=False)
-bias_checker = pipeline(
-    "text-classification",
-    model=bias_checker_model_name,
-    tokenizer=bias_checker_model_name,
-)
-gc.collect()
-bias_corrector = pipeline(
-    "text2text-generation", model=bias_corrector_model_name, accelerator="ort"
-
-)
+    models_1on1[model_name] = (
+        AutoModelForSequenceClassification.from_pretrained(model).to(device)
+    )
 
 # proxy models for explainability
 mini_bc_model_name = "polygraf-ai/bc-model-bert-mini"
@@ -90,7 +63,9 @@ bc_model_mini = AutoModelForSequenceClassification.from_pretrained(
     mini_bc_model_name
 ).to(device_needed)
 mini_humanizer_model_name = "polygraf-ai/quillbot-detector-bert-mini-9K"
-humanizer_tokenizer_mini = AutoTokenizer.from_pretrained(mini_humanizer_model_name)
+humanizer_tokenizer_mini = AutoTokenizer.from_pretrained(
+    mini_humanizer_model_name
+)
 humanizer_model_mini = AutoModelForSequenceClassification.from_pretrained(
     mini_humanizer_model_name
 ).to(device_needed)
@@ -289,9 +264,52 @@ def predict_mc(model, tokenizer, text):
         output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
         return output_norm
 
+
+def predict_mc_scores(input):
+    bc_scores = []
+    mc_scores = []
+
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
+    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    for i in range(samples_len_bc):
+        cleaned_text_bc = remove_special_characters(segments_bc[i])
+        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
+        bc_scores.append(bc_score)
+    bc_scores_array = np.array(bc_scores)
+    average_bc_scores = np.mean(bc_scores_array, axis=0)
+    bc_score_list = average_bc_scores.tolist()
+    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
+    segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    samples_len_mc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    )
+    for i in range(samples_len_mc):
+        cleaned_text_mc = remove_special_characters(segments_mc[i])
+        mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
+        mc_scores.append(mc_score)
+    mc_scores_array = np.array(mc_scores)
+    average_mc_scores = np.mean(mc_scores_array, axis=0)
+    mc_score_list = average_mc_scores.tolist()
+    mc_score = {}
+    for score, label in zip(mc_score_list, mc_label_map):
+        mc_score[label.upper()] = score
+
+    sum_prob = 1 - bc_score["HUMAN"]
+    for key, value in mc_score.items():
+        mc_score[key] = value * sum_prob
+    if sum_prob < 0.01:
+        mc_score = {}
+
+    return mc_score
+
+
 def predict_bc_scores(input):
     bc_scores = []
-    samples_len_bc = len(split_text_allow_complete_sentences_nltk(input, type_det="bc"))
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
     segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
     for i in range(samples_len_bc):
         cleaned_text_bc = remove_special_characters(segments_bc[i])
@@ -300,7 +318,9 @@ def predict_bc_scores(input):
     bc_scores_array = np.array(bc_scores)
     average_bc_scores = np.mean(bc_scores_array, axis=0)
     bc_score_list = average_bc_scores.tolist()
-    print(f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}")
+    print(
+        f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}"
+    )
     # isotonic regression calibration
     ai_score = iso_reg.predict([bc_score_list[1]])[0]
     human_score = 1 - ai_score
@@ -335,7 +355,9 @@ def predict_1on1_combined(input):
 
 
 def predict_1on1_single(input, model):
-    predictions = predict_1on1(models_1on1[model], tokenizers_1on1[model], input)[1]
+    predictions = predict_1on1(
+        models_1on1[model], tokenizers_1on1[model], input
+    )[1]
     return predictions
 
 
@@ -347,7 +369,9 @@ def predict_mc_scores(input, models):
     print(f"Models to Test: {models}")
     # BC SCORE
     bc_scores = []
-    samples_len_bc = len(split_text_allow_complete_sentences_nltk(input, type_det="bc"))
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
     segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
     for i in range(samples_len_bc):
         cleaned_text_bc = remove_special_characters(segments_bc[i])
@@ -356,24 +380,30 @@ def predict_mc_scores(input, models):
     bc_scores_array = np.array(bc_scores)
     average_bc_scores = np.mean(bc_scores_array, axis=0)
     bc_score_list = average_bc_scores.tolist()
-    print(f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}")
+    print(
+        f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}"
+    )
     # isotonic regression calibration
     ai_score = iso_reg.predict([bc_score_list[1]])[0]
     human_score = 1 - ai_score
     bc_score = {"AI": ai_score, "HUMAN": human_score}
     print(f"Calibration BC scores: AI: {ai_score}, HUMAN: {human_score}")
-    
+
     # MC SCORE
     if len(models) > 1:
         print("Starting MC")
         mc_scores = []
-        segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+        segments_mc = split_text_allow_complete_sentences_nltk(
+            input, type_det="mc"
+        )
         samples_len_mc = len(
             split_text_allow_complete_sentences_nltk(input, type_det="mc")
         )
         for i in range(samples_len_mc):
             cleaned_text_mc = remove_special_characters(segments_mc[i])
-            mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
+            mc_score = predict_mc(
+                text_mc_model, text_mc_tokenizer, cleaned_text_mc
+            )
             mc_scores.append(mc_score)
         mc_scores_array = np.array(mc_scores)
         average_mc_scores = np.mean(mc_scores_array, axis=0)
@@ -383,7 +413,9 @@ def predict_mc_scores(input, models):
             mc_score[label.upper()] = score
 
         mc_score = {
-            key: mc_score[key.upper()] for key in models if key.upper() in mc_score
+            key: mc_score[key.upper()]
+            for key in models
+            if key.upper() in mc_score
         }
         total = sum(mc_score.values())
         # Normalize each value by dividing it by the total
@@ -391,14 +423,16 @@ def predict_mc_scores(input, models):
         sum_prob = 1 - bc_score["HUMAN"]
         for key, value in mc_score.items():
             mc_score[key] = value * sum_prob
-        print('MC Score:',mc_score)
+        print("MC Score:", mc_score)
         if sum_prob < 0.01:
             mc_score = {}
 
     elif len(models) == 1:
         print("Starting 1on1")
         mc_scores = []
-        segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+        segments_mc = split_text_allow_complete_sentences_nltk(
+            input, type_det="mc"
+        )
         samples_len_mc = len(
             split_text_allow_complete_sentences_nltk(input, type_det="mc")
         )

requirements.txt

@@ -16,7 +16,7 @@ joblib
 evaluate
 tensorflow
 keras
-spacy
+spacy==3.7.2
 textstat
 plotly
 tqdm

utils.py

@@ -1,28 +1,11 @@
-from urllib.request import urlopen, Request
-from googleapiclient.discovery import build
-import requests
-import httpx
 import re
-from bs4 import BeautifulSoup
-import re, math
-from collections import Counter
-import numpy as np
-import asyncio
-import nltk
+import re
 from sentence_transformers import SentenceTransformer, util
-import threading
-import torch
 import re
-import numpy as np
-import asyncio
-from datetime import date
-import nltk
 from unidecode import unidecode
-from scipy.special import softmax
 from transformers import AutoTokenizer
 import yaml
 import fitz
-import os
 
 
 def remove_accents(input_str):
@@ -63,9 +46,6 @@ def update_character_count(text):
     return f"{len(text)} characters"
 
 
-nltk.download("punkt")
-
-
 with open("config.yaml", "r") as file:
     params = yaml.safe_load(file)
 
@@ -92,4 +72,4 @@ def extract_text_from_pdf(pdf_path):
 
 
 WORD = re.compile(r"\w+")
-model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")