Update app.py

app.py

@@ -21,6 +21,23 @@ clf = joblib.load(f'data/gpt2-large-model')
 
 CROSS_ENTROPY = torch.nn.CrossEntropyLoss(reduction='none')
 
+example = """\
+The perplexity (PPL) is commonly used as a metric for evaluating the performance of language models (LM). It is defined as the \
+exponential of the negative average log-likelihood of the text under the LM. A lower PPL indicates that the language model is more confident \
+in its predictions, and is therefore considered to be a better model. The training of LMs is carried out on large-scale text corpora, it can \
+be considered that it has learned some common language patterns and text structures. Therefore, PPL can be used to measure how well a text \
+conforms to common characteristics.
+
+I used all variants of the open-source GPT-2 model except xl size to compute the PPL (both text-level and sentence-level PPLs) of the collected \
+texts. It is observed that, regardless of whether it is at the text level or the sentence level, the content generated by LLMs have relatively \
+lower PPLs compared to the text written by humans. LLM captured common patterns and structures in the text it was trained on, and is very good at \
+reproducing them. As a result, text generated by LLMs have relatively concentrated low PPLs.
+
+Humans have the ability to express themselves in a wide variety of ways, depending on the context, audience, and purpose of the text they are \
+writing. This can include using creative or imaginative elements, such as metaphors, similes, and unique word choices, which can make it more \
+difficult for GPT2 to predict. The PPL distributions of text written by humans and text generated by LLMs are shown in the figure below.\
+"""
+
 
 def gpt2_features(text, tokenizer, model, sent_cut):
     # Tokenize
@@ -87,7 +104,7 @@ def gpt2_features(text, tokenizer, model, sent_cut):
     return ppls + counter  # type: ignore
 
 
-def predict(features, classifier, id_to_label):
+def predict_out(features, classifier, id_to_label):
     x = np.asarray([features])
     pred = classifier.predict(x)[0]
     prob = classifier.predict_proba(x)[0, pred]
@@ -97,50 +114,44 @@ def predict(features, classifier, id_to_label):
 def predict(text):
     with torch.no_grad():
         feats = gpt2_features(text, tokenizer, model, sent_cut_en)
-    out = predict(feats, clf, ['Human Written', 'LLM Generated'])
+    out = predict_out(feats, clf, ['Human Written', 'LLM Generated'])
     return out
 
 
-print("Building Gradio Interface...")
 with gr.Blocks() as demo:
     gr.Markdown(
         """
-        ## ChatGPT Detector 🔬 (Linguistic version / 语言学版)
-
-        Visit our project on Github: [chatgpt-comparison-detection project](https://github.com/Hello-SimpleAI/chatgpt-comparison-detection)<br>
-        欢迎在 Github 上关注我们的 [ChatGPT 对比与检测项目](https://github.com/Hello-SimpleAI/chatgpt-comparison-detection)<br>
-        We provide three kinds of detectors, all in Bilingual / 我们提供了三个版本的检测器，且都支持中英文:
-        - [QA version / 问答版](https://www.modelscope.cn/studios/simpleai/chatgpt-detector-qa)<br>
-            detect whether an **answer** is generated by ChatGPT for certain **question**, using PLM-based classifiers / 判断某个**问题的回答**是否由ChatGPT生成，使用基于PTM的分类器来开发;
-        - [Sinlge-text version / 独立文本版](https://www.modelscope.cn/studios/simpleai/chatgpt-detector-single)<br>
-            detect whether a piece of text is ChatGPT generated, using PLM-based classifiers / 判断**单条文本**是否由ChatGPT生成，使用基于PTM的分类器来开发;
-        - [**Linguistic version / 语言学版** (👈 Current / 当前使用)](https://www.modelscope.cn/studios/simpleai/chatgpt-detector-ling)<br>
-            detect whether a piece of text is ChatGPT generated, using linguistic features / 判断**单条文本**是否由ChatGPT生成，使用基于语言学特征的模型来开发;
+        ## Detect text generated using LLMs 🤖
 
-        """
-    )
+        Linguistic features such as Perplexity and other SOTA methods such as GLTR were used to classify between Human written and LLM Generated \
+        texts. This solution scored an ROC of 0.956 and 8th position in the DAIGT LLM Competition on Kaggle. Fork of and credits to this github repo
 
-    gr.Markdown(
-        """
-        ## Introduction:
-        Two Logistic regression models trained with two kinds of features:
-        1. [GLTR](https://aclanthology.org/P19-3019) Test-2, Language model predict token rank top-k buckets, top 10, 10-100, 100-1000, 1000+.
-        2. PPL-based, text ppl, sentence ppl, etc.
+        Competition: [https://www.kaggle.com/competitions/llm-detect-ai-generated-text/leaderboard](https://www.kaggle.com/competitions/llm-detect-ai-generated-text/leaderboard)
+        Solution WriteUp: [https://www.kaggle.com/competitions/llm-detect-ai-generated-text/discussion/470224](https://www.kaggle.com/competitions/llm-detect-ai-generated-text/discussion/470224)
 
-        English LM is [GPT2-small](https://huggingface.co/gpt2).
+        ### Linguistic Analysis: Language Model Perplexity
+        The perplexity (PPL) is commonly used as a metric for evaluating the performance of language models (LM). It is defined as the exponential \
+        of the negative average log-likelihood of the text under the LM. A lower PPL indicates that the language model is more confident in its \
+        predictions, and is therefore considered to be a better model. The training of LMs is carried out on large-scale text corpora, it can \
+        be considered that it has learned some common language patterns and text structures. Therefore, PPL can be used to measure how \
+        well a text conforms to common characteristics.
 
-        Note: Providing more text to the `Text` box can make the prediction more accurate!
+        ### GLTR: Giant Language Model Test Room
+        This idea originates from the following paper: arxiv.org/pdf/1906.04043.pdf. It studies 3 tests to compute features of an input text. Their \
+        major assumption is that to generate fluent and natural-looking text, most decoding strategies sample high probability tokens from the head \
+        of the distribution. I selected the most powerful Test-2 feature, which is the number of tokens in the Top-10, Top-100, Top-1000, and 1000+ \
+        ranks from the LM predicted probability distributions.
+
+        ### Modelling
+        Scikit-learn's VotingClassifier consisting of XGBClassifier, LGBMClassifier, CatBoostClassifier and RandomForestClassifier with default parameters
         """
     )
-    a1 = gr.Textbox(
-        lines=5, label='Text',
-        value="There are a few things that can help protect your credit card information from being misused when you give it to a restaurant or any other business:\n\nEncryption: Many businesses use encryption to protect your credit card information when it is being transmitted or stored. This means that the information is transformed into a code that is difficult for anyone to read without the right key."
-    )
+    a1 = gr.Textbox( lines=7, label='Text', value=example )
     button1 = gr.Button("🤖 Predict!")
-    gr.Markdown("GLTR")
-    label1_gltr = gr.Textbox(lines=1, label='GLTR Predicted Label 🎃')
-    score1_gltr = gr.Textbox(lines=1, label='GLTR Probability')
+    gr.Markdown("Prediction:")
+    label1 = gr.Textbox(lines=1, label='Predicted Label')
+    score1 = gr.Textbox(lines=1, label='Predicted Probability')
 
-    button1.click(predict, inputs=[a1], outputs=[label1_gltr, score1_gltr])
+    button1.click(predict, inputs=[a1], outputs=[label1, score1])
 
 demo.launch()