keywords extraction and intention classification

app.py

@@ -24,6 +24,8 @@ login(os.environ["HF_TOKEN"])
 dt = datetime.datetime.now()
 print(dt)
 print("loading models")
+classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
+
 tokenizer = GPT2Tokenizer.from_pretrained('microsoft/DialoGPT-medium')
 original_model = GPT2LMHeadModel.from_pretrained('microsoft/DialoGPT-medium')
 untethered_model = GPT2LMHeadModel.from_pretrained('zmbfeng/untethered_20240225_epochs_500')
@@ -86,7 +88,37 @@ def create_response_paraphrase(input_str, max_length,num_return_sequences):
     #     results.append(line)
     # return results
     return result_output_str
-
+import string
+def contains_digit_or_punctuation(s):
+    return any(char.isdigit() or char in string.punctuation for char in s)
+rake = Rake()
+def create_response_keywords_extraction(input_str):
+    rake.extract_keywords_from_text(input_str)
+    keywords_with_scores = rake.get_ranked_phrases_with_scores()
+    filtered_keywords = []
+    seen_keywords = set()
+    for score, keyword in keywords_with_scores:
+        # Apply filters: score must be greater than 1, keyword must not contain digits or punctuation
+        if score > 1 and not contains_digit_or_punctuation(keyword) and keyword not in seen_keywords:
+            filtered_keywords.append((score, keyword))
+            seen_keywords.add(keyword)
+    output_string=""
+    for score, keyword in filtered_keywords:
+        #print(f"Score: {score}, Keyword: {keyword}")
+        output_string= output_string + f"Score: {score}, Keyword: {keyword} <br/>"
+
+    return output_string
+
+def create_response_intention_classification(input_str):
+    labels = ["dialogue", "long content generation"]
+
+    # Perform classification
+    output_string=""
+    result = classifier(input_str, labels)
+    for label, score in zip(result["labels"], result["scores"]):
+        output_string= output_string + f"Label: {label}, Score: {score:.4f} <br/>"
+
+    return output_string
 
 interface_question_generation = gr.Interface(fn=create_response_question_generation,
     title="Question Generation",
@@ -118,9 +150,26 @@ interface_paraphrase = gr.Interface(fn=create_response_paraphrase,
     ],
     outputs="html"
     )
+interface_extract_keywords = gr.Interface(fn=create_response_keywords_extraction,
+    title="Extract Keywords",
+    description="Extract Keywords ",
+    #examples=examples,
+    inputs=[
+   gr.Textbox(label="input text here", lines=3, value="It is truly a great cosmic paradox that one of the best teachers in all of life turns out to be death. No person or situation could ever teach you as much as death has to teach you. "),
+    ],
+    outputs="html"
+    )
+interface_intention_classification = gr.Interface(fn=create_response_intention_classification,
+    title="Intention Classification",
+    description="Find if question intention is short dialog or long content generation. How are you? versus WWhat are the implications of quantum computing on global security?  (difference not very dramatic as of now)",
+    #examples=examples,
+    inputs=[
+   gr.Textbox(label="input text here", lines=3, value="What are the implications of quantum computing on global security?"),
+    ],
+    outputs="html"
+    )
 
 
-
-demo = gr.TabbedInterface([interface_question_generation, interface_paraphrase], ["Question Generation", "Paraphrase"])
+demo = gr.TabbedInterface([interface_question_generation, interface_paraphrase,interface_extract_keywords,interface_intention_classification], ["Question Generation", "Paraphrase", "Keywords Extraction", "Intention Classification"])
 
 demo.launch()