Classification Report:
precision recall f1-score support
fragment 0.93 0.94 0.94 597
statement 0.87 0.88 0.88 1811
question 0.94 0.96 0.95 1786
command 0.91 0.90 0.90 1296
rhetorical question 0.72 0.59 0.65 174
rhetorical command 0.70 0.74 0.72 108
intonation-dependent utterance 0.58 0.55 0.57 273
accuracy 0.89 6045
macro avg 0.81 0.79 0.80 6045
weighted avg 0.89 0.89 0.89 6045
Run:
import torch
from transformers import RobertaTokenizerFast, RobertaForSequenceClassification, TextClassificationPipeline
from test_case_IC_iq import test_cases
import os
from datetime import datetime
# Check if GPU is available
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")
model_name = "jooni22/custom-dst-roberta-base"
loaded_tokenizer = RobertaTokenizerFast.from_pretrained(model_name)
loaded_model = RobertaForSequenceClassification.from_pretrained(
model_name,
num_labels=7,
id2label={
0: "fragment",
1: "statement",
2: "question",
3: "command",
4: "rhetorical question",
5: "rhetorical command",
6: "intonation-dependent utterance"
},
label2id={
"fragment": 0,
"statement": 1,
"question": 2,
"command": 3,
"rhetorical question": 4,
"rhetorical command": 5,
"intonation-dependent utterance": 6
}
).to(device)
# Using Pipeline
text_classifier = TextClassificationPipeline(
tokenizer=loaded_tokenizer,
model=loaded_model,
device=0 if device == "cuda" else -1, # Use GPU if available
top_k=None # This returns all scores
)
# Function to classify a single text
def classify_text(text):
preds_list = text_classifier(text)
best_pred = preds_list[0][0] # Access the first prediction of the first (and only) input
# Treat "question", "command", "rhetorical question", and "rhetorical command" as "question"
if best_pred['label'] in ["question", "command", "rhetorical question", "rhetorical command"]:
return "question", best_pred['score']
else:
return best_pred['label'], best_pred['score']
# Process test cases
correct_predictions = 0
total_cases = len(test_cases)
# Prepare results string
results = []
for text, expected_is_question in test_cases:
label, score = classify_text(text)
predicted_is_question = label == "question"
is_correct = predicted_is_question == expected_is_question
if is_correct:
correct_predictions += 1
result = f"Text: {text}\n"
result += f"Predicted: {'Question' if predicted_is_question else 'Not a question'} (Label: {label}, Score: {score:.4f})\n"
result += f"Expected: {'Question' if expected_is_question else 'Not a question'}\n"
result += f"Correct: {'Yes' if is_correct else 'No'}\n"
result += "-" * 50 + "\n"
results.append(result)
print(result)
# Calculate accuracy
accuracy = correct_predictions / total_cases
print(f"\nAccuracy: {accuracy:.2%}")
