custom-dst-roberta-base

Classification Report:

                                precision    recall  f1-score   support

                      fragment       0.93      0.94      0.94       597
                     statement       0.87      0.88      0.88      1811
                      question       0.94      0.96      0.95      1786
                       command       0.91      0.90      0.90      1296
           rhetorical question       0.72      0.59      0.65       174
            rhetorical command       0.70      0.74      0.72       108
intonation-dependent utterance       0.58      0.55      0.57       273

                      accuracy                           0.89      6045
                     macro avg       0.81      0.79      0.80      6045
                  weighted avg       0.89      0.89      0.89      6045

Run:

import torch
from transformers import RobertaTokenizerFast, RobertaForSequenceClassification, TextClassificationPipeline
from test_case_IC_iq import test_cases
import os
from datetime import datetime

# Check if GPU is available
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")

model_name = "jooni22/custom-dst-roberta-base"
loaded_tokenizer = RobertaTokenizerFast.from_pretrained(model_name)
loaded_model = RobertaForSequenceClassification.from_pretrained(
    model_name,
    num_labels=7,
    id2label={
        0: "fragment",
        1: "statement",
        2: "question",
        3: "command",
        4: "rhetorical question",
        5: "rhetorical command",
        6: "intonation-dependent utterance"
    },
    label2id={
        "fragment": 0,
        "statement": 1,
        "question": 2,
        "command": 3,
        "rhetorical question": 4,
        "rhetorical command": 5,
        "intonation-dependent utterance": 6
    }
).to(device)

# Using Pipeline
text_classifier = TextClassificationPipeline(
    tokenizer=loaded_tokenizer, 
    model=loaded_model,
    device=0 if device == "cuda" else -1,  # Use GPU if available
    top_k=None  # This returns all scores
)

# Function to classify a single text
def classify_text(text):
    preds_list = text_classifier(text)
    best_pred = preds_list[0][0]  # Access the first prediction of the first (and only) input
    
    # Treat "question", "command", "rhetorical question", and "rhetorical command" as "question"
    if best_pred['label'] in ["question", "command", "rhetorical question", "rhetorical command"]:
        return "question", best_pred['score']
    else:
        return best_pred['label'], best_pred['score']

# Process test cases
correct_predictions = 0
total_cases = len(test_cases)

# Prepare results string
results = []

for text, expected_is_question in test_cases:
    label, score = classify_text(text)
    predicted_is_question = label == "question"
    
    is_correct = predicted_is_question == expected_is_question
    if is_correct:
        correct_predictions += 1
    
    result = f"Text: {text}\n"
    result += f"Predicted: {'Question' if predicted_is_question else 'Not a question'} (Label: {label}, Score: {score:.4f})\n"
    result += f"Expected: {'Question' if expected_is_question else 'Not a question'}\n"
    result += f"Correct: {'Yes' if is_correct else 'No'}\n"
    result += "-" * 50 + "\n"
    
    results.append(result)
    print(result)

# Calculate accuracy
accuracy = correct_predictions / total_cases
print(f"\nAccuracy: {accuracy:.2%}")