susurofu/literary-bert-sentiment-eng

BERT Model for Literary Sentiment Analysis

This is a fine-tuned version of google-bert/bert-large-uncased, designed specifically for literary sentiment analysis.

This early but fully operational version was fine-tuned on 10,889 samples selected from the Project Gutenberg collection.

Unlike most BERT-based sentiment analysis models, it features:

1. A focus on literary texts and contexts rather than financial or review data.
2. Ten ordinal sentiment classes, ranging from 1 (the most negative) to 10 (the most positive). Instead of predicting three classes—positive, neutral, and negative—and extrapolating a continuous sentiment score from their probabilities, the model directly predicts sentiment on this ten-point scale.

Metrics

The model's perfomance was evaluated on marked up by human annotators the text of E. Hemingway's The Old Man and the Sea (dataset can be found in Bizzoni & Feldkamp (2024)). It was compared against cardiffnlp/twitter-roberta-base-sentiment-lates, SOTA model used in Digital Humanities and literary sentiment analysis.

Metric	This Model	RoBERTa
MAE	0.72	0.82
Accuracy (within 1 class error)	0.75	0.70
Precision (within 1 class error)	0.75	0.18
Kendall's Correlation	0.57	0.50

How to Use

To run this model you need the class for model and the class to run it. Ensure you have them both in your script.

Model Class

import torch
import numpy as np

import torch.nn as nn
import torch.nn.functional as F

from transformers import (
    AutoTokenizer,
    BertModel,
    BertPreTrainedModel,
)

from transformers.modeling_outputs import SequenceClassifierOutput


class BertForOrdinalRegression(BertPreTrainedModel):

    def __init__(self, config):
        super().__init__(config)

        self.num_classes = config.num_labels

        self.bert = BertModel(config)

        dropout_probability = (
            config.classifier_dropout
            if config.classifier_dropout is not None
            else config.hidden_dropout_prob
        )

        self.dropout = nn.Dropout(dropout_probability)

        self.sentiment_score = nn.Linear(
            config.hidden_size,
            1,
        )

        self.first_threshold = nn.Parameter(
            torch.tensor(-2.0)
        )

        self.threshold_deltas = nn.Parameter(
            torch.zeros(self.num_classes - 2)
        )

        self.post_init()

    def get_ordered_thresholds(self):

        if self.num_classes == 2:
            return self.first_threshold.reshape(1)

        positive_deltas = F.softplus(
            self.threshold_deltas
        )

        remaining_thresholds = (
            self.first_threshold
            + torch.cumsum(
                positive_deltas,
                dim=0
            )
        )

        return torch.cat([
            self.first_threshold.reshape(1),
            remaining_thresholds,
        ])

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        token_type_ids=None,
        labels=None,
        **kwargs,
    ):

        outputs = self.bert(
            input_ids=input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            return_dict=True,
        )

        pooled_output = self.dropout(
            outputs.pooler_output
        )

        latent_score = self.sentiment_score(
            pooled_output
        )

        thresholds = self.get_ordered_thresholds()

        logits = (
            latent_score
            - thresholds.unsqueeze(0)
        )

        loss = None

        if labels is not None:

            labels = labels.long()

            levels = torch.arange(
                1,
                self.num_classes,
                device=labels.device,
            ).unsqueeze(0)

            ordinal_targets = (
                labels.unsqueeze(1) > levels
            ).float()

            loss = F.binary_cross_entropy_with_logits(
                logits,
                ordinal_targets,
            )

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
        )

Analyzer Class

class SentimentAnalyzer:

    def __init__(
        self,
        model_path,
        device=None,
        max_length=256,
        batch_size=32,
    ):

        self.model_path = model_path
        self.max_length = max_length
        self.batch_size = batch_size

        if device is None:

            if torch.cuda.is_available():
                device = "cuda"

            elif (
                hasattr(torch.backends, "mps")
                and torch.backends.mps.is_available()
            ):
                device = "mps"

            else:
                device = "cpu"

        self.device = torch.device(device)


        self.tokenizer = AutoTokenizer.from_pretrained(
            model_path
        )


        self.model = (
            BertForOrdinalRegression.from_pretrained(
                model_path
            )
        )

        self.model.to(self.device)
        self.model.eval()

        self.num_classes = self.model.config.num_labels

        print(
            f"Model loaded from: {model_path}"
        )

        print(
            f"Device: {self.device}"
        )

        print(
            f"Sentiment scale: 1-{self.num_classes}"
        )

    def _predict_batch(self, texts):

        inputs = self.tokenizer(
            texts,
            return_tensors="pt",
            padding=True,
            truncation=True,
            max_length=self.max_length,
        )

        inputs = {
            key: value.to(self.device)
            for key, value in inputs.items()
        }

        with torch.inference_mode():

            outputs = self.model(
                **inputs
            )

        # P(y > k)
        cumulative_probabilities = torch.sigmoid(
            outputs.logits
        )

        # Expected ordinal sentiment:
        # E[Y] = 1 + sum(P(Y > k))
        scores = (
            1.0
            + cumulative_probabilities.sum(dim=1)
        )

        scores = (
            scores
            .detach()
            .cpu()
            .numpy()
        )

        return scores



    def predict(self, sentence):
    # if you need to predict only for a single sentence


        if not isinstance(sentence, str):
            raise TypeError(
                "sentence must be a string"
            )

        scores = self._predict_batch(
            [sentence]
        )

        return float(scores[0])

    def predict_list(self, sentences):
    # predictions for a list of sentences

        if not isinstance(sentences, list):
            raise TypeError(
                "sentences must be a list of strings"
            )

        if not all(
            isinstance(sentence, str)
            for sentence in sentences
        ):
            raise TypeError(
                "Every element must be a string"
            )

        if len(sentences) == 0:
            return []

        all_scores = []

        for start in range(
            0,
            len(sentences),
            self.batch_size,
        ):

            batch = sentences[
                start:start + self.batch_size
            ]

            scores = self._predict_batch(
                batch
            )

            all_scores.extend(
                scores.tolist()
            )

        return all_scores

Finally, run it for sentiment analysis

sentiment_model = SentimentAnalyzer(
    model_path="susurofu/literary-bert-sentiment-eng"
)

sentences = [
    "In the first forty days a boy had been with him.",
    "They were as old as erosions in a fishless desert.",
    "The old man had taught the boy to fish and the boy loved him.",
]

scores = sentiment_model.predict_list(sentences)

print(scores)

# Outputs:
# [4.938718795776367, 4.118151664733887, 7.0]

References

The dataset was taken from: Bizzoni, Yuri, and Pascale Feldkamp. “Sentiment Analysis for Literary Texts: Hemingway as a Case-Study.” Journal of Data Mining & Digital Humanities, vol. NLP4DH, Apr. 2024, pp. 1–20, https://doi.org/10.46298/jdmdh.13155.

Citation

If you use this model, you can cite our paper: [TBA] Soon, we will provide this info.