ai_t5_model1.py

# -*- coding: utf-8 -*-
"""AI_t5_model2.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1cLG3m6CnABOLIGgwQuZUJfRZjsMHk6y7
"""

!pip install transformers[torch] accelerate

# Uninstall conflicting packages
!pip uninstall -y requests google-colab

# Reinstall google-colab which will bring the compatible requests version
!pip install google-colab

pip install requests==2.31.0

!pip install rouge_score
!pip install evaluate
# !pip install datasets

import numpy as np
import pandas as pd
from datasets import Dataset, DatasetDict
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, \
    Seq2SeqTrainer, Seq2SeqTrainingArguments, DataCollatorForSeq2Seq, get_scheduler
import evaluate
import nltk
from nltk.tokenize import sent_tokenize
import warnings
warnings.simplefilter(action='ignore')

data = pd.read_csv('news_summary.csv', encoding='cp437')
data = data.dropna()
data.info()

# headlines - column containing headlines which will be used as reference summarizations
# ctext - column containing full texts of news articles
# taking a look at the average lengths of both

def length(text):
    return len(text.split())

print('Mean headline length (words):', data['headlines'].apply(length).mean())
print('Mean text length (words):', data['ctext'].apply(length).mean())

# splitting the data into train, val, and test, and converting it into Dataset format

train_size = int(0.8 * len(data))
val_size = int(0.1 * len(data))
test_size = len(data) - train_size - val_size

train_data = data[:train_size]
val_data = data[train_size:train_size+val_size]
test_data = data[train_size+val_size:]

train_dataset = Dataset.from_pandas(train_data)
val_dataset = Dataset.from_pandas(val_data)
test_dataset = Dataset.from_pandas(test_data)

dataset = DatasetDict({
    "train": train_dataset,
    "validation": val_dataset,
    "test": test_dataset
})

dataset

# loading the model tokenizer

model_checkpoint = "google/mt5-small"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)

# creating tokenization function with length limits for headlines and texts

max_input_length = 512
max_target_length = 30

def preprocess_function(examples):
    model_inputs = tokenizer(
        examples["ctext"],
        max_length=max_input_length,
        truncation=True,
    )
    labels = tokenizer(
        examples["headlines"], max_length=max_target_length, truncation=True
    )
    model_inputs["labels"] = labels["input_ids"]
    return model_inputs

# tokenizing the datasets

tokenized_datasets = dataset.map(preprocess_function, batched=True)

# loading ROUGE metric

rouge_score = evaluate.load("rouge")

import nltk
nltk.download('punkt')

def three_sentence_summary(text):
    return "\n".join(sent_tokenize(text)[:3])


print(three_sentence_summary(dataset["train"][1]["ctext"]))

def evaluate_baseline(dataset, metric):
    summaries = [three_sentence_summary(text) for text in dataset["ctext"]]
    return metric.compute(predictions=summaries, references=dataset["headlines"])

# getting baseline metrics

score = evaluate_baseline(dataset["validation"], rouge_score)
rouge_names = ["rouge1", "rouge2", "rougeL", "rougeLsum"]
rouge_dict = dict((rn, round(score[rn] * 100, 2)) for rn in rouge_names)
rouge_dict

# logging in to Hugging Face Hub

from huggingface_hub import notebook_login

notebook_login()

# loading the pre-trained Seq2Seq model and the data collator

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)

# setting arguments

batch_size = 10
num_train_epochs = 12
# Show the training loss with every epoch
logging_steps = len(tokenized_datasets["train"]) // batch_size
output_dir = "mt5-small-finetuned-news-summary-kaggle"

args = Seq2SeqTrainingArguments(
    output_dir=output_dir,
    evaluation_strategy="steps",
    learning_rate=4e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    weight_decay=0.005,
    save_total_limit=3,
    num_train_epochs=num_train_epochs,
    predict_with_generate=True,       # calculate ROUGE for every epoch
    logging_steps=logging_steps,
    push_to_hub=True,
)

# function for computing ROUGE metrics

def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
    labels= np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
    decoded_preds = ["\n".join(sent_tokenize(pred.strip())) for pred in decoded_preds]
    decoded_labels = ["\n".join(sent_tokenize(label.strip())) for label in decoded_labels]
    result = rouge_score.compute(
       predictions=decoded_preds, references=decoded_labels, use_stemmer=True
    )
    result = {key: value * 100 for key, value in result.items()}
    return {k: round(v, 4) for k, v in result.items()}

# removing columns containing strings

tokenized_datasets = tokenized_datasets.remove_columns(
    dataset["train"].column_names
)

# defining Trainer

trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)

# training the model

trainer.train()

# evaluating the model

trainer.evaluate()

trainer.args.output_dir = "mt5-small-finetuned-news-summary-model-2"

# pushing to Hugging Face Hub

trainer.push_to_hub(commit_message="Training complete", tags="summarization")

from transformers import pipeline

hub_model_id = "shivraj221/mt5-small-finetuned-news-summary-kaggle"
summarizer = pipeline("summarization", model=hub_model_id)

# function to get a summary of an article with index idx

def print_summary(idx):
    review = dataset["test"][idx]["ctext"]
    title = dataset["test"][idx]["headlines"]
    summary = summarizer(dataset["test"][idx]["ctext"])[0]["summary_text"]
    print(f"'>>> Article: {review}'")
    print(f"\n'>>> Headline: {title}'")
    print(f"\n'>>> Summary: {summary}'")

print_summary(20)