Run bert for multi-classification but loss never decrease
Author: MrKZZ
Pull request: no
Labels: name: wontfix

## ❓ Questions & Help

<!-- A clear and concise description of the question. -->
I copy the code in example/run_glue.py for text classification task with bert model. All i change in this code is changed to a multi-classification task. I didn't change the main stream of this code. however, after 80 epoches running, i just found the eval result didn't change from the begging to the end. Next time i print the loss and i found the loss nearly unchanged from the begining to the end. i change loss from 1e-5 to 1e-3 and it is not the case, the loss is still nearly unchanged. i just sum the loss together in an single epoch, and after several epochs it only changed from 4703 to 4700 . So that's my problem and any ideas will be appreciated .


`

import torch
from transformers import BertForSequenceClassification, BertTokenizer, InputExample, AdamW, WarmupLinearSchedule
from torch.utils.data import DataLoader, Dataset, SequentialSampler, RandomSampler, TensorDataset
from torch.utils.data.distributed import DistributedSampler
import random
import numpy as np
import os, pickle
import argparse
from tqdm import tqdm, trange
import copy, json
import glob
from apex import amp
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from Bert_eval import evaluate


def train(args, model, tokenizer):
    with open("../../data/train_texts", "r") as fr:
        texts = fr.readlines()
    with open("../../data/train_labels", "r") as fr:
        labels = fr.readlines()
    examples = load_dataset(texts, labels)
    label_list = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
    features = convert_examples_to_features(examples, tokenizer,
                                 label_list=label_list,
                                 output_mode="classification") 
    cached_path = "cached_file"
    torch.save(features, cached_path)
    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
    all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
    all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)
    all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
    train_dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels)
    train_dataloader = DataLoader(train_dataset, batch_size=args.train_batch_size)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay},
        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) 
    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=args.t_total) 
    
    model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)

    # multi-gpu training (should be after apex fp16 initialization)
    if args.n_gpu > 1:
        model = torch.nn.DataParallel(model)
    # Distributed training (should be after apex fp16 initialization)
    if args.local_rank != -1:
        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
                                                          output_device=args.local_rank,
                                                          find_unused_parameters=True)
    # Train!
    print("***** Running training *****")
    print("  Num examples = %d", len(train_dataset))
    print("  Num Epochs = %d", args.num_train_epochs)
    print("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
    print("  Total optimization steps = %d", args.t_total)
    global_step = 0
    train_iterator = trange(int(args.num_train_epochs), desc="Epoch")#, disable=args.local_rank not in [-1, 0])
    set_seed(args) 
    for _ in train_iterator:
        tr_loss = 0.0
        print("global_step: ", global_step)
        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
        print("optimizer: ", optimizer)
        for step, batch in enumerate(epoch_iterator):
            # print("batch: ", len(batch))
            model.train()
            batch = tuple(t.to(args.device) for t in batch)
            inputs = {'input_ids':      batch[0],
                    'attention_mask': batch[1],
                    'token_type_ids': batch[2],
                    'labels':         batch[3]}

            outputs = model(**inputs)
            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)
            print("loss:", loss)
            loss.backward()
            tr_loss += loss.item()
            if (step + 1) % args.gradient_accumulation_steps == 0:
                if args.fp16:
                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
                else:
                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)

                optimizer.step()
                scheduler.step()  # Update learning rate schedule
                model.zero_grad()
                global_step += 1
        output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step))
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        result = evaluate(args, model, tokenizer)
        print("result: ", result)
        model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
        model_to_save.save_pretrained(output_dir)
        torch.save(args, os.path.join(output_dir, 'training_args.bin'))
  
def load_dataset(lines, labels):
    """
        convert examples for the training sets for document classification
    """
    examples = []
    for (i, (line, label)) in enumerate(zip(lines, labels)):
        line = line.strip()
        label = label.strip()
        # label = str(i % 2)
        guid = i
        examples.append(
            InputExample(guid=guid, text_a=line, label=label)
        )
    return examples

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--data_dir", type=str, default="./data/",
                        help="data path for train or test")
    parser.add_argument("--train_batch_size", type=int, default=32, help="training batch size")
    parser.add_argument("--t_total", type=int, default=100, help="training epoch")
    parser.add_argument("--weight_decay", default=0.0, type=float,
                        help="Weight decay if we apply some.")
    parser.add_argument("--output_mode", default="classification", type=str, 
                        help="task name.")
    parser.add_argument("--learning_rate", default=1e-3, type=float,
                        help="The initial learning rate for Adam.")
    parser.add_argument("--max_grad_norm", default=1.0, type=float,
                        help="Max gradient norm.")
    parser.add_argument("--num_train_epochs", default=100, type=float,
                        help="Total number of training epochs to perform.")
    parser.add_argument("--max_steps", default=-1, type=int,
                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
    parser.add_argument("--warmup_steps", default=0, type=int,
                        help="Linear warmup over warmup_steps.")
    parser.add_argument("--n_gpu", type=int, default=1, help="number of gpus to run")
    parser.add_argument('--logging_steps', type=int, default=50,
                        help="Log every X updates steps.")
    parser.add_argument('--save_steps', type=int, default=5000,
                        help="Save checkpoint every X updates steps.")
    parser.add_argument("--per_gpu_eval_batch_size", default=256, type=int, 
                        help="task name.")
    parser.add_argument("--do_train", action="store_true", help="train model flag")
    parser.add_argument("--do_eval", action="store_true", help="eval model flag")
    parser.add_argument("--eval_all_checkpoints", action='store_true',
                        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")
    parser.add_argument("--no_cuda", action='store_true',
                        help="Avoid using CUDA when available")
    parser.add_argument('--overwrite_output_dir', action='store_true',
                        help="Overwrite the content of the output directory")
    parser.add_argument('--overwrite_cache', action='store_true',
                        help="Overwrite the cached training and evaluation sets")
    parser.add_argument('--seed', type=int, default=42,
                        help="random seed for initialization")
    parser.add_argument("--adam_epsilon", default=1e-8, type=float,
                        help="Epsilon for Adam optimizer.")
    parser.add_argument('--fp16', action='store_true',
                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
    parser.add_argument('--fp16_opt_level', type=str, default='O1',
                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
                             "See details at https://nvidia.github.io/apex/amp.html")
    parser.add_argument("--local_rank", type=int, default=-1,
                        help="For distributed training: local_rank")
    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
                        help="Number of updates steps to accumulate before performing a backward/update pass.")
    parser.add_argument("--output_dir", default="./checkpoint", type=str, 
                        help="The output directory where the model predictions and checkpoints will be written.")
    parser.add_argument("--eval_checkpoint", type=str, default="1730", help="the checkpoint to reload") 
    
    # parser.add_argument("--gpu", type=int, default=0, help="choose gpu device")
    args = parser.parse_args()
    if args.local_rank == -1 or args.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        args.n_gpu = torch.cuda.device_count()
    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
        torch.cuda.set_device(args.local_rank)
        device = torch.device("cuda", args.local_rank)
        torch.distributed.init_process_group(backend='nccl')
        args.n_gpu = 1
    args.device = device
    model_class = BertForSequenceClassification
    tokenizer_class = BertTokenizer
    pretrained_weights = "bert-base-chinese"
    tokenizer = tokenizer_class.from_pretrained(pretrained_weights)
    model = model_class.from_pretrained(pretrained_weights, num_labels=10)#, output_hidden_states=True, output_attentions=True)
    if torch.cuda.is_available():
        model.to(args.device)
    train(args, model, tokenizer)
    if True:
        # Create output directory if needed
        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
            os.makedirs(args.output_dir)
        print("model saved")
        model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
        model_to_save.save_pretrained(args.output_dir)
        tokenizer.save_pretrained(args.output_dir)
        print("model saved into ", args.output_dir)
        # Good practice: save your training arguments together with the trained model
        torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))

        # Load a trained model and vocabulary that you have fine-tuned
        model = model_class.from_pretrained(args.output_dir)
        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
        model.to(args.device)
    results = {}
    if True:
        tokenizer = tokenizer_class.from_pretrained(args.output_dir)#, do_lower_case=args.do_lower_case)
        checkpoints = [args.output_dir]
        if args.eval_all_checkpoints:
            checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True)))
        for checkpoint in checkpoints:
            global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""
            prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else ""
            
            model = model_class.from_pretrained(checkpoint)
            model.to(args.device)
            result = evaluate(args, model, tokenizer, prefix=prefix)
            result = dict((k + '_{}'.format(global_step), v) for k, v in result.items())
            results.update(result)
    print("result: ", results)

`

Comments:
Your copy&pasta is broken. Perhaps Roberto can help you? [Text classification with RoBERTa](https://rsilveira79.github.io/fermenting_gradients/machine_learning/nlp/pytorch/text_classification_roberta/)

> Your copy&pasta is broken. Perhaps Roberto can help you? [Text classification with RoBERTa](https://rsilveira79.github.io/fermenting_gradients/machine_learning/nlp/pytorch/text_classification_roberta/)

hahaha, actually i solve this problem last night. It's because WarmupLinearSchedule, and I set a t_total args which is smaller than warmup steps , so the lr is nearly zero actually.

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.