Why my simple Bert model for text classification could not learn anything?
Author: Hhhhhhhzf
Pull request: no

Hello, I try  `transformers.BertModel` to deal with a simple text classification, but the result makes me puzzled.
the code is simple,I implement the model with pytorch. 
they are...
```
# a Dataset class for BertModel
class BertDataset(Dataset):
    def __init__(self, train_file, tokenizer):
        super(BertDataset, self).__init__()
        self.train_file = train_file
        self.data = []
        self.label2id = {}
        self.id2label = {}
        self.tokenizer = tokenizer
        self.init()

    def init(self):
        with open(self.train_file, 'r', encoding='utf-8') as f:
            for line in f:
                blocks = line.strip().split('\t')
                if blocks[1] not in self.label2id:
                    self.label2id[blocks[1]] = len(self.label2id)
                    self.id2label[len(self.id2label)] = blocks[1]
                self.data.append({'token': self.tokenizer(blocks[0], add_special_tokens=True, max_length=100,
                                                          padding='max_length', return_tensors='pt',
                                                          truncation=True),
                                  'label': self.label2id[blocks[1]]})

    def __getitem__(self, item):
        return self.data[item]

    def __len__(self):
        return len(self.data)

# a collate function for torch.utils.data.DataLoader
def bert_collate_fn(batch_data):
    input_ids, token_type_ids, attention_mask, labels = [], [], [], []
    for instance in copy.deepcopy(batch_data):
        input_ids.append(instance['token']['input_ids'][0].squeeze(0))
        token_type_ids.append(instance['token']['token_type_ids'][0].squeeze(0))
        attention_mask.append(instance['token']['attention_mask'][0].squeeze(0))
        labels.append(instance['label'])
    return torch.stack(input_ids), torch.stack(token_type_ids), \
           torch.stack(attention_mask), torch.tensor(labels)

# Model
class PTModel(nn.Module):
    def __init__(self, model, n_class):
        super(PTModel, self).__init__()
        self.n_class = n_class
        self.model = model
        self.linear = nn.Linear(768, self.n_class)
        self.softmax = nn.Softmax(dim=-1)

    def forward(self, input_ids, token_type_ids=None, attention_mask=None):
        cls_emb = self.model(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)
        cls_emb = cls_emb[0][:, 0, :].squeeze(1)
        logits = self.linear(cls_emb)
        # logits = self.softmax(logits)
        return logits

# train code 
def train1():
    # data
    batch_size = 16
    tokenizer = BertTokenizer.from_pretrained(pretrained_path)
    dataset = BertDataset('../data/dataset/data.txt', tokenizer)
    train_len = int(len(dataset)*0.8)
    train_dataset, dev_dataset = random_split(dataset=dataset, lengths=[train_len, len(dataset)-train_len])
    train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=bert_collate_fn)
    dev_dataloader = DataLoader(dev_dataset, batch_size=batch_size, shuffle=True, collate_fn=bert_collate_fn)
    # model
    device = torch.device('cuda:{}'.format(args.cuda))
    bert_model = BertModel.from_pretrained(pretrained_path)
    model = PTModel(model=bert_model, n_class=len(dataset.label2id)).to(device)
    optimizer = torch.optim.Adam(params=model.parameters(), lr=args.lr)
    scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer=optimizer, milestones=[30, 40], gamma=0.1)
    loss_func = torch.nn.CrossEntropyLoss()
    # train
    for i in range(args.epoch):
        model.train()
        train_loss, dev_loss, f1_train, f1_dev = [], [], [], []
        dev_pred_list, dev_gold_list = [], []
        for input_ids, token_type_ids, attention_mask, label in tqdm(train_dataloader):
            input_ids, token_type_ids, attention_mask, label = input_ids.to(device), token_type_ids.to(device), \
                                                               attention_mask.to(device), label.to(device),
            outputs = model(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)
            array_outputs = np.array(outputs.cuda().data.cpu())
            optimizer.zero_grad()
            loss = loss_func(outputs, label)
            results = outputs.cuda().data.cpu().argmax(dim=1)
            score = f1_score(label.cuda().data.cpu(), results, average='micro')
            train_loss.append(loss.item())
            f1_train.append(score)
            # optim
            loss.backward()
            optimizer.step()
        scheduler.step()
        print('epoch {}'.format(i))
        print('train_loss:{}'.format(np.mean(train_loss)))
        print('train_f1:{}'.format(np.mean(f1_train)))

```

The train log is following(only 10 epoches). And the result was already clear: The model could not learn anything!!!!
PS: the learning rate  was 1e-3.
```
100%|█████████████████████████████████████████| 250/250 [00:43<00:00,  5.72it/s]
epoch 0
train_loss:4.217772917747498
train_f1:0.081
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.52it/s]
dev_f1:0.08928571428571429
dev_loss:4.111690880760314
100%|█████████████████████████████████████████| 250/250 [00:43<00:00,  5.71it/s]
epoch 1
train_loss:4.094675525665283
train_f1:0.084
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.16it/s]
dev_f1:0.0882936507936508
dev_loss:4.1316274839734275
100%|█████████████████████████████████████████| 250/250 [00:43<00:00,  5.71it/s]
epoch 2
train_loss:4.084259546279907
train_f1:0.08525
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.37it/s]
dev_f1:0.08928571428571429
dev_loss:4.108004717599778
100%|█████████████████████████████████████████| 250/250 [00:44<00:00,  5.62it/s]
epoch 3
train_loss:4.0770455904006955
train_f1:0.09425
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.07it/s]
dev_f1:0.08928571428571429
dev_loss:4.1077501395392035
100%|█████████████████████████████████████████| 250/250 [00:45<00:00,  5.54it/s]
epoch 4
train_loss:4.070150758743286
train_f1:0.086
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.41it/s]
dev_f1:0.09027777777777778
dev_loss:4.103204295748756
100%|█████████████████████████████████████████| 250/250 [00:45<00:00,  5.52it/s]
epoch 5
train_loss:4.064209712982178
train_f1:0.0895
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.31it/s]
dev_f1:0.08928571428571429
dev_loss:4.117827377622089
100%|█████████████████████████████████████████| 250/250 [00:43<00:00,  5.70it/s]
epoch 6
train_loss:4.065111406326294
train_f1:0.08425
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.34it/s]
dev_f1:0.0882936507936508
dev_loss:4.099656305615864
100%|█████████████████████████████████████████| 250/250 [00:44<00:00,  5.58it/s]
epoch 7
train_loss:4.0547873935699466
train_f1:0.09175
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.30it/s]
dev_f1:0.08928571428571429
dev_loss:4.105985126798115
100%|█████████████████████████████████████████| 250/250 [00:43<00:00,  5.76it/s]
epoch 8
train_loss:4.0595885887145995
train_f1:0.08875
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.26it/s]
dev_f1:0.09027777777777778
dev_loss:4.121003010916332
100%|█████████████████████████████████████████| 250/250 [00:45<00:00,  5.46it/s]
epoch 9
train_loss:4.054850312232971
train_f1:0.08825
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 18.86it/s]
dev_f1:0.08928571428571429
dev_loss:4.12501887669639
100%|█████████████████████████████████████████| 250/250 [00:45<00:00,  5.46it/s]
epoch 10
train_loss:4.0566882238388065
train_f1:0.08525
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 18.85it/s]
dev_f1:0.09126984126984126
dev_loss:4.103033436669244
```

Before this BertModel, I have tried LSTM, and the LSTM worked well. the dev f1 reached 0.96.  
```
# LSTM
class SimpleModel(nn.Module):
    def __init__(self, **kwargs):
        super(SimpleModel, self).__init__()
        self.embedding = nn.Embedding.from_pretrained(kwargs['pretrained_embedding'], freeze=False)
        self.lstm = nn.LSTM(kwargs['pretrained_embedding'].shape[1],
                            kwargs['hidden_size'],
                            batch_first=True,
                            bidirectional=True)
        self.linear = nn.Linear(kwargs['hidden_size']*2, kwargs['n_class'])

    def forward(self, inputs, lens):
        inputs = self.embedding(inputs)
        _, (h, _) = self.lstm(pack_padded_sequence(inputs, lens, batch_first=True, enforce_sorted=False))
        h = h.permute(1, 0, 2).contiguous().view(h.shape[1], -1)
        logits = self.linear(h)
        logits = logits.softmax(dim=-1)
        return logits
```

Could  any good man tell me why this code can't work.
Is there something wrong with my writing?
I have been confused  for days....
Thank you very much!


Comments:
Hi there!

Please use the forum https://discuss.huggingface.co/ to ask such questions :)  We use issues to report bugs and for feature requests.

> Hi there!
> 
> Please use the forum https://discuss.huggingface.co/ to ask such questions :) We use issues to report bugs and for feature requests.

I am sorry

I find the key of this problem, only need to change the learning rate to 1e-5.

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md) are likely to be ignored.