F1 score > 1
🐛 Bug
F1 is > 1
From comet:

Thank you!
Cheers,
Francesco
Please reproduce using the BoringModel
class MySystem(pl.LightningModule):

    def __init__(self, model: nn.Module, criterion: Callable = nn.CrossEntropyLoss, lr :float = 1e-3):
        super().__init__()
        self.model = model
        self.criterion = criterion()
        self.lr = lr
        self.metrics = nn.ModuleDict({
            'accuracy': Accuracy(),
            'f1': F1()
        })
        # this apis are so bad
        
    def compute_metrics(self, out: Tensor, batch: Union[Tensor, Tensor]) -> Dict[str, Tensor]:
        x, y = batch
        metrics = {}
        for name, metric in self.metrics.items():
            metrics[name] = metric(out, y)
        return metrics

    def basic_step(self, batch: Union[Tensor, Tensor], name_space: str) -> Tensor:
        x, y = batch
        out = self.model(x)
        loss = self.criterion(out, y)
        metrics = self.compute_metrics(out, batch)
        self.log_dict({f'{name_space}_loss': loss, **
                       {f'{name_space}_{k}': v for k, v in metrics.items()}})
        return loss

    def training_step(self, batch: Union[Tensor, Tensor], batch_idx: int) -> Tensor:
        loss = self.basic_step(batch, name_space='train')
        return loss

    def validation_step(self, batch: Union[Tensor, Tensor], batch_idx: int) -> Tensor:
        loss = self.basic_step(batch, name_space='val')
        return loss

    def test_step(self, batch: Union[Tensor, Tensor], batch_idx: int) -> Tensor:
        loss = self.basic_step(batch, name_space='test')
        return loss

    def configure_optimizers(self):
        optimizer = optim.AdamW(self.parameters(), lr=self.lr)
        scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer)
        return {
            'optimizer': optimizer,
            'lr_scheduler': scheduler,
            'monitor': 'val_loss'
        }
The training is correct as loss and accuracy are consistent,
...
    # our callbacks
    callbacks = [
        ModelCheckpoint(
            monitor='val_loss',
            dirpath=model_checkpoint_dir,
            filename='{epoch:02d}-{val_loss:.2f}'
        ),
        EarlyStopping(monitor='val_loss', patience=10, verbose=True)
    ]
    # using commet
    logger = CometLogger(
        api_key=secrets['COMET_API_KEY'],
        project_name="test",
        workspace="francescosaveriozuppichini"
    )
    logger.log_hyperparams(params)
    
    system = MySystem(model=cnn, lr=params['lr'])

    trainer = Trainer(gpus=1, min_epochs=params['epochs'],
                      progress_bar_refresh_rate=20, logger=logger,
                      callbacks=callbacks)

    trainer.fit(system, train_dl, val_dl)

    print(trainer.test(test_dataloaders=test_dl))
Expected behavior

Environment

CUDA:
- GPU:
- GeForce GTX 1080 Ti
- available:         True
- version:           11.0
Packages:
- numpy:             1.18.4
- pyTorch_debug:     True
- pyTorch_version:   1.7.0+cu110
- pytorch-lightning: 1.1.5
- tqdm:              4.46.0
System:
- OS:                Linux
- architecture:
- 64bit
- ELF
- processor:         x86_64
- python:            3.8.2
- version:           #45~20.04.1-Ubuntu SMP Fri Jan 15 11:35:04 UTC 2021

[Edit] Forgot to say thank you :)