File size: 4,244 Bytes
2f044c1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
from typing import Any, Union
import hydra
import lightning as pl
import torch
from omegaconf import DictConfig
from relik.retriever.common.model_inputs import ModelInputs
class GoldenRetrieverPLModule(pl.LightningModule):
def __init__(
self,
model: Union[torch.nn.Module, DictConfig],
optimizer: Union[torch.optim.Optimizer, DictConfig],
lr_scheduler: Union[torch.optim.lr_scheduler.LRScheduler, DictConfig] = None,
*args,
**kwargs,
) -> None:
super().__init__()
self.save_hyperparameters(ignore=["model"])
if isinstance(model, DictConfig):
self.model = hydra.utils.instantiate(model)
else:
self.model = model
self.optimizer_config = optimizer
self.lr_scheduler_config = lr_scheduler
def forward(self, **kwargs) -> dict:
"""
Method for the forward pass.
'training_step', 'validation_step' and 'test_step' should call
this method in order to compute the output predictions and the loss.
Returns:
output_dict: forward output containing the predictions (output logits ecc...) and the loss if any.
"""
return self.model(**kwargs)
def training_step(self, batch: ModelInputs, batch_idx: int) -> torch.Tensor:
forward_output = self.forward(**batch, return_loss=True)
self.log(
"loss",
forward_output["loss"],
batch_size=batch["questions"]["input_ids"].size(0),
prog_bar=True,
)
return forward_output["loss"]
def validation_step(self, batch: ModelInputs, batch_idx: int) -> None:
forward_output = self.forward(**batch, return_loss=True)
self.log(
"val_loss",
forward_output["loss"],
batch_size=batch["questions"]["input_ids"].size(0),
)
def test_step(self, batch: ModelInputs, batch_idx: int) -> Any:
forward_output = self.forward(**batch, return_loss=True)
self.log(
"test_loss",
forward_output["loss"],
batch_size=batch["questions"]["input_ids"].size(0),
)
def configure_optimizers(self):
if isinstance(self.optimizer_config, DictConfig):
param_optimizer = list(self.named_parameters())
no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p for n, p in param_optimizer if "layer_norm_layer" in n
],
"weight_decay": self.hparams.optimizer.weight_decay,
"lr": 1e-4,
},
{
"params": [
p
for n, p in param_optimizer
if all(nd not in n for nd in no_decay)
and "layer_norm_layer" not in n
],
"weight_decay": self.hparams.optimizer.weight_decay,
},
{
"params": [
p
for n, p in param_optimizer
if "layer_norm_layer" not in n
and any(nd in n for nd in no_decay)
],
"weight_decay": 0.0,
},
]
optimizer = hydra.utils.instantiate(
self.optimizer_config,
# params=self.parameters(),
params=optimizer_grouped_parameters,
_convert_="partial",
)
else:
optimizer = self.optimizer_config
if self.lr_scheduler_config is None:
return optimizer
if isinstance(self.lr_scheduler_config, DictConfig):
lr_scheduler = hydra.utils.instantiate(
self.lr_scheduler_config, optimizer=optimizer
)
else:
lr_scheduler = self.lr_scheduler_config
lr_scheduler_config = {
"scheduler": lr_scheduler,
"interval": "step",
"frequency": 1,
}
return [optimizer], [lr_scheduler_config]
|