[#2] Eval script added. It still needs testing

config.yaml

@@ -12,9 +12,10 @@ train:
 upload:
   idioms:
     ver: d-1-2
-    description: the set of idioms in the traning set of literal2idiomatic:d-1-2
+    description: the set of idioms in the traning set of literal2idiomatic:d-1-2.
   literal2idiomatic:
     ver: d-1-2
-    description: PIE data split into train & test set (80 / 20 split)
+    description: PIE data split into train & test set (80 / 20 split). There is no validation set, because I don't intend to
+    do hyperparameter tuning on this set.
     train_ratio: 0.8
     seed: 104

explore/explore_torchmetrics_bleu.py

@@ -0,0 +1,28 @@
+
+from torchmetrics import BLEUScore
+from transformers import BartTokenizer
+
+
+pairs = [
+    ("I knew you could do it", "I knew you could do it"),
+    ("I knew you could do it", "you knew you could do it")
+]
+
+
+def main():
+    tokenizer = BartTokenizer.from_pretrained('facebook/bart-base')
+    metric = BLEUScore()
+    preds = tokenizer([pred for pred, _ in pairs])['input_ids']
+    targets = tokenizer([target for _, target in pairs])['input_ids']
+    print(preds)
+    print(targets)
+    print(metric(preds, targets))
+    # arghhh, so bleu score does not support tensors...
+    """
+    AttributeError: 'int' object has no attribute 'split'
+    """
+    # let's just go for the accuracies then.
+
+
+if __name__ == '__main__':
+    main()

idiomify/metrics.py

@@ -1,4 +0,0 @@
-"""
-you may want to include bleu score.
-and more metrics for paraphrasing.
-"""

idiomify/models.py

@@ -7,7 +7,7 @@ from torch.nn import functional as F
 import pytorch_lightning as pl
 from transformers import BartForConditionalGeneration, BartTokenizer
 from idiomify.builders import SourcesBuilder
-
+from torchmetrics import Accuracy
 
 class Idiomifier(pl.LightningModule):  # noqa
     """
@@ -15,8 +15,11 @@ class Idiomifier(pl.LightningModule):  # noqa
     """
     def __init__(self, bart: BartForConditionalGeneration, lr: float, bos_token_id: int, pad_token_id: int):  # noqa
         super().__init__()
-        self.bart = bart
         self.save_hyperparameters(ignore=["bart"])
+        self.bart = bart
+        # metrics (using accuracies as of right now)
+        self.acc_train = Accuracy(ignore_index=pad_token_id)
+        self.acc_test = Accuracy(ignore_index=pad_token_id)
 
     def forward(self, srcs: torch.Tensor, tgts_r: torch.Tensor) -> torch.Tensor:
         """
@@ -40,13 +43,27 @@ class Idiomifier(pl.LightningModule):  # noqa
         logits = self.forward(srcs, tgts_r).transpose(1, 2)  # ... -> (N, L, |V|) -> (N, |V|, L)
         loss = F.cross_entropy(logits, tgts, ignore_index=self.hparams['pad_token_id'])\
                 .sum()  # (N, L, |V|), (N, L) -> (N,) -> (1,)
+        self.acc_train.update(logits.detach(), target=tgts.detach())
         return {
             "loss": loss
         }
 
-    def on_train_batch_end(self, outputs: dict, *args, **kwargs):
+    def on_train_batch_end(self, outputs: dict, **kwargs):
         self.log("Train/Loss", outputs['loss'])
 
+    def on_train_epoch_end(self, *args, **kwargs) -> None:
+        self.log("Train/Accuracy", self.acc_train.compute())
+        self.acc_train.reset()
+
+    def test_step(self, batch: Tuple[torch.Tensor, torch.Tensor, torch.Tensor], **kwargs):
+        srcs, tgts_r, tgts = batch  # (N, 2, L_s), (N, 2, L_t), (N, 2, L_t)
+        logits = self.forward(srcs, tgts_r).transpose(1, 2)  # ... -> (N, L, |V|) -> (N, |V|, L)
+        self.acc_test.update(logits.detach(), target=tgts.detach())
+
+    def on_test_end(self):
+        self.log("Test/Accuracy", self.acc_test.compute())
+        self.acc_test.reset()
+
     def configure_optimizers(self) -> torch.optim.Optimizer:
         """
         Instantiates and returns the optimizer to be used for this model

main_eval.py

@@ -0,0 +1,34 @@
+import torch
+import argparse
+import os
+import wandb
+import pytorch_lightning as pl
+from pytorch_lightning.loggers import WandbLogger
+from transformers import BartTokenizer
+from idiomify.data import IdiomifyDataModule
+from idiomify.fetchers import fetch_config, fetch_idiomifier
+from paths import ROOT_DIR
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--num_workers", type=int, default=os.cpu_count())
+    args = parser.parse_args()
+    config = fetch_config()['train']
+    config.update(vars(args))
+    # prepare the model
+    tokenizer = BartTokenizer.from_pretrained(config['bart'])
+    # prepare the datamodule
+    with wandb.init(entity="eubinecto", project="idiomify", config=config) as run:
+        model = fetch_idiomifier(config['ver'], run)
+        datamodule = IdiomifyDataModule(config, tokenizer, run)
+        logger = WandbLogger(log_model=False)
+        trainer = pl.Trainer(fast_dev_run=config['fast_dev_run'],
+                             gpus=torch.cuda.device_count(),
+                             default_root_dir=str(ROOT_DIR),
+                             logger=logger)
+        trainer.test(model, datamodule)
+
+
+if __name__ == '__main__':
+    main()