split added

params.yml

@@ -2,9 +2,10 @@ data: cnn_dailymail
 batch_size: 4
 num_workers: 2
 model_type: t5
-model_name: t5-base
+model_name: t5-small
 learning_rate: 1e-4
 epochs: 5
 source_dir: src
 model_dir: models
-metric: rouge
+metric: rouge
+split: 0.001

src/models/train_model.py

@@ -15,6 +15,9 @@ def train_model():
     train_df = pd.read_csv('data/processed/train.csv')
     eval_df = pd.read_csv('data/processed/validation.csv')
 
+    train_df = train_df.sample(frac=params['split'], replace=True, random_state=1)
+    eval_df = eval_df.sample(frac=params['split'], replace=True, random_state=1)
+
     model = Summarization()
     model.from_pretrained(model_type=params['model_type'], model_name=params['model_name'])