src/models/train_model.py

import json

import yaml

from model import Summarization
import pandas as pd


def train_model():
    """
    Train the model
    """
    with open("params.yml") as f:
        params = yaml.safe_load(f)

    # Load the data
    train_df = pd.read_csv('data/processed/train.csv')
    eval_df = pd.read_csv('data/processed/validation.csv')

    train_df = train_df.sample(frac=params['split'], replace=True, random_state=1)
    eval_df = eval_df.sample(frac=params['split'], replace=True, random_state=1)

    model = Summarization()
    model.from_pretrained(model_type=params['model_type'], model_name=params['model_name'])

    model.train(train_df=train_df, eval_df=eval_df,
                batch_size=params['batch_size'], max_epochs=params['epochs'],
                use_gpu=params['use_gpu'], learning_rate=float(params['learning_rate']),
                num_workers=int(params['num_workers']))

    model.save_model(model_dir=params['model_dir'])

    with open('wandb/latest-run/files/wandb-summary.json') as json_file:
        data = json.load(json_file)

    with open('reports/training_metrics.txt', 'w') as fp:
        json.dump(data, fp)


if __name__ == '__main__':
    train_model()