import math import argparse import pprint from distutils.util import strtobool from pathlib import Path from loguru import logger as loguru_logger import pytorch_lightning as pl from pytorch_lightning.utilities import rank_zero_only from pytorch_lightning.loggers import TensorBoardLogger from pytorch_lightning.callbacks import ModelCheckpoint, LearningRateMonitor from pytorch_lightning.plugins import DDPPlugin from src.config.default import get_cfg_defaults from src.utils.misc import get_rank_zero_only_logger, setup_gpus from src.utils.profiler import build_profiler from src.lightning_trainer.data import MultiSceneDataModule from src.lightning_trainer.trainer import PL_Trainer loguru_logger = get_rank_zero_only_logger(loguru_logger) def parse_args(): # init a costum parser which will be added into pl.Trainer parser # check documentation: https://pytorch-lightning.readthedocs.io/en/latest/common/trainer.html#trainer-flags parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument("data_cfg_path", type=str, help="data config path") parser.add_argument("main_cfg_path", type=str, help="main config path") parser.add_argument("--exp_name", type=str, default="default_exp_name") parser.add_argument("--batch_size", type=int, default=4, help="batch_size per gpu") parser.add_argument("--num_workers", type=int, default=4) parser.add_argument( "--pin_memory", type=lambda x: bool(strtobool(x)), nargs="?", default=True, help="whether loading data to pinned memory or not", ) parser.add_argument( "--ckpt_path", type=str, default=None, help="pretrained checkpoint path, helpful for using a pre-trained coarse-only LoFTR", ) parser.add_argument( "--disable_ckpt", action="store_true", help="disable checkpoint saving (useful for debugging).", ) parser.add_argument( "--profiler_name", type=str, default=None, help="options: [inference, pytorch], or leave it unset", ) parser.add_argument( "--parallel_load_data", action="store_true", help="load datasets in with multiple processes.", ) parser = pl.Trainer.add_argparse_args(parser) return parser.parse_args() def main(): # parse arguments args = parse_args() rank_zero_only(pprint.pprint)(vars(args)) # init default-cfg and merge it with the main- and data-cfg config = get_cfg_defaults() config.merge_from_file(args.main_cfg_path) config.merge_from_file(args.data_cfg_path) pl.seed_everything(config.TRAINER.SEED) # reproducibility # TODO: Use different seeds for each dataloader workers # This is needed for data augmentation # scale lr and warmup-step automatically args.gpus = _n_gpus = setup_gpus(args.gpus) config.TRAINER.WORLD_SIZE = _n_gpus * args.num_nodes config.TRAINER.TRUE_BATCH_SIZE = config.TRAINER.WORLD_SIZE * args.batch_size _scaling = config.TRAINER.TRUE_BATCH_SIZE / config.TRAINER.CANONICAL_BS config.TRAINER.SCALING = _scaling config.TRAINER.TRUE_LR = config.TRAINER.CANONICAL_LR * _scaling config.TRAINER.WARMUP_STEP = math.floor(config.TRAINER.WARMUP_STEP / _scaling) # lightning module profiler = build_profiler(args.profiler_name) model = PL_Trainer(config, pretrained_ckpt=args.ckpt_path, profiler=profiler) loguru_logger.info(f"Model LightningModule initialized!") # lightning data data_module = MultiSceneDataModule(args, config) loguru_logger.info(f"Model DataModule initialized!") # TensorBoard Logger logger = TensorBoardLogger( save_dir="logs/tb_logs", name=args.exp_name, default_hp_metric=False ) ckpt_dir = Path(logger.log_dir) / "checkpoints" # Callbacks # TODO: update ModelCheckpoint to monitor multiple metrics ckpt_callback = ModelCheckpoint( monitor="auc@10", verbose=True, save_top_k=5, mode="max", save_last=True, dirpath=str(ckpt_dir), filename="{epoch}-{auc@5:.3f}-{auc@10:.3f}-{auc@20:.3f}", ) lr_monitor = LearningRateMonitor(logging_interval="step") callbacks = [lr_monitor] if not args.disable_ckpt: callbacks.append(ckpt_callback) # Lightning Trainer trainer = pl.Trainer.from_argparse_args( args, plugins=DDPPlugin( find_unused_parameters=False, num_nodes=args.num_nodes, sync_batchnorm=config.TRAINER.WORLD_SIZE > 0, ), gradient_clip_val=config.TRAINER.GRADIENT_CLIPPING, callbacks=callbacks, logger=logger, sync_batchnorm=config.TRAINER.WORLD_SIZE > 0, replace_sampler_ddp=False, # use custom sampler reload_dataloaders_every_epoch=False, # avoid repeated samples! weights_summary="full", profiler=profiler, ) loguru_logger.info(f"Trainer initialized!") loguru_logger.info(f"Start training!") trainer.fit(model, datamodule=data_module) if __name__ == "__main__": main()