I've applied all your suggestions except for compining model I think I have more than one model centernet, decoupled head , yolo, convolutional  classifier, and NAS model it think you need to review my whole training file after your suggestions and build a more advanced professional combining model
import sys
import traceback
import warnings
from typing import Dict, Any, OrderedDict, Tuple, Optional, List
import catalyst
import os
import numpy as np
import pandas as pd
import torch
import hydra
from torch.optim import Optimizer
from torch.utils.data import Dataset, Sampler, DataLoader
from torch import nn
from omegaconf import DictConfig
from functools import partial
import albumentations as A
from catalyst.callbacks import ControlFlowCallback, CriterionCallback, SchedulerCallback, TimerCallback
from catalyst.core import Callback
from pytorch_toolbelt.utils.catalyst import (
    ShowPolarBatchesCallback,
    BestMetricCheckpointCallback,
    HyperParametersCallback,
    MixupCriterionCallback,
    MixupInputCallback,
)
from pytorch_toolbelt.optimization.functional import freeze_model, get_lr_decay_parameters, get_optimizable_parameters
from pytorch_toolbelt.utils import fs, transfer_weights, count_parameters
from pytorch_toolbelt.utils.distributed import is_main_process
from xview3.centernet.bboxer import MultilabelCircleNetCoder
from xview3.early_stopping import EarlyStoppingCallback
from xview3.nas import ProgressiveNAS
from xview3.centernet.models.heads.yolo_head import YOLOHead, generate_yolo_targets
from xview3.centernet.models.transformer_classifier import ConvolutionalClassifier
from xview3.centernet.losses.circlenet_loss import CircleNetLoss, YOLOLoss
from xview3.centernet import *
from xview3.pipeline import *
from xview3.factory import *
from xview3.augmentations import *
from xview3.samplers import *
from xview3.dataset import *
from xview3.ema import *
from xview3.evaluation import *
from xview3.inference import *
from xview3.metric import *
from xview3.constants import INPUT_SCENE_ID_KEY, TARGET_CENTERS_KEY, TARGET_FISHING_KEY,TARGET_VESSEL_KEY,INPUT_CONFIDENCE_KEY,TARGET_LENGTHS_KEY
import logging
from torch.cuda.amp import autocast, GradScaler
from xview3.utils import TorchDevice
from xview3.evaluation import evaluate_on_scenes
from xview3.inference import predict_multilabel_scenes
from omegaconf import OmegaConf
from datetime import datetime

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

def warn_with_traceback(message, category, filename, lineno, file=None, line=None):
    log = file if hasattr(file, "write") else sys.stderr
    traceback.print_stack(file=log)
    log.write(warnings.formatwarning(message, category, filename, lineno))

warnings.showwarning = warn_with_traceback

if 'XVIEW3_DIR' not in os.environ:
    os.environ['XVIEW3_DIR'] = '/Volumes/Crucial/xview3_dataset'

class CenternetPipeline(Pipeline):
    def __init__(self, cfg):
        super().__init__(cfg)
        # Initialize mappings
        self.scaler = GradScaler()
        self.accumulation_steps = cfg.get('accumulation_steps', 1)
        self.confidence_mapping = {0: 0.0, 1: 1.0}  # Example mapping
        self.vessel_mapping = {0: 0, 1: 1}         # Example mapping
        self.fishing_mapping = {0: 0, 1: 1}        # Example mapping
        self.ignore_value = -1                      # Value for ignored labels
        #self.box_coder: Optional[MultilabelCircleNetCoder] = None
        self.box_coder = MultilabelCircleNetCoder(
            image_size=(cfg.train.train_image_size[0], cfg.train.train_image_size[1]),
            output_stride=32,    # Adjust based on your model's stride
            max_objects=512,     # Adjust based on your dataset
            heatmap_encoding="umich",
            labels_encoding="circle",
            ignore_value=self.ignore_value
        )
        self.model: Optional[nn.Module] = None  # Ensure model is initialized
        self.criterion: Optional[nn.Module] = None 
        self.shore_root = os.path.join(os.environ.get('XVIEW3_DIR', ''), "full_dataset", "shoreline")
        self.device = TorchDevice.choose_torch_device()        
        #self.feature_extractor = nn.Sequential(nn.Conv2d(3, 256, kernel_size=3, padding=1), nn.ReLU())
        #self.yolo_head = YOLOHead(num_classes=2, anchors=[[10, 13], [16, 30], [33, 23]])  # Example anchors

        if self.box_coder is None:
            raise ValueError("Box Coder initialization failed.")
        
        # Initialize other components if necessary
        self.feature_extractor = nn.Sequential(
            nn.Conv2d(3, 256, kernel_size=3, padding=1),
            nn.ReLU()
        )
        
        # Initialize YOLOHead here or in build_model
        self.yolo_head = YOLOHead(
            num_classes=2,  # Adjust based on your dataset
            anchors=[[10, 13], [16, 30], [33, 23]]
        ).to(self.device)
        
    def build_datasets(self, config) -> Tuple[Dataset, Dataset, Optional[Sampler], List[Callback]]:
        dataset = MultilabelCircleNetDataModule(data_dir=os.path.join(os.environ.get('XVIEW3_DIR', '/Volumes/Crucial/xview3_dataset'), 'full_dataset'))
        self.box_coder = MultilabelCircleNetCoder(
            image_size=(config.train.train_image_size[0], config.train.train_image_size[1]),
            output_stride=4,  # Adjust this value as needed
            max_objects=512,  # Adjust this value as needed
            heatmap_encoding="umich",
            labels_encoding="circle",
            ignore_value=-1
        )
        if self.box_coder is None:
            raise ValueError("self.box_coder is None in build_datasets.")

        train_df, valid_df, _, shore_root = dataset.train_val_split(
            splitter=config.dataset.splitter,
            fold=config.dataset.fold,
            num_folds=config.dataset.num_folds,
        )

        self.valid_df = valid_df.copy()  # Keep a copy of unmodified groundtruth

        self.master_print("Dataset", config.dataset.slug)
        self.master_print("\tignore_low_confidence_detections", config.dataset.ignore_low_confidence_detections)
        self.master_print("\tignore_low_confidence_labels    ", config.dataset.ignore_low_confidence_labels)
        self.master_print("\tfilter_low_confidence_objects   ", config.dataset.filter_low_confidence_objects)

        self.master_print("Train dataset")
        self.master_print("\tUnknown vessels", np.isnan(train_df.is_vessel.values.astype(float)).sum())
        self.master_print("\tUnknown fishing", np.isnan(train_df.is_fishing.values.astype(float)).sum())
        self.master_print("Valid dataset")
        self.master_print("\tUnknown vessels", np.isnan(valid_df.is_vessel.values.astype(float)).sum())
        self.master_print("\tUnknown fishing", np.isnan(valid_df.is_fishing.values.astype(float)).sum())

        if config.dataset.filter_low_confidence_objects and config.dataset.ignore_low_confidence_labels:
            raise ValueError(f"filter_low_confidence_objects and ignore_low_confidence_labels are mutually exclusive")

        if config.dataset.filter_low_confidence_objects:
            train_df = filter_low_confidence_objects(train_df)
            valid_df = filter_low_confidence_objects(valid_df)
            self.master_print("Excluding low-confidence objects from training & validation")
        elif config.dataset.ignore_low_confidence_labels:
            train_df = ignore_low_confidence_objects(train_df)
            valid_df = ignore_low_confidence_objects(valid_df)
            self.master_print("Ignoring low-confidence labels from training & validation")

        self.shore_root = os.path.join(os.environ.get('XVIEW3_DIR', ''), "full_dataset", "shoreline")

        normalization = build_normalization(config.normalization)

        augmentations = [
            A.HorizontalFlip(p=0.5),
            A.VerticalFlip(p=0.5),
            A.RandomRotate90(p=0.5),
            A.Resize(height=config.train.train_image_size[0], width=config.train.train_image_size[1], always_apply=True),
        ]

        transform = A.Compose(
            augmentations,
            keypoint_params=A.KeypointParams(
                format='xys',
                label_fields=['confidences', 'is_vessel', 'is_fishing', 'is_near_shore'],
                remove_invisible=False
            )
        )

        individual_augmentations = dict(
            [(key, A.Compose(build_augmentations([value]))) for key, value in config.augs.individual.items() if value is not None]
        )

        for channel in config.dataset.channels:
            if channel not in normalization:
                normalization[channel] = A.NoOp()

        if config.sampler.sampler_type == "each_object":
            train_ds, train_sampler = dataset.get_random_crop_each_object_dataset(
                train_df,
                train_image_size=config.train.train_image_size,
                input_channels=config.dataset.channels,
                box_coder=self.box_coder,  # Pass `self.box_coder` here
                num_samples=config.sampler.num_samples,
                individual_augmentations=individual_augmentations,
                augmentations=augmentations,
                transform=transform,
                normalization=normalization,
                balance_crowd=config.sampler.balance_crowd,
                balance_near_shore=config.sampler.balance_near_shore,
                balance_vessel_type=config.sampler.balance_vessel_type,
                balance_fishing_type=config.sampler.balance_fishing_type,
                balance_per_scene=config.sampler.balance_per_scene,
                balance_location=config.sampler.balance_location,
                channels_last=config.torch.channels_last,
            )
        else:
            train_ds, train_sampler = dataset.get_random_crop_training_dataset(
                train_df,
                train_image_size=config.train.train_image_size,
                input_channels=config.dataset.channels,
                box_coder=self.box_coder, 
                num_samples=config.sampler.num_samples,
                crop_around_ship_p=config.sampler.crop_around_ship_p,
                individual_augmentations=individual_augmentations,
                augmentations=augmentations,
                normalization=normalization,
                balance_near_shore=config.sampler.balance_near_shore,
                balance_crowd=config.sampler.balance_crowd,
                balance_location=config.sampler.balance_location,
                balance_folder=config.sampler.balance_folder,
                channels_last=config.torch.channels_last,
                shore_root=shore_root,
            )

        valid_ds = dataset.get_validation_dataset(
            valid_df,
            valid_crop_size=config.train.valid_image_size,
            input_channels=config.dataset.channels,
            box_coder=self.box_coder,
            normalization=normalization,
            channels_last=config.torch.channels_last,
        )

        self.master_print(
            "Train",
            "instances",
            len(train_df),
            "dataset",
            len(train_ds),
            "sampler",
            len(train_sampler) if train_sampler else "N/A",
        )
        self.master_print("Valid", "instances", len(valid_df), "dataset", len(valid_ds))

        return train_ds, valid_ds, train_sampler, []

    def build_model(self, config: Dict, loaders: Dict) -> nn.Module:
        model = super().build_model(config)
        logger.debug(f"Train loader sample: {next(iter(loaders['train']))}")
        logger.debug(f"Valid loader sample: {next(iter(loaders['valid']))}")
        self.model = model
        self.box_coder = MultilabelCircleNetCoder(
            image_size=(config.train.train_image_size[0], config.train.train_image_size[1]),
            output_stride=4,  
            max_objects=512,  
            heatmap_encoding="umich",
            labels_encoding="circle",
            ignore_value=-1
        )
        # Initialize ProgressiveNAS with the evaluation function
        nas = ProgressiveNAS(
            base_model=model,
            search_space=config.get('nas', {}).get('search_space', {}),
            evaluation_function=self.evaluate_architecture  # Ensure this method exists
        )

        # Search for the best model architecture
        self.model = nas.search(
            num_iterations=config.get('nas', {}).get('num_iterations', 4),
            train_loader=loaders['train'],
            val_loader=loaders['valid'],
            device=self.device
        )


        # Reinitialize YOLOHead and Classifier if needed
        self.model = YOLOHead(num_classes=self.cfg.model.num_classes, anchors=self.cfg.model.anchors).to(self.device)
        self.model.classifier = ConvolutionalClassifier(num_classes=config.model.num_classes)
        self.model.to(self.device)
        logger.debug(f"Model built: {self.model}")
        if self.model is None:
            self.model = YOLOHead(num_classes=self.cfg.model.num_classes, anchors=self.cfg.model.anchors).to(self.device)
            logger.debug("Model initialized successfully.")

        return self.model  # Return the best model found by NAS

    def build_criterions(self, config: DictConfig) -> Tuple[Dict[str, nn.Module], List[Callback]]:
        criterions = {}
        callbacks = []
        
        # Initialize CenterNet Loss
        self.criterion = CircleNetLoss().to(self.device)  # Replace with your actual CenterNet loss
        
        # Initialize YOLO Loss
        self.criterion_yolo = YOLOLoss(
            num_classes=2,  # Adjust based on your dataset
            anchors=[[10, 13], [16, 30], [33, 23]]
        ).to(self.device)
        
        return criterions, callbacks

    def build_metrics(self, config, loaders, model) -> List[Callback]:
        callbacks = super().build_metrics(config, loaders, model)
        
        # Initialize CircleNet ROC AUC Metrics Callback
        metrics_callback = CircleNetRocAucMetricsCallback(prefix="train_metrics")
        callbacks.append(metrics_callback)
        
        return callbacks

    def build_distributed_params(self, config: DictConfig) -> Dict:
        fp16 = config.optimizer.get("fp16", False)

        if self.distributed:
            local_rank = torch.distributed.get_rank()
            distributed_params = {"rank": local_rank, "syncbn": True}
            if fp16:
                distributed_params["amp"] = True
            if self.cfg.find_unused:
                distributed_params["find_unused_parameters"] = True
        else:
            if fp16:
                distributed_params = {}
                distributed_params["amp"] = True
            else:
                distributed_params = False

        return distributed_params

    def build_optimizer(self, model: nn.Module, loaders: OrderedDict) -> Tuple[Optimizer, Any, List[Callback]]:
        optimizer_config = self.cfg["optimizer"]
        optimizer_use_fp16 = optimizer_config.get("fp16", False)

        optimizer_name = str(optimizer_config["name"])

        wd_on_bias = bool(optimizer_config.get("wd_on_bias", True))
        accumulation_steps = int(optimizer_config.get("accumulation", 1))
        optimizer_params = optimizer_config["params"]
        optimizer_params = scale_learning_rate_for_ddp(optimizer_params)

        optimizer = self.get_optimizer(
            model=model,
            optimizer_name=optimizer_name,
            optimizer_params=optimizer_params,
            apply_weight_decay_to_bias=wd_on_bias,
            layerwise_params=optimizer_config.get("layerwise_params", None),
        )

        if optimizer_use_fp16:
            opt_callback = AMPOptimizerCallback(accumulation_steps=accumulation_steps)
        else:
            opt_callback = OptimizerCallback(accumulation_steps=accumulation_steps, decouple_weight_decay=False)

        scheduler_params = self.cfg["scheduler"]
        num_epochs = self.cfg["train"]["epochs"]

        scheduler_name = str(scheduler_params["scheduler_name"])
        scheduler = get_scheduler(
            optimizer,
            learning_rate=optimizer_params["lr"],
            num_epochs=num_epochs,
            batches_in_epoch=len(loaders["train"]),
            **scheduler_params,
        )

        callbacks = [opt_callback]
        if isinstance(scheduler, (CyclicLR, OneCycleLRWithWarmup)):
            callbacks += [SchedulerCallback(mode="batch")]
        else:
            callbacks += [SchedulerCallback(mode="epoch")]

        self.master_print("Optimizer        :", optimizer_name)
        self.master_print("Model            :", self.cfg.model.config.slug)
        self.master_print(
            "  Parameters     :",
            count_parameters(
                model, ["backbone", "rpn", "roi_heads", "encoder", "decoder", "head", "fuse", "extra_stages", "center", "mask"]
            ),
        )
        self.master_print("  FP16           :", optimizer_use_fp16)
        self.master_print("  Learning rate  :", optimizer_params["lr"])
        self.master_print("  Weight decay   :", optimizer_params.get("weight_decay", 0))
        self.master_print("  WD on bias     :", wd_on_bias)
        self.master_print("  Accumulation   :", accumulation_steps)
        self.master_print("Params           :")
        for k, v in optimizer_params.items():
            self.master_print(f"  {k}:", v)
        self.master_print("Scheduler        :", scheduler_name)

        return optimizer, scheduler, callbacks

    def get_optimizer(
        self,
        model: nn.Module,
        optimizer_name: str,
        optimizer_params: Dict[str, Any],
        apply_weight_decay_to_bias: bool = True,
        layerwise_params=None,
    ) -> Optimizer:
        """
        Construct an Optimizer for given model
        Args:
            model: Model to optimize. Only parameters that require_grad will be used
            optimizer_name: Name of the optimizer (case-insensitive). Supports native pytorch optimizers, apex and
                optimizers from pytorch-optimizers package.
            optimizer_params: Dict of optimizer params (lr, weight_decay, eps, etc)
            apply_weight_decay_to_bias: Whether to apply weight decay on bias parameters. Default is True
        Returns:
            Optimizer instance
        """

        # Optimizer parameter groups
        if layerwise_params is not None:
            if not apply_weight_decay_to_bias:
                raise ValueError("Layerwise params and no wd on bias are mutually exclusive")

            parameters = get_lr_decay_parameters(model, optimizer_params["lr"], layerwise_params)
        else:
            if apply_weight_decay_to_bias:
                parameters = get_optimizable_parameters(model)
            else:
                default_pg, biases_pg = [], []

                for k, v in model.named_parameters():
                    if v.requires_grad:
                        if str.endswith(k, ".bias"):
                            biases_pg.append(v)  # biases
                        else:
                            default_pg.append(v)  # all else

                if apply_weight_decay_to_bias:
                    parameters = default_pg + biases_pg
                else:
                    parameters = default_pg

        optimizer_cls = get_optimizer_cls(optimizer_name)
        optimizer: Optimizer = optimizer_cls(
            parameters,
            **optimizer_params,
        )

        if not apply_weight_decay_to_bias:
            optimizer.add_param_group({"params": biases_pg, "weight_decay": 0.0})

        return optimizer

    def get_experiment_dir(self, config) -> str:
        if self.cfg.experiment:
            return self.cfg.experiment

        current_time = datetime.now().strftime("%y%m%d_%H_%M")
        experiment_slug = f"{current_time}_{config.model.config.slug}_{config.dataset.slug}_{config.loss.slug}_{config.augs.slug}"

        fold = config["dataset"].get("fold", None)
        if fold is not None:
            experiment_slug += f"_fold{fold}"

        log_dir = os.path.join("runs", experiment_slug)
        return log_dir

    def get_model(self, model_config: DictConfig) -> nn.Module:
        model: nn.Module = get_detection_model(model_config)
        print(f"[DEBUG] model.box_coder after initialization: {model.box_coder}")
        if model.box_coder is None:
            raise ValueError("model.box_coder is None after model initialization.")
        self.box_coder = model.box_coder
        self.master_print("BoxCoder")
        self.master_print(self.box_coder)
        return model

    def build_loaders(self, config):
        train_ds, valid_ds, train_sampler, dataset_callbacks = self.build_datasets(config)
        
        train_loader = DataLoader(
            train_ds,
            batch_size=config.get('batch_size', 8),
            num_workers=config.get('num_workers', 4),
            sampler=train_sampler if train_sampler else None,
            pin_memory=True,
            collate_fn=partial(centernet_collate, channels_last=train_ds.channels_last),
        )
        
        valid_loader = DataLoader(
            valid_ds,
            batch_size=config.get('batch_size', 8),
            num_workers=config.get('num_workers', 4),
            pin_memory=True,
            collate_fn=partial(centernet_collate, channels_last=valid_ds.channels_last),
        )

        return {"train": train_loader, "valid": valid_loader}, dataset_callbacks

    def calculate_bounding_boxes(self, predictions: pd.DataFrame) -> List[Dict[str, float]]:
        if predictions.empty:
            return []
        centers = predictions[['detect_scene_column', 'detect_scene_row']].values
        lengths = predictions['vessel_length_m'].values
        boxes = []
        for (cx, cy), length in zip(centers, lengths):
            if length > 0:
                half_length = length / 2
                box = {
                    'xmin': cx - half_length,
                    'ymin': cy - half_length,
                    'xmax': cx + half_length,
                    'ymax': cy + half_length,
                }
                boxes.append(box)
            else:
                boxes.append({'xmin': None, 'ymin': None, 'xmax': None, 'ymax': None})
        return boxes

    
    def encode_batch_targets(self, batch, device):
        """
        Encodes both CenterNet and YOLO targets from the batch.
        """
        # Extract relevant batch components
        centers = batch[TARGET_CENTERS_KEY].cpu().numpy()
        confidences = batch[INPUT_CONFIDENCE_KEY]
        is_vessel = batch[TARGET_VESSEL_KEY].cpu().numpy()
        is_fishing = batch[TARGET_FISHING_KEY].cpu().numpy()
        lengths = batch[TARGET_LENGTHS_KEY].cpu().numpy()
        
        # Heatmap and other CenterNet targets are already in batch
        heatmap = batch[CENTERNET_TARGET_OBJECTNESS_MAP].to(device)
        vessel_map = batch[CENTERNET_TARGET_VESSEL_MAP].to(device)
        fishing_map = batch[CENTERNET_TARGET_FISHING_MAP].to(device)
        size_map = batch[CENTERNET_TARGET_SIZE].to(device)
        offset_map = batch[CENTERNET_TARGET_OFFSET].to(device)
        
        # YOLO targets
        yolo_targets = batch[CENTERNET_TARGET_YOLO].to(device)
        
        encoded_targets = {
            'CENTERNET': {
                'objectness_map': heatmap,
                'vessel_map': vessel_map,
                'fishing_map': fishing_map,
                'size_map': size_map,
                'offset_map': offset_map,
            },
            'YOLO': yolo_targets
        }
        
        return encoded_targets

    def train_step(self, batch, batch_idx):
        self.model.train()
        with autocast(device_type=self.device.type):
            images = batch['INPUT_IMAGE_KEY'].to(self.device)
            encoded_targets = self.encode_batch_targets(batch, self.device)
            
            # Forward pass
            outputs = self.model(images)
            
            # Compute CenterNet Loss
            centernet_loss = self.criterion(
                outputs['CENTERNET_OUTPUT_OBJECTNESS_MAP'],
                encoded_targets['CENTERNET']['objectness_map'],
                outputs['CENTERNET_OUTPUT_VESSEL_MAP'],
                encoded_targets['CENTERNET']['vessel_map'],
                outputs['CENTERNET_OUTPUT_FISHING_MAP'],
                encoded_targets['CENTERNET']['fishing_map'],
                outputs['CENTERNET_OUTPUT_SIZE'],
                encoded_targets['CENTERNET']['size_map'],
                outputs['CENTERNET_OUTPUT_OFFSET'],
                encoded_targets['CENTERNET']['offset_map'],
            )
            
            # Compute YOLO Loss
            yolo_loss = self.criterion_yolo(
                outputs['CENTERNET_OUTPUT_YOLO'],
                encoded_targets['YOLO']
            )
            
            # Combine Losses
            total_loss = centernet_loss + yolo_loss
        
        # Scale the loss
        scaled_loss = total_loss / self.accumulation_steps
        self.scaler.scale(scaled_loss).backward()
    
        if (batch_idx + 1) % self.accumulation_steps == 0:
            self.scaler.step(self.optimizer)
            self.scaler.update()
            self.optimizer.zero_grad()
    
        return {'loss': total_loss}

    def evaluate_architecture(self, model, train_loader, val_loader, device):
        if self.model is None or self.criterion is None or self.criterion_yolo is None:
            raise RuntimeError("Model or criterion not initialized.")
        
        self.device = device
        self.model.eval()
        total_centernet_loss = 0.0
        total_yolo_loss = 0.0
        total_samples = 0
        
        with torch.no_grad():
            for batch_idx, batch in enumerate(val_loader):
                images = batch['INPUT_IMAGE_KEY'].to(device)
                encoded_targets = self.encode_batch_targets(batch, device)
        
                # Forward pass
                outputs = self.model(images)
        
                # Compute CenterNet Loss
                centernet_loss = self.criterion(
                    outputs['CENTERNET_OUTPUT_OBJECTNESS_MAP'],
                    encoded_targets['CENTERNET']['objectness_map'],
                    outputs['CENTERNET_OUTPUT_VESSEL_MAP'],
                    encoded_targets['CENTERNET']['vessel_map'],
                    outputs['CENTERNET_OUTPUT_FISHING_MAP'],
                    encoded_targets['CENTERNET']['fishing_map'],
                    outputs['CENTERNET_OUTPUT_SIZE'],
                    encoded_targets['CENTERNET']['size_map'],
                    outputs['CENTERNET_OUTPUT_OFFSET'],
                    encoded_targets['CENTERNET']['offset_map'],
                )
        
                # Compute YOLO Loss
                yolo_loss = self.criterion_yolo(
                    outputs['CENTERNET_OUTPUT_YOLO'],
                    encoded_targets['YOLO']
                )
        
                # Aggregate Losses
                total_centernet_loss += centernet_loss.item() * images.size(0)
                total_yolo_loss += yolo_loss.item() * images.size(0)
                total_samples += images.size(0)
        
        average_centernet_loss = total_centernet_loss / total_samples
        average_yolo_loss = total_yolo_loss / total_samples
        
        print(f"[DEBUG] Average CenterNet Loss: {average_centernet_loss}")
        print(f"[DEBUG] Average YOLO Loss: {average_yolo_loss}")
        
        return -(average_centernet_loss + average_yolo_loss)


    def parse_evaluation_results(self, output_dir):
        """
        Implement parsing of evaluation results to extract metrics like average loss.
        This depends on how 'evaluate_on_scenes' outputs results.
        """
        # Example:
        thresholds_summary_path = os.path.join(output_dir, "nas_evaluation__step_4_thresholds_summary_.csv")
        if os.path.exists(thresholds_summary_path):
            thresholds_summary = pd.read_csv(thresholds_summary_path)
            average_loss = thresholds_summary['aggregate'].mean()
            return average_loss
        else:
            raise FileNotFoundError(f"Thresholds summary not found at {thresholds_summary_path}")

    def train(self):
        config = self.cfg
        self.loaders, dataset_callbacks = self.build_loaders(config)
        self.build_criterions(config)
        model = self.build_model(config, self.loaders)
        self.box_coder = model.box_coder 

        # Initialize YOLOLoss with the correct number of classes and anchors
        num_classes = config.get('num_classes', 1)  # Adjust as per your dataset
        anchors = config.get('anchors', [[10,13], [16,30], [33,23]])  # Example anchors

        self.criterion = YOLOLoss(num_classes=num_classes, anchors=anchors).to(self.device)

        criterions, criterions_callbacks = self.build_criterions(config)
        metric_callbacks = self.build_metrics(config, self.loaders, model)
        all_callbacks = dataset_callbacks + criterions + metric_callbacks
        optimizer, scheduler, optimizer_callbacks = self.build_optimizer(self.model, self.loaders) 
        all_callbacks += optimizer_callbacks
        nas = ProgressiveNAS(
            base_model=model,
            search_space=config.get('nas', {}).get('search_space', {}),
            evaluation_function=self.evaluate_architecture
        )
        self.model = nas.search(
            num_iterations=config.get('nas', {}).get('num_iterations', 4),
            train_loader=self.loaders['train'],
            val_loader=self.loaders['valid'],
            device=self.device
        )
        model = self.model  # Update reference to best_model

        experiment_dir = self.get_experiment_dir(config)
        self.experiment_dir = experiment_dir

        if self.is_master:
            os.makedirs(experiment_dir, exist_ok=True)
            dst_config_fname = os.path.join(experiment_dir, "config.yaml")
            with open(dst_config_fname, "w") as f:
                OmegaConf.save(self.cfg, f)

        runner_config = config.get("runner", {})
        input_key = runner_config.get("input_key", INPUT_SCENE_ID_KEY)
        output_key = runner_config.get("output_key", None)

        if input_key is not None and not isinstance(input_key, str):
            input_key = OmegaConf.to_container(input_key, resolve=True)
        if output_key is not None and not isinstance(output_key, str):
            output_key = OmegaConf.to_container(output_key, resolve=True)

        device = TorchDevice.choose_torch_device()
        runner = SupervisedRunner(input_key=input_key, output_key=output_key, device=self.device)

        # Initialize GradScaler for mixed precision training
        self.scaler = GradScaler()

        # Set up gradient accumulation
        self.accumulation_steps = config.get('accumulation_steps', 1)

        extra_callbacks = [TimerCallback()]

        runner.train(
            fp16=self.build_distributed_params(config),
            model=self.model,
            criterion=self.criterion,  # Use the initialized YOLOLoss
            optimizer=optimizer,
            scheduler=scheduler,
            callbacks=all_callbacks,
            loaders=self.loaders,
            logdir=experiment_dir,
            num_epochs=config.train.epochs,
            verbose=True,
            main_metric=config.runner.main_metric,
            minimize_metric=config.runner.main_metric_minimize,
            checkpoint_data=self.get_checkpoint_data(),
            train_step=self.train_step,  # Ensure train_step is correctly referenced
        )

        self.on_experiment_finished()

    def get_checkpoint_data(self):
        return {
            "config": OmegaConf.to_container(self.cfg, resolve=True),
            "version": get_version(),
        }

    def on_experiment_finished(self):
        pass

    def build_quircks(self, config: DictConfig) -> List[Callback]:
        extra_callbacks = []

        if config.ema.callback is not None:
            cb = hydra.utils.instantiate(config.ema.callback)
            extra_callbacks.append(cb)
            self.master_print("Using EMA Callback", cb)

        return extra_callbacks

class TrainOnlyCriterionCallback(CriterionCallback):
    def on_batch_end(self, runner) -> None:
        if runner.is_train_loader:
            return super(TrainOnlyCriterionCallback, self).on_batch_end(runner)
        else:
            runner.batch_metrics[self.prefix] = torch.tensor(0, device="mps")

class CombinedYOLOCenterNetModel(nn.Module):
    def __init__(self, backbone, centernet_head, yolo_head):
        super().__init__()
        self.backbone = backbone
        self.centernet_head = centernet_head
        self.yolo_head = yolo_head
    
    def forward(self, x):
        features = self.backbone(x)
        
        # CenterNet predictions
        centernet_outputs = self.centernet_head(features)
        
        # YOLO predictions
        yolo_outputs = self.yolo_head(features)
        
        # Combine outputs into a dictionary
        return {
            'CENTERNET_OUTPUT_OBJECTNESS_MAP': centernet_outputs['objectness_map'],
            'CENTERNET_OUTPUT_VESSEL_MAP': centernet_outputs['vessel_map'],
            'CENTERNET_OUTPUT_FISHING_MAP': centernet_outputs['fishing_map'],
            'CENTERNET_OUTPUT_SIZE': centernet_outputs['size_map'],
            'CENTERNET_OUTPUT_OFFSET': centernet_outputs['offset_map'],
            'CENTERNET_OUTPUT_YOLO': yolo_outputs,
        }
    
@hydra.main(config_path="configs", config_name="multilabel_circle_net", version_base=None)
def main(cfg: DictConfig) -> None:
    torch.mps.empty_cache()
    torch.manual_seed(int(cfg.seed))

    torch.set_anomaly_enabled(cfg.torch.detect_anomaly)

    torch.backends.cudnn.deterministic = cfg.torch.deterministic
    torch.backends.cudnn.benchmark = cfg.torch.benchmark

    # The flag below controls whether to allow TF32 on matmul. This flag defaults to True.
    torch.backends.cuda.matmul.allow_tf32 = cfg.torch.cuda_allow_tf32
    # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True.
    torch.backends.cudnn.allow_tf32 = cfg.torch.cudnn.allow_tf32

    if cfg.dataset.data_dir is None:
        raise ValueError("--data-dir must be set")

    CenternetPipeline(cfg).train()

if __name__ == "__main__":
    main()

#test