after these modifications I think I receive 5  layer which makes 5 * 256 = 1280 

so review these files to help modify the give group weight of size which gives 1024 only and help fix the error at the end of the question

#first file centernet_unet.py

from functools import partial
from typing import List, Dict
from omegaconf import DictConfig
from pytorch_toolbelt.modules import (
    ACT_RELU,
    decoders as D,
    EncoderModule,
    ResidualDeconvolutionUpsample2d,
)
from torch import nn
import torch
import timm
from hydra.utils import instantiate
import random
import torch.nn.functional as F
from xview3.centernet.models.custom_encoder import CustomTimmEncoder
from .unet_blocks import IRBlock, get_unet_block, SqueezeExcitation
from .necks import FPNNeck
from xview3.nas import NAS as NeuralArchitectureSearch
from .heads.yolo_head import YOLOHead
from .transformer_classifier import ConvolutionalClassifier
from xview3.common import FilterResponseNormalization, ResidualBlock, FeaturePyramidNetwork
from ..constants import (
    CENTERNET_OUTPUT_CLASS_MAP,
    CENTERNET_OUTPUT_OBJECTNESS_MAP,
    CENTERNET_OUTPUT_OFFSET,
    CENTERNET_OUTPUT_SIZE,
    CENTERNET_OUTPUT_VESSEL_MAP,
    CENTERNET_OUTPUT_FISHING_MAP,
)
__all__ = ["CenterNetUNetModel"]

class CenterNetUNetModel(nn.Module):
    def __init__(
        self,
        encoder: EncoderModule,
        decoder_features: List[int],
        head: nn.Module,
        box_coder,
        neck_out_channels: int = 256,  # Add this parameter
        input_upsample_factor: int = 1,
        activation=ACT_RELU,
        num_extra_blocks=0,
        block_type="UnetBlock",
        upsample_type="bilinear",
        extra_block=partial(IRBlock, stride=2),
        num_classes=2,
        anchors=[(10, 13), (16, 30), (33, 23)],
        **kwargs
    ):
        super().__init__()
        self.output_stride = getattr(encoder, 'strides', [32])[0]
        self.initial_upsample = (
            nn.UpsamplingBilinear2d(scale_factor=input_upsample_factor)
            if input_upsample_factor != 1 else nn.Identity()
        )
        unet_block = get_unet_block(block_type, activation=activation)
        self.encoder = encoder
        self.extra_encoder_blocks = nn.ModuleList(
            [extra_block(encoder.channels[-1], encoder.channels[-1]) for _ in range(num_extra_blocks)]
        )
        upsample_block = {
            "nearest": nn.UpsamplingNearest2d,
            "bilinear": nn.UpsamplingBilinear2d,
            "rdtsc": ResidualDeconvolutionUpsample2d,
            "shuffle": nn.PixelShuffle,
        }[upsample_type]

        if hasattr(encoder, 'channels'):
            encoder_channels = list(encoder.channels)
        else:
            encoder_channels = [32, 24, 40, 112, 320]  # Default for EfficientNet
        encoder_channels += [encoder_channels[-1]] * num_extra_blocks

        #self.neck = FPNNeck(self.decoder_channels, neck_out_channels)
        #self.neck_conv = nn.Conv2d(self.neck_output_channels, neck_out_channels, kernel_size=1)

        self.head = head
        self.head_output_channels = head.out_channels  # Ensure your head module defines 'out_channels'

        # Ensure self.nas is a module that can be called during forward
        self.nas = NeuralArchitectureSearch(
            search_space={
                'num_layers': (2, 5),
                'num_filters': (32, 128),
                'kernel_size': (3, 5),
                'activation': ['relu', 'leaky_relu']
            },
            evaluation_function=self.evaluate_architecture
        )
        self.nas_output_channels = self.nas.output_channels  # Define this in your NAS module

        self.yolo_head = YOLOHead(num_classes=num_classes, anchors=anchors)
        self.yolo_output_channels = self.yolo_head.num_anchors * (num_classes + 5)  # Adjust as per YOLO implementation

        self.classifier = ConvolutionalClassifier(in_channels=neck_out_channels, num_classes=num_classes)
        self.classifier_output_channels = self.classifier.out_channels
        self.box_coder = box_coder
        self.head_output_channels = getattr(self.head, 'out_channels', 256)
        self.nas_output_channels = getattr(self.nas, 'output_channels', 256)
        self.yolo_output_channels = self.yolo_head.num_anchors * (num_classes + 5)
        self.classifier_output_channels = getattr(self.classifier, 'out_channels', num_classes)

        # Ensure all channel values are integers
        total_input_channels = (
            int(self.head_output_channels) +
            int(self.nas_output_channels) +
            int(self.yolo_output_channels) +
            int(self.classifier_output_channels)
        )
        self.fusion = nn.Conv2d(total_input_channels, self.head_output_channels, kernel_size=1)

        self.dropout = nn.Dropout(p=0.5)

        #self.decoder_channels = decoder_features

        
        #encoder_channels = list(encoder.channels)
        self.num_decoded_features = len(encoder_channels)
        num_decoder_stages = self.num_decoded_features
        self.neck_output_channels = neck_out_channels * self.num_decoded_features
        print(f"[DEBUG] neck_output_channels: {self.neck_output_channels}")
        num_decoder_stages = self.num_decoded_features
        # Adjust decoder_channels if necessary
        
        self.decoder_channels = decoder_features

        encoder_channels = list(encoder.channels)
        num_blocks = len(encoder_channels) - 1  # UNetDecoder expects decoder_features of length len(encoder_channels) - 1

        # Adjust decoder_channels to match num_blocks
        if len(self.decoder_channels) < num_blocks:
            # Extend decoder_channels by repeating the last value
            self.decoder_channels.extend(
                [self.decoder_channels[-1]] * (num_blocks - len(self.decoder_channels))
            )
        elif len(self.decoder_channels) > num_blocks:
            # Trim decoder_channels to the required length
            self.decoder_channels = self.decoder_channels[:num_blocks]

        print(f"[DEBUG] encoder_channels: {encoder_channels}")  # Should be of length 5
        print(f"[DEBUG] decoder_channels: {self.decoder_channels}")  # Should be of length 4
        print(f"[DEBUG] num_blocks: {num_blocks}")

        self.decoder = D.UNetDecoder(
            encoder_channels,
            self.decoder_channels,
            unet_block=unet_block,
            upsample_block=upsample_block,
        )
        self.neck = FPNNeck(self.decoder_channels, neck_out_channels)
        self.neck_conv = nn.Conv2d(
            in_channels=self.neck_output_channels,
            out_channels=neck_out_channels,
            kernel_size=1
        )

        print(f"[DEBUG] neck_output_channels: {self.neck_output_channels}")

        self.batch_norm = FilterResponseNormalization(self.neck_output_channels)

    def evaluate_architecture(self, architecture):
        # Implement a simple evaluation function
        return random.random()  # Replace with actual evaluation logic

    def forward(self, x):
        x = self.initial_upsample(x)
        print(f"[DEBUG] x shape after initial_upsample: {x.shape}")
        feature_maps = self.encoder(x)
        print(f"[DEBUG] feature_maps type: {type(feature_maps)}")
        if isinstance(feature_maps, list):
            for idx, fmap in enumerate(feature_maps):
                print(f"[DEBUG] feature_maps[{idx}] shape: {fmap.shape}")
        else:
            print("[ERROR] feature_maps is not a list")
            print(f"feature_maps shape: {feature_maps.shape}")
            print(f"feature_maps type: {type(feature_maps)}")
            # Convert feature_maps to a list if needed
            feature_maps = [feature_maps]
        for extra_block in self.extra_encoder_blocks:
            feature_maps.append(extra_block(feature_maps[-1]))
        decoded_features = self.decoder(feature_maps)
        print(f"[DEBUG] decoded_features type: {type(decoded_features)}")
        if isinstance(decoded_features, list):
            for idx, fmap in enumerate(decoded_features):
                print(f"[DEBUG] decoded_features[{idx}] shape: {fmap.shape}")
        else:
            print("[ERROR] decoded_features is not a list")
            print(f"decoded_features shape: {decoded_features.shape}")
            print(f"decoded_features type: {type(decoded_features)}")
            # Convert decoded_features to a list if needed
            decoded_features = [decoded_features]
        neck_features = self.neck(decoded_features)
        print(f"[DEBUG] neck_features shape before batch_norm: {neck_features.shape}")
        print(f"[DEBUG] neck_features channels: {neck_features.size(1)}")
        print(f"[DEBUG] neck_features shape before batch_norm: {neck_features.shape}")
        neck_features = self.batch_norm(neck_features)
        print(f"[DEBUG] neck_features shape after batch_norm: {neck_features.shape}")
        neck_features = self.dropout(neck_features)
        neck_features = self.neck_conv(neck_features)

        #self.head.channels = self.nas_output_channels  # Assuming self.nas_output_channels is the correct channel size
        head_output = self.head(neck_features)
        nas_output = self.nas(neck_features)
        yolo_output = self.yolo_head(neck_features)
        classification_output = self.classifier(neck_features)

        # Fusion layer
        fused_output = self.fusion(torch.cat([head_output, nas_output, yolo_output, classification_output], dim=1))

        return {
            CENTERNET_OUTPUT_OBJECTNESS_MAP: fused_output[:, :1],
            CENTERNET_OUTPUT_VESSEL_MAP: fused_output[:, 1:2],
            CENTERNET_OUTPUT_FISHING_MAP: fused_output[:, 2:3],
            CENTERNET_OUTPUT_SIZE: fused_output[:, 3:4],
            CENTERNET_OUTPUT_OFFSET: fused_output[:, 4:6],
        }

    @classmethod
    def from_config(cls, **config):
        # Extract and pop the encoder configuration
        encoder_config = config.pop('encoder')
        num_channels = config.pop('num_channels', 3)
        model_name = encoder_config.get('model_name', 'tf_efficientnet_b5_ns')
        pretrained = encoder_config.get('pretrained', False)

        # Instantiate the custom encoder
        encoder_module = CustomTimmEncoder(
            model_name=model_name,
            num_channels=num_channels,
            features_only=True,
            pretrained=pretrained
        )

        # Pop other configurations
        input_upsample_factor = config.pop("input_upsample_factor", 1)
        activation = config.pop('activation', ACT_RELU)
        num_extra_blocks = config.pop('num_extra_blocks', 0)
        num_classes = config.pop("num_classes", 2)
        anchors = config.pop("anchors", [(10, 13), (16, 30), (33, 23)])

        # Pop decoder configuration
        decoder_config = config.pop('decoder')
        decoder_features = decoder_config['channels']
        block_type = decoder_config.get("block_type", "UnetBlock")
        upsample_type = decoder_config.get("upsample_type", "bilinear")

        neck_out_channels = config.get('neck_out_channels', 256)
        neck_output_channels = neck_out_channels * len(decoder_features)
        print(f"Neck output channels: {neck_output_channels}")

        # Instantiate head and box_coder
        head_config = config.pop('head')
        head = instantiate(head_config, channels=neck_output_channels)

        box_coder_config = config.pop('box_coder')
        box_coder = instantiate(
            box_coder_config,
            output_stride=encoder_module.strides[0] // (head.upsample_factor * input_upsample_factor),
        )

        # Now initialize the model instance with the remaining config
        return cls(
            encoder=encoder_module,
            decoder_features=decoder_features,
            head=head,
            box_coder=box_coder,
            neck_out_channels=neck_out_channels,
            input_upsample_factor=input_upsample_factor,
            activation=activation,
            block_type=block_type,
            upsample_type=upsample_type,
            num_extra_blocks=num_extra_blocks,
            num_classes=num_classes,
            anchors=anchors,
            **config
        )

def adjust_encoder_input_channels(encoder_module, in_chans):
    # Access the underlying encoder model
    if hasattr(encoder_module, 'encoder'):
        base_encoder = encoder_module.encoder
    else:
        base_encoder = encoder_module

    if hasattr(base_encoder, 'conv_stem'):
        old_conv = base_encoder.conv_stem
        print(f"[DEBUG] Before adjustment, encoder.conv_stem.in_channels: {old_conv.in_channels}")

        new_conv = nn.Conv2d(
            in_channels=in_chans,
            out_channels=old_conv.out_channels,
            kernel_size=old_conv.kernel_size,
            stride=old_conv.stride,
            padding=old_conv.padding,
            bias=old_conv.bias is not None
        )

        # Initialize new_conv weights
        with torch.no_grad():
            # Copy weights for existing channels
            num_existing_in_channels = min(old_conv.in_channels, in_chans)
            new_conv.weight[:, :num_existing_in_channels, ...] = old_conv.weight[:, :num_existing_in_channels, ...]
            if in_chans > old_conv.in_channels:
                # Initialize added channels
                torch.nn.init.kaiming_normal_(new_conv.weight[:, old_conv.in_channels:, ...])

        base_encoder.conv_stem = new_conv
        print(f"[DEBUG] After adjustment, encoder.conv_stem.in_channels: {base_encoder.conv_stem.in_channels}")
    else:
        raise NotImplementedError("Adjusting input channels for this encoder is not implemented.")
    return encoder_module

class EnhancedCenterNetUNet(nn.Module):
    def __init__(self, in_channels, num_classes):
        super().__init__()
        self.backbone = CenterNetUNetModel(in_channels, num_classes)
        self.fpn = FeaturePyramidNetwork([64, 128, 256, 512], 256)
        self.se_blocks = nn.ModuleList([SqueezeExcitation(c) for c in [64, 128, 256, 512]])

    def forward(self, x):
        features = self.backbone(x)
        for i, feature in enumerate(features):
            features[i] = self.se_blocks[i](feature)
        return self.fpn(features)


#from yolo_head.py file the yolo head class

class YOLOHead(nn.Module):
    def __init__(self, num_classes, anchors):
        super(YOLOHead, self).__init__()
        self.num_classes = num_classes
        self.anchors = anchors
        self.num_anchors = len(anchors)
        self.fpn = FPNFuse([256, 512, 1024], 256)
        self.out_channels = self.num_anchors * (5 + self.num_classes)
        self.conv = nn.Conv2d(256, self.out_channels, 1)

    def forward(self, x):
        print(f"[DEBUG] x shape in YOLOHead before fpn: {x.shape}")
        x = self.fpn(x)
        print(f"[DEBUG] x shape in YOLOHead after fpn: {x.shape}")
        batch_size, _, height, width = x.shape
        # Continue with the rest of the code
        return self.conv(x).view(batch_size, len(self.anchors), 5 + self.num_classes, height, width).permute(0, 1, 3, 4, 2)

#the next file is the decoupled_head.py contains the ceneternet head

from collections import OrderedDict
from typing import Dict, Tuple, List

from pytorch_toolbelt.modules import ACT_RELU, get_activation_block, conv1x1
from torch import nn, Tensor

from ...constants import (
    CENTERNET_OUTPUT_VESSEL_MAP,
    CENTERNET_OUTPUT_FISHING_MAP,
    CENTERNET_OUTPUT_OBJECTNESS_MAP,
    CENTERNET_OUTPUT_SIZE,
    CENTERNET_OUTPUT_OFFSET,
)

__all__ = ["DecoupledHeadGroupNormLateShuffle"]


class FeatureRecombination(nn.Module):
    def __init__(self, channels, modulating_features):
        super().__init__()
        self.project = conv1x1(modulating_features, channels)

    def forward(self, x, modulation):
        return x * self.project(modulation).sigmoid()


def build_regression_block(
    input_channels: int,
    embedding_size: int,
    dropout_rate: float,
    num_blocks: int,
    activation: str,
    extra_final_layers: List[Tuple[str, nn.Module]],
    num_groups=16,
):
    activation_block = get_activation_block(activation)

    layers = [
        (
            "project",
            nn.Conv2d(input_channels, embedding_size, kernel_size=(1, 1), padding=0, bias=False),
        ),
    ]

    for layer_index in range(num_blocks):
        layers += [
            (
                f"conv{layer_index}",
                nn.Conv2d(embedding_size, embedding_size, kernel_size=(3, 3), padding=1, bias=False),
            ),
            (f"bn{layer_index}", nn.GroupNorm(num_groups, embedding_size)),
            (f"act{layer_index}", activation_block(inplace=True)),
        ]
    layers += [
        ("drop", nn.Dropout2d(dropout_rate, inplace=False)),
        ("conv1", nn.Conv2d(embedding_size, embedding_size, kernel_size=(1, 1), padding=0, bias=False)),
    ] + extra_final_layers

    return nn.Sequential(OrderedDict(layers))


class DecoupledHeadGroupNormLateShuffle(nn.Module):
    upsample_factor: int = 2

    def __init__(
        self,
        channels: int,
        classifier_dim: int,
        dropout_rate=0.25,
        objectness_dim: int = 32,
        size_dim: int = 32,
        offset_dim: int = 32,
        num_blocks: int = 3,
        activation: str = ACT_RELU,
        num_groups=16,
    ):
        super().__init__()
        self.objectness = build_regression_block(
            channels,
            embedding_size=objectness_dim,
            dropout_rate=dropout_rate,
            activation=activation,
            num_groups=num_groups,
            num_blocks=num_blocks,
            extra_final_layers=[
                (("shuffle"), nn.PixelShuffle(2)),
                ("final", nn.Conv2d(objectness_dim // 4, 1, kernel_size=(3, 3), padding=1)),
            ],
        )

        self.classifier = build_regression_block(
            channels,
            embedding_size=classifier_dim,
            dropout_rate=dropout_rate,
            num_groups=num_groups,
            activation=activation,
            num_blocks=num_blocks,
            extra_final_layers=[],
        )

        self.size = build_regression_block(
            channels,
            embedding_size=size_dim,
            dropout_rate=dropout_rate,
            activation=activation,
            num_groups=num_groups,
            num_blocks=num_blocks,
            extra_final_layers=[],
        )

        self.classifier_recombination = FeatureRecombination(classifier_dim, size_dim)
        self.classifier_up = nn.Sequential(
            OrderedDict(
                [
                    ("conv1", nn.Conv2d(classifier_dim, classifier_dim, kernel_size=(3, 3), padding=1)),
                    ("shuffle", nn.PixelShuffle(2)),
                    ("final", nn.Conv2d(classifier_dim // 4, 2, kernel_size=(3, 3), padding=1)),
                ]
            )
        )
        self.size_recombination = FeatureRecombination(size_dim, classifier_dim)
        self.size_up = nn.Sequential(
            OrderedDict(
                [
                    ("conv1", nn.Conv2d(size_dim, size_dim, kernel_size=(3, 3), padding=1)),
                    ("shuffle", nn.PixelShuffle(2)),
                    ("final", nn.Conv2d(size_dim // 4, 1, kernel_size=(3, 3), padding=1)),
                    ("activation", nn.Softplus()),
                ]
            )
        )

        if offset_dim:
            self.offset = build_regression_block(
                channels,
                embedding_size=offset_dim,
                dropout_rate=dropout_rate,
                activation=activation,
                num_groups=num_groups,
                num_blocks=num_blocks,
                extra_final_layers=[
                    (("shuffle"), nn.PixelShuffle(2)),
                    ("final", nn.Conv2d(offset_dim // 4, 2, kernel_size=(3, 3), padding=1)),
                ],
            )
        else:
            self.offset = None

        # Determine if offset is included
        if self.offset is not None:
            self.out_channels = 6  # Total output channels
        else:
            self.out_channels = 4  # Exclude offset channels

        self._init_weights()

    def _init_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight)
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        # self.classifier_up.final.bias.data.fill_(0)
        # self.objectness.final.bias.data.fill_(-4)

        self.objectness.final.bias.data.fill_(-5)
        self.classifier_up.final.bias.data.fill_(-1)
        self.size_up.final.bias.data.fill_(2)

    def forward(self, features: Tensor) -> Dict[str, Tensor]:
        class_features = self.classifier(features)
        length_features = self.size(features)

        new_class_features = self.classifier_recombination(class_features, length_features)
        new_length_features = self.size_recombination(length_features, class_features)

        predicted_classes = self.classifier_up(new_class_features)
        predicted_length = self.size_up(new_length_features)

        output = {
            CENTERNET_OUTPUT_OBJECTNESS_MAP: self.objectness(features),
            CENTERNET_OUTPUT_VESSEL_MAP: predicted_classes[:, 0:1],
            CENTERNET_OUTPUT_FISHING_MAP: predicted_classes[:, 1:2],
            CENTERNET_OUTPUT_SIZE: predicted_length,
        }
        if self.offset is not None:
            output[CENTERNET_OUTPUT_OFFSET] = self.offset(features).clip(0, 1)
        return output

#from the necks file here's the FBNNeck class

class FPNNeck(nn.Module):
    def __init__(self, channels: List[int], out_channels: int):
        super().__init__()
        self.fpn = FPNFuse(channels, out_channels, mode='bilinear', align_corners=False)
        self.out_conv = nn.Conv2d(out_channels * len(channels), out_channels, kernel_size=1)

    def forward(self, feature_maps: List[Tensor]) -> Tensor:
        fpn_output = self.fpn(feature_maps)
        return self.out_conv(fpn_output)

#also from the transformer_classifier file here's the  ConvolutionalClassifier class

class ConvolutionalClassifier(nn.Module):
    def __init__(self, in_channels, num_classes):
        super(ConvolutionalClassifier, self).__init__()
        self.conv = nn.Conv2d(in_channels, num_classes, kernel_size=1)
        self.out_channels = num_classes

    def forward(self, x):
        return self.conv(x)

#almost here the nas.py file

import torch
import torch.nn as nn
import torch.nn.functional as F
import random
import numpy as np
from skopt import gp_minimize
from skopt.space import Real, Integer, Categorical
import math
import copy
from torch import Tensor
from torch.optim import Adam
from torch.utils.data import DataLoader
from xview3.centernet.bboxer.multilabel_circle_coder import MultilabelCircleNetCoder

class ProgressiveNAS(nn.Module):
    def __init__(self, base_model, search_space):
        super(ProgressiveNAS, self).__init__()
        self.base_model = base_model
        self.search_space = search_space
        self.box_coder = MultilabelCircleNetCoder(
            image_size=(1024, 1024),
            output_stride=2,
            max_objects=512,
            heatmap_encoding="umich",
            labels_encoding="circle",
            ignore_value=-1
        )
        self.population_size = 20
        self.tournament_size = 5
        self.output_channels = 256 

    def search(self, num_iterations, train_loader, val_loader):
        population = [self.mutate_model(self.base_model) for _ in range(self.population_size)]
        
        for _ in range(num_iterations):
            scores = [self.evaluate_model(model, train_loader, val_loader) for model in population]
            
            new_population = []
            for _ in range(self.population_size):
                parent1 = self.tournament_selection(population, scores)
                parent2 = self.tournament_selection(population, scores)
                child = self.crossover(parent1, parent2)
                child = self.mutate_model(child)
                new_population.append(child)
            
            population = new_population

        best_model = max(population, key=lambda model: self.evaluate_model(model, train_loader, val_loader))
        return best_model

    def tournament_selection(self, population, scores):
        tournament = random.sample(list(zip(population, scores)), self.tournament_size)
        return max(tournament, key=lambda x: x[1])[0]

    def crossover(self, parent1, parent2):
        child = copy.deepcopy(parent1)
        for name, param in child.named_parameters():
            if random.random() < 0.5:
                param.data = parent2.state_dict()[name].clone()
        return child

    def mutate_model(self, model):
        new_model = copy.deepcopy(model)
        for param in new_model.parameters():
            if random.random() < 0.1:
                param.data += torch.randn_like(param.data) * 0.1
        
        # Structure mutation
        if random.random() < 0.2:
            new_layer = nn.Linear(64, 64)  # Example layer, adjust as needed
            if hasattr(new_model, 'encoder') and hasattr(new_model.encoder, 'add_module'):
                new_model.encoder.add_module(f"extra_layer_{id(new_layer)}", new_layer)
            elif hasattr(new_model, 'decoder') and hasattr(new_model.decoder, 'add_module'):
                new_model.decoder.add_module(f"extra_layer_{id(new_layer)}", new_layer)
        
        return new_model

    def evaluate_model(self, model, train_loader, val_loader):
        # Set the model to evaluation mode
        model.eval()
        total_loss = 0.0
        total_correct = 0
        total_samples = 0

        with torch.no_grad():
            for batch_idx, batch in enumerate(train_loader):
                # Debug batch structure
                print(f"[DEBUG] Batch index: {batch_idx}")
                print(f"[DEBUG] Batch keys: {list(batch.keys())}")
                
                # Extract inputs
                inputs = batch.get('INPUT_IMAGE_KEY')
                if inputs is None:
                    raise ValueError("Batch does not contain 'INPUT_IMAGE_KEY'. Available keys: {}".format(list(batch.keys())))
                
                # Extract targets
                targets = {
                    'CENTERNET_TARGET_OBJECTNESS_MAP': batch.get('CENTERNET_TARGET_OBJECTNESS_MAP'),
                    'CENTERNET_TARGET_VESSEL_MAP': batch.get('CENTERNET_TARGET_VESSEL_MAP'),
                    'CENTERNET_TARGET_FISHING_MAP': batch.get('CENTERNET_TARGET_FISHING_MAP'),
                    'CENTERNET_TARGET_SIZE': batch.get('CENTERNET_TARGET_SIZE'),
                    'CENTERNET_TARGET_OFFSET': batch.get('CENTERNET_TARGET_OFFSET'),
                }
                # Ensure all targets are present
                if None in targets.values():
                    missing_keys = [k for k, v in targets.items() if v is None]
                    raise ValueError(f"Batch missing target keys: {missing_keys}")
                
                # Move data to device
                device = next(model.parameters()).device
                inputs = inputs.to(device)
                targets = {k: v.to(device) for k, v in targets.items()}

                # Proceed with evaluation
                outputs = model(inputs)
                loss = self.calculate_loss(outputs, targets)

                # Update metrics (This will depend on your task)
                total_loss += loss.item() * inputs.size(0)
                total_samples += inputs.size(0)
                # For object detection, calculating 'accuracy' might not be straightforward.
                # You may need to implement a custom metric, e.g., Average Precision.
                
        # Return evaluation score (e.g., total loss)
        average_loss = total_loss / total_samples
        return -average_loss  # Negative loss if you're maximizing the evaluation score
        
    def calculate_loss(self, outputs, targets):
        # Assuming outputs and targets are dictionaries with matching keys
        # You should define the loss functions for each output-target pair
        total_loss = 0

        # Heatmap loss (e.g., Focal Loss)
        heatmap_loss = self.focal_loss(outputs['heatmap'], targets['CENTERNET_TARGET_OBJECTNESS_MAP'])
        total_loss += heatmap_loss

        # Size loss (e.g., L1 Loss)
        size_loss = F.l1_loss(outputs['size'], targets['CENTERNET_TARGET_SIZE'])
        total_loss += size_loss

        # Offset loss (e.g., L1 Loss)
        offset_loss = F.l1_loss(outputs['offset'], targets['CENTERNET_TARGET_OFFSET'])
        total_loss += offset_loss

        return total_loss

    def focal_loss(self, pred, gt):
        # Implement Focal Loss as used in CenterNet
        pos_inds = gt.eq(1).float()
        neg_inds = gt.lt(1).float()
        neg_weights = torch.pow(1 - gt, 4)

        pred = torch.clamp(pred, 1e-6, 1 - 1e-6)
        pos_loss = -torch.log(pred) * torch.pow(1 - pred, 2) * pos_inds
        neg_loss = -torch.log(1 - pred) * torch.pow(pred, 2) * neg_weights * neg_inds

        num_pos  = pos_inds.float().sum()
        loss = (pos_loss + neg_loss).sum() / num_pos.clamp(min=1)
        return loss

class NAS(nn.Module):
    def __init__(self, search_space, evaluation_function):
        super(NAS, self).__init__()
        self.search_space = search_space
        self.evaluation_function = evaluation_function
        self.layers = nn.ModuleList()
        self.output_channels = 256

    def build_architecture(self):
        architecture = self.sample_architecture()
        num_layers = int(architecture.get('num_layers', 2))
        num_filters = int(architecture.get('num_filters', 64))
        kernel_size = int(architecture.get('kernel_size', 3))
        activation = architecture.get('activation', 'relu')

        input_channels = 256  # Assuming neck_features has 256 channels

        for i in range(num_layers):
            conv = nn.Conv2d(input_channels, num_filters, kernel_size, padding=kernel_size // 2, bias=False)
            bn = nn.BatchNorm2d(num_filters)
            act = nn.ReLU(inplace=True) if activation == 'relu' else nn.LeakyReLU(0.1, inplace=True)
            self.layers.append(nn.Sequential(conv, bn, act))
            input_channels = num_filters

        self.output_channels = num_filters

    def sample_architecture(self):
        # For simplicity, we'll randomly sample from the search space
        architecture = {}
        for key, value_range in self.search_space.items():
            if isinstance(value_range, list):
                architecture[key] = random.choice(value_range)
            elif isinstance(value_range, tuple):
                architecture[key] = random.randint(*value_range)
        return architecture

    def forward(self, x):
        for layer in self.layers:
            x = layer(x)
        return x

    def search(self, num_iterations, train_loader, val_loader):
        population = self.initialize_population()
        
        for _ in range(num_iterations):
            scores = [self.evaluate_model(model, train_loader, val_loader) for model in population]
            best_model = population[np.argmax(scores)]
            
            new_population = [best_model]
            while len(new_population) < len(population):
                parent1, parent2 = self.selection(population, scores)
                child = self.crossover(parent1, parent2)
                child = self.mutate(child)
                new_population.append(child)
            
            population = new_population
        
        return population[np.argmax([self.evaluate_model(model, train_loader, val_loader) for model in population])]


    def apply_to_model(self, model, architecture):
        for name, module in model.named_modules():
            if isinstance(module, nn.Conv2d):
                if 'num_filters' in architecture:
                    in_channels = module.in_channels
                    out_channels = architecture['num_filters']
                    kernel_size = architecture.get('kernel_size', module.kernel_size)
                    new_conv = nn.Conv2d(in_channels, out_channels, kernel_size, padding=kernel_size[0]//2)
                    setattr(model, name, new_conv)
            elif isinstance(module, nn.ReLU):
                if architecture.get('activation') == 'leaky_relu':
                    setattr(model, name, nn.LeakyReLU())
        
        if 'num_layers' in architecture:
            current_layers = len(list(model.children()))
            if architecture['num_layers'] > current_layers:
                for _ in range(architecture['num_layers'] - current_layers):
                    model.add_module(f'extra_layer_{_}', nn.Linear(64, 64))  # Adjust as needed
            elif architecture['num_layers'] < current_layers:
                for _ in range(current_layers - architecture['num_layers']):
                    model = nn.Sequential(*list(model.children())[:-1])
        
        return model

# Example usage
base_model = nn.Sequential(
    nn.Conv2d(3, 64, 3, padding=1),
    nn.ReLU(),
    nn.Conv2d(64, 64, 3, padding=1),
    nn.ReLU(),
    nn.MaxPool2d(2)
)

search_space = {
    'num_layers': (2, 5),
    'num_filters': (32, 128),
    'kernel_size': (3, 5),
    'activation': ['relu', 'leaky_relu']
}

nas = ProgressiveNAS(base_model, search_space)
# Ensure to provide train_loader and val_loader when calling search
# best_model = nas.search(num_iterations=10, train_loader=train_loader, val_loader=val_loader)

#andhere's  the train script which is train_mutlilabel.py

import sys
import traceback
import warnings
from typing import Dict, Any, OrderedDict, Tuple, Optional, List
import catalyst
import os
import numpy as np
import pandas as pd
import torch
import hydra
from torch.optim import Optimizer
from torch.utils.data import Dataset, Sampler, DataLoader
from torch import nn
from omegaconf import DictConfig
from functools import partial
import albumentations as A
from catalyst.callbacks import ControlFlowCallback, CriterionCallback, SchedulerCallback, TimerCallback
from catalyst.core import Callback
from pytorch_toolbelt.utils.catalyst import (
    ShowPolarBatchesCallback,
    BestMetricCheckpointCallback,
    HyperParametersCallback,
    MixupCriterionCallback,
    MixupInputCallback,
)
from pytorch_toolbelt.optimization.functional import freeze_model, get_lr_decay_parameters, get_optimizable_parameters
from pytorch_toolbelt.utils import fs, transfer_weights, count_parameters
from pytorch_toolbelt.utils.distributed import is_main_process
from xview3.centernet.bboxer import MultilabelCircleNetCoder
from xview3.early_stopping import EarlyStoppingCallback
from xview3.nas import ProgressiveNAS
from xview3.centernet.models.heads.yolo_head import YOLOHead
from xview3.centernet.models.transformer_classifier import ConvolutionalClassifier
from xview3.centernet.losses.circlenet_loss import CircleNetLoss, YOLOLoss
from xview3.centernet import *
from xview3.pipeline import *
from xview3.factory import *
from xview3.augmentations import *
from xview3.samplers import *
from xview3.dataset import *
from xview3.ema import *
from xview3.evaluation import *
from xview3.inference import *
from xview3.metric import *
from xview3.constants import INPUT_SCENE_ID_KEY
import logging

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

def warn_with_traceback(message, category, filename, lineno, file=None, line=None):
    log = file if hasattr(file, "write") else sys.stderr
    traceback.print_stack(file=log)
    log.write(warnings.formatwarning(message, category, filename, lineno))

warnings.showwarning = warn_with_traceback

if 'XVIEW3_DIR' not in os.environ:
    os.environ['XVIEW3_DIR'] = '/Volumes/Crucial/xview3_dataset'

class CenternetPipeline(Pipeline):
    def __init__(self, cfg):
        super().__init__(cfg)
        self.box_coder: Optional[MultilabelCircleNetCoder] = None
        self.shore_root = os.path.join(os.environ.get('XVIEW3_DIR', ''), "full_dataset", "shoreline")

    def build_datasets(self, config) -> Tuple[Dataset, Dataset, Optional[Sampler], List[Callback]]:
        dataset = MultilabelCircleNetDataModule(data_dir=os.path.join(os.environ.get('XVIEW3_DIR', '/Volumes/Crucial/xview3_dataset'), 'full_dataset'))
        self.box_coder = MultilabelCircleNetCoder(
            image_size=(config.train.train_image_size[0], config.train.train_image_size[1]),
            output_stride=4,  # Adjust this value as needed
            max_objects=512,  # Adjust this value as needed
            heatmap_encoding="umich",
            labels_encoding="circle",
            ignore_value=-1
        )
        print(f"[DEBUG] self.box_coder in build_datasets: {self.box_coder}")
        if self.box_coder is None:
            raise ValueError("self.box_coder is None in build_datasets.")

        train_df, valid_df, _, shore_root = dataset.train_val_split(
            splitter=config.dataset.splitter,
            fold=config.dataset.fold,
            num_folds=config.dataset.num_folds,
        )

        self.valid_df = valid_df.copy()  # Keep a copy of unmodified groundtruth

        self.master_print("Dataset", config.dataset.slug)
        self.master_print("\tignore_low_confidence_detections", config.dataset.ignore_low_confidence_detections)
        self.master_print("\tignore_low_confidence_labels    ", config.dataset.ignore_low_confidence_labels)
        self.master_print("\tfilter_low_confidence_objects   ", config.dataset.filter_low_confidence_objects)

        self.master_print("Train dataset")
        self.master_print("\tUnknown vessels", np.isnan(train_df.is_vessel.values.astype(float)).sum())
        self.master_print("\tUnknown fishing", np.isnan(train_df.is_fishing.values.astype(float)).sum())
        self.master_print("Valid dataset")
        self.master_print("\tUnknown vessels", np.isnan(valid_df.is_vessel.values.astype(float)).sum())
        self.master_print("\tUnknown fishing", np.isnan(valid_df.is_fishing.values.astype(float)).sum())

        if config.dataset.filter_low_confidence_objects and config.dataset.ignore_low_confidence_labels:
            raise ValueError(f"filter_low_confidence_objects and ignore_low_confidence_labels are mutually exclusive")

        if config.dataset.filter_low_confidence_objects:
            train_df = filter_low_confidence_objects(train_df)
            valid_df = filter_low_confidence_objects(valid_df)
            self.master_print("Excluding low-confidence objects from training & validation")
        elif config.dataset.ignore_low_confidence_labels:
            train_df = ignore_low_confidence_objects(train_df)
            valid_df = ignore_low_confidence_objects(valid_df)
            self.master_print("Ignoring low-confidence labels from training & validation")

        self.shore_root = os.path.join(os.environ.get('XVIEW3_DIR', ''), "full_dataset", "shoreline")

        normalization = build_normalization(config.normalization)

        augmentations = [
            A.HorizontalFlip(p=0.5),
            A.VerticalFlip(p=0.5),
            A.RandomRotate90(p=0.5),
            A.Resize(height=config.train.train_image_size[0], width=config.train.train_image_size[1], always_apply=True),
        ]

        transform = A.Compose(
            augmentations,
            keypoint_params=A.KeypointParams(
                format='xys',
                label_fields=['confidences', 'is_vessel', 'is_fishing', 'is_near_shore'],
                remove_invisible=False
            )
        )


        individual_augmentations = dict(
            [(key, A.Compose(build_augmentations([value]))) for key, value in config.augs.individual.items() if value is not None]
        )

        for channel in config.dataset.channels:
            if channel not in normalization:
                normalization[channel] = A.NoOp()

        if config.sampler.sampler_type == "each_object":
            train_ds, train_sampler = dataset.get_random_crop_each_object_dataset(
                train_df,
                train_image_size=config.train.train_image_size,
                input_channels=config.dataset.channels,
                box_coder=self.box_coder,  # Pass `self.box_coder` here
                num_samples=config.sampler.num_samples,
                individual_augmentations=individual_augmentations,
                augmentations=augmentations,
                transform=transform,
                normalization=normalization,
                balance_crowd=config.sampler.balance_crowd,
                balance_near_shore=config.sampler.balance_near_shore,
                balance_vessel_type=config.sampler.balance_vessel_type,
                balance_fishing_type=config.sampler.balance_fishing_type,
                balance_per_scene=config.sampler.balance_per_scene,
                balance_location=config.sampler.balance_location,
                channels_last=config.torch.channels_last,
            )
        else:
            train_ds, train_sampler = dataset.get_random_crop_training_dataset(
                train_df,
                train_image_size=config.train.train_image_size,
                input_channels=config.dataset.channels,
                box_coder=self.box_coder, 
                num_samples=config.sampler.num_samples,
                crop_around_ship_p=config.sampler.crop_around_ship_p,
                individual_augmentations=individual_augmentations,
                augmentations=augmentations,
                normalization=normalization,
                balance_near_shore=config.sampler.balance_near_shore,
                balance_crowd=config.sampler.balance_crowd,
                balance_location=config.sampler.balance_location,
                balance_folder=config.sampler.balance_folder,
                channels_last=config.torch.channels_last,
                shore_root=shore_root,
            )

        valid_ds = dataset.get_validation_dataset(
            valid_df,
            valid_crop_size=config.train.valid_image_size,
            input_channels=config.dataset.channels,
            box_coder=self.box_coder,
            normalization=normalization,
            channels_last=config.torch.channels_last,
        )

        self.master_print(
            "Train",
            "instances",
            len(train_df),
            "dataset",
            len(train_ds),
            "sampler",
            len(train_sampler) if train_sampler else "N/A",
        )
        self.master_print("Valid", "instances", len(valid_df), "dataset", len(valid_ds))

        return train_ds, valid_ds, train_sampler, []

    def build_model(self, config: Dict, loaders: Dict) -> nn.Module:
        model = super().build_model(config)
        logger.debug(f"Train loader sample: {next(iter(loaders['train']))}")
        logger.debug(f"Valid loader sample: {next(iter(loaders['valid']))}")
        nas = ProgressiveNAS(model, config.nas.search_space)
        model = nas.search(config.nas.num_iterations, loaders['train'], loaders['valid'])

        model.yolo_head = YOLOHead(num_classes=config.model.num_classes, anchors=config.model.anchors)
        model.classifier = ConvolutionalClassifier(num_classes=config.model.num_classes)
        return model

    def build_criterions(self, config: Dict) -> Tuple[Dict[str, nn.Module], List[Callback]]:
        criterions, callbacks = super().build_criterions(config)
        
        criterions['yolo_loss'] = YOLOLoss(config.model.num_classes, config.model.anchors)
        criterions['transformer_loss'] = nn.CrossEntropyLoss()
        
        return criterions, callbacks

    def build_metrics(self, config, loaders, model) -> List[Callback]:
        callbacks = super().build_metrics(config, loaders, model)
        
        # Add YOLO and transformer metrics
        callbacks.append(YOLOMetricsCallback(prefix="yolo_metrics"))
        callbacks.append(TransformerMetricsCallback(prefix="transformer_metrics"))
        
        return callbacks
    def build_distributed_params(self, config: DictConfig) -> Dict:
        fp16 = config.optimizer.get("fp16", False)

        if self.distributed:
            local_rank = torch.distributed.get_rank()
            distributed_params = {"rank": local_rank, "syncbn": True}
            if fp16:
                distributed_params["amp"] = True
            if self.cfg.find_unused:
                distributed_params["find_unused_parameters"] = True
        else:
            if fp16:
                distributed_params = {}
                distributed_params["amp"] = True
            else:
                distributed_params = False

        return distributed_params

    def build_optimizer(self, model: nn.Module, loaders: OrderedDict) -> Tuple[Optimizer, Any, List[Callback]]:
        optimizer_config = self.cfg["optimizer"]
        optimizer_use_fp16 = optimizer_config.get("fp16", False)

        optimizer_name = str(optimizer_config["name"])

        wd_on_bias = bool(optimizer_config.get("wd_on_bias", True))
        accumulation_steps = int(optimizer_config.get("accumulation", 1))
        optimizer_params = optimizer_config["params"]
        optimizer_params = scale_learning_rate_for_ddp(optimizer_params)

        optimizer = self.get_optimizer(
            model=model,
            optimizer_name=optimizer_name,
            optimizer_params=optimizer_params,
            apply_weight_decay_to_bias=wd_on_bias,
            layerwise_params=optimizer_config.get("layerwise_params", None),
        )

        if optimizer_use_fp16:
            opt_callback = AMPOptimizerCallback(accumulation_steps=accumulation_steps)
        else:
            opt_callback = OptimizerCallback(accumulation_steps=accumulation_steps, decouple_weight_decay=False)

        scheduler_params = self.cfg["scheduler"]
        num_epochs = self.cfg["train"]["epochs"]

        scheduler_name = str(scheduler_params["scheduler_name"])
        scheduler = get_scheduler(
            optimizer,
            learning_rate=optimizer_params["lr"],
            num_epochs=num_epochs,
            batches_in_epoch=len(loaders["train"]),
            **scheduler_params,
        )

        callbacks = [opt_callback]
        if isinstance(scheduler, (CyclicLR, OneCycleLRWithWarmup)):
            callbacks += [SchedulerCallback(mode="batch")]
        else:
            callbacks += [SchedulerCallback(mode="epoch")]

        self.master_print("Optimizer        :", optimizer_name)
        self.master_print("Model            :", self.cfg.model.config.slug)
        self.master_print(
            "  Parameters     :",
            count_parameters(
                model, ["backbone", "rpn", "roi_heads", "encoder", "decoder", "head", "fuse", "extra_stages", "center", "mask"]
            ),
        )
        self.master_print("  FP16           :", optimizer_use_fp16)
        self.master_print("  Learning rate  :", optimizer_params["lr"])
        self.master_print("  Weight decay   :", optimizer_params.get("weight_decay", 0))
        self.master_print("  WD on bias     :", wd_on_bias)
        self.master_print("  Accumulation   :", accumulation_steps)
        self.master_print("Params           :")
        for k, v in optimizer_params.items():
            self.master_print(f"  {k}:", v)
        self.master_print("Scheduler        :", scheduler_name)

        return optimizer, scheduler, callbacks

    def get_optimizer(
        self,
        model: nn.Module,
        optimizer_name: str,
        optimizer_params: Dict[str, Any],
        apply_weight_decay_to_bias: bool = True,
        layerwise_params=None,
    ) -> Optimizer:
        """
        Construct an Optimizer for given model
        Args:
            model: Model to optimize. Only parameters that require_grad will be used
            optimizer_name: Name of the optimizer (case-insensitive). Supports native pytorch optimizers, apex and
                optimizers from pytorch-optimizers package.
            optimizer_params: Dict of optimizer params (lr, weight_decay, eps, etc)
            apply_weight_decay_to_bias: Whether to apply weight decay on bias parameters. Default is True
        Returns:
            Optimizer instance
        """

        # Optimizer parameter groups
        if layerwise_params is not None:
            if not apply_weight_decay_to_bias:
                raise ValueError("Layerwise params and no wd on bias are mutually exclusive")

            parameters = get_lr_decay_parameters(model, optimizer_params["lr"], layerwise_params)
        else:
            if apply_weight_decay_to_bias:
                parameters = get_optimizable_parameters(model)
            else:
                default_pg, biases_pg = [], []

                for k, v in model.named_parameters():
                    if v.requires_grad:
                        if str.endswith(k, ".bias"):
                            biases_pg.append(v)  # biases
                        else:
                            default_pg.append(v)  # all else

                if apply_weight_decay_to_bias:
                    parameters = default_pg + biases_pg
                else:
                    parameters = default_pg

        optimizer_cls = get_optimizer_cls(optimizer_name)
        optimizer: Optimizer = optimizer_cls(
            parameters,
            **optimizer_params,
        )

        if not apply_weight_decay_to_bias:
            optimizer.add_param_group({"params": biases_pg, "weight_decay": 0.0})

        return optimizer

    def get_experiment_dir(self, config) -> str:
        if self.cfg.experiment:
            return self.cfg.experiment

        current_time = datetime.now().strftime("%y%m%d_%H_%M")
        experiment_slug = f"{current_time}_{config.model.config.slug}_{config.dataset.slug}_{config.loss.slug}_{config.augs.slug}"

        fold = config["dataset"].get("fold", None)
        if fold is not None:
            experiment_slug += f"_fold{fold}"

        log_dir = os.path.join("runs", experiment_slug)
        return log_dir
    
    def get_model(self, model_config: DictConfig) -> nn.Module:
        model: nn.Module = get_detection_model(model_config)
        print(f"[DEBUG] model.box_coder after initialization: {model.box_coder}")
        if model.box_coder is None:
            raise ValueError("model.box_coder is None after model initialization.")
        self.box_coder = model.box_coder
        self.master_print("BoxCoder")
        self.master_print(self.box_coder)
        return model

    def build_loaders(self, config):
        train_ds, valid_ds, train_sampler, dataset_callbacks = self.build_datasets(config)
        
        train_loader = DataLoader(
            train_ds,
            batch_size=config.get('batch_size', 8),
            num_workers=config.get('num_workers', 4),
            sampler=train_sampler,
            pin_memory=True,
            collate_fn=train_ds.get_collate_fn(),
        )
        
        valid_loader = DataLoader(
            valid_ds,
            batch_size=config.get('batch_size', 4),
            num_workers=config.get('num_workers', 4),
            pin_memory=True,
            collate_fn=valid_ds.get_collate_fn(),
        )
        
        return {"train": train_loader, "valid": valid_loader}, dataset_callbacks



    def train(self):
        config = self.cfg
        self.loaders, dataset_callbacks = self.build_loaders(config)
        model = self.build_model(config, self.loaders)
        self.box_coder = model.box_coder  # Set `self.box_coder` after building the model
        # ... rest of your code ...
        #self.loaders, dataset_callbacks = self.build_loaders(config)
        #model = self.build_model(config, self.loaders)
        optimizer, scheduler, optimizer_callbacks = self.build_optimizer(model, self.loaders)
        criterions, criterions_callbacks = self.build_criterions(config)
        metric_callbacks = self.build_metrics(config, self.loaders, model)
        #loaders, dataset_callbacks = self.build_loaders(config)
        nas = ProgressiveNAS(model, config.nas.search_space)
        model = nas.search(config.nas.num_iterations, self.loaders['train'], self.loaders['valid'])
        #optimizer, scheduler, optimizer_callbacks = self.build_optimizer(model, loaders)
        #criterions, criterions_callbacks = self.build_criterions(config)
        #metric_callbacks = self.build_metrics(config, loaders, model)

        experiment_dir = self.get_experiment_dir(config)
        self.experiment_dir = experiment_dir

        if self.is_master:
            os.makedirs(experiment_dir, exist_ok=True)
            dst_config_fname = os.path.join(experiment_dir, "config.yaml")
            with open(dst_config_fname, "w") as f:
                OmegaConf.save(self.cfg, f)

        # model training
        runner_config = config.get("runner", {})
        input_key = runner_config.get("input_key", INPUT_SCENE_ID_KEY)
        output_key = runner_config.get("output_key", None)

        if input_key is not None and not isinstance(input_key, str):
            input_key = OmegaConf.to_container(input_key, resolve=True)
        if output_key is not None and not isinstance(output_key, str):
            output_key = OmegaConf.to_container(output_key, resolve=True)

        device = 'mps' if torch.backends.mps.is_available() else 'cuda' if torch.cuda.is_available() else 'cpu'
        runner = SupervisedRunner(input_key=input_key, output_key=output_key, device=device)

        extra_callbacks = [TimerCallback()]
        # try:
        runner.train(
            fp16=self.build_distributed_params(config),
            model=model,
            criterion=criterions,
            optimizer=optimizer,
            scheduler=scheduler,
            callbacks=dataset_callbacks + metric_callbacks + optimizer_callbacks + criterions_callbacks + extra_callbacks,
            loaders=loaders,
            logdir=experiment_dir,
            num_epochs=config.train.epochs,
            verbose=True,
            main_metric=config.runner.main_metric,
            minimize_metric=config.runner.main_metric_minimize,
            checkpoint_data=self.get_checkpoint_data(),
        )
        # except Exception as e:
        #     with open(os.path.join(experiment_dir, "exception.log"), "w") as logf:
        #         logf.write(str(e))
        #         traceback.print_exc(file=logf)
        #         print(e)

        self.on_experiment_finished()

    def get_checkpoint_data(self):
        return {
            "config": OmegaConf.to_container(self.cfg, resolve=True),
            "version": get_version(),
        }

    def on_experiment_finished(self):
        pass

    def build_quircks(self, config: DictConfig) -> List[Callback]:
        extra_callbacks = []

        if config.ema.callback is not None:
            cb = hydra.utils.instantiate(config.ema.callback)
            extra_callbacks.append(cb)
            self.master_print("Using EMA Callback", cb)

        return extra_callbacks


class TrainOnlyCriterionCallback(CriterionCallback):
    def on_batch_end(self, runner) -> None:
        if runner.is_train_loader:
            return super(TrainOnlyCriterionCallback, self).on_batch_end(runner)
        else:
            runner.batch_metrics[self.prefix] = torch.tensor(0, device="mps")



@hydra.main(config_path="configs", config_name="multilabel_circle_net", version_base=None)
def main(cfg: DictConfig) -> None:
    torch.mps.empty_cache()
    torch.manual_seed(int(cfg.seed))

    torch.set_anomaly_enabled(cfg.torch.detect_anomaly)

    torch.backends.cudnn.deterministic = cfg.torch.deterministic
    torch.backends.cudnn.benchmark = cfg.torch.benchmark

    # The flag below controls whether to allow TF32 on matmul. This flag defaults to True.
    torch.backends.cuda.matmul.allow_tf32 = cfg.torch.cuda_allow_tf32
    # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True.
    torch.backends.cudnn.allow_tf32 = cfg.torch.cudnn.allow_tf32

    if cfg.dataset.data_dir is None:
        raise ValueError("--data-dir must be set")

    CenternetPipeline(cfg).train()


if __name__ == "__main__":
    main()

#and finally the config file for the model

defaults:
  - _self_
  - dataset: valid_sar_cv
  - augs: flips_light
  - sampler: balanced
  - normalization: default
  - loss: centernet/rfl_bce_mse
  - optimizer: adamw
  - scheduler: cos
  - train: default
  - runner: default
  - ema: disabled
  - torch: default

batch_size: 8
channels_last: False

local_rank: ${oc.env:LOCAL_RANK, 0}
world_size: ${oc.env:WORLD_SIZE, 0}

experiment: null
seed: 42
deterministic: False
detect_anomaly: False
find_unused: False

transfer: null
checkpoint: null

nas:
  search_space:
    num_layers: [2, 5]
    num_filters: [32, 128]
    kernel_size: [3, 5]
    activation: ['relu', 'leaky_relu']
  num_iterations: 10

model:
  _target_: xview3.centernet.models.centernet_unet.CenterNetUNetModel.from_config
  num_channels: 2
  num_classes: 2
  anchors:
    - [10, 13]
    - [16, 30]
    - [33, 23]
  input_upsample_factor: 1
  encoder:
    _target_: custom_encoder.CustomTimmEncoder
    model_name: 'tf_efficientnet_b5_ns'
    num_channels: 2
    features_only: True
    pretrained: False
  decoder:
    channels: [512, 384, 256, 128]
    block_type: UnetBlock
    upsample_type: bilinear
  head:
    _target_: xview3.centernet.models.heads.decoupled_head.DecoupledHeadGroupNormLateShuffle
    channels: 32
    classifier_dim: 64
    dropout_rate: 0.1
  box_coder:
    _target_: xview3.centernet.bboxer.MultilabelCircleNetCoder
    image_size: [768, 768]
    output_stride: 4
    max_objects: 512
    heatmap_encoding: umich
    labels_encoding: circle
    ignore_value: -1
  activation: silu
  num_extra_blocks: 0

yolo:
  enabled: true
  loss_weight: 1.0

transformer:
  enabled: true
  loss_weight: 1.0

train:
  epochs: 100
  early_stopping: 25
  train_image_size: [768, 768]
  valid_image_size: [2048, 2048]

torch:
  cudnn:
    allow_tf32: True
  cuda_allow_tf32: True
  detect_anomaly: False
  deterministic: False
  benchmark: True

hydra:
  output_subdir: null
  run:
    dir: .
  sweep:
    dir: .
    subdir: .

augs:
  spatial:
    - albumentations.HorizontalFlip
    - albumentations.VerticalFlip
  individual:
    vv0:
      _target_: xview3.augmentations.unclipped_random_brightness.UnclippedRandomBrightnessContrast
      always_apply: false
      p: 0.5

normalization:
  channels:
    vv:
      _target_: xview3.SigmoidNormalization
      midpoint: -20.0
      temperature: 0.18
    vh:
      _target_: xview3.SigmoidNormalization
      midpoint: -20.0
      temperature: 0.18
    bathymetry:
      _target_: xview3.CubicRootNormalization
    wind_direction:
      _target_: albumentations.NoOp
    wind_speed:
      _target_: albumentations.NoOp
    mask:
      _target_: albumentations.NoOp

dataset:
  data_dir: ${oc.env:XVIEW3_DIR}/full_dataset
  splitter: valid_sar_cv
  fold: 0
  num_folds: 4


#here's the running error

DEBUG:rasterio.env:Entered env context: <rasterio.env.Env object at 0x340aad760>
DEBUG:rasterio._io:Output nodata value read from file: -32768.0
DEBUG:rasterio._io:Output nodata values: [-32768.0]
DEBUG:rasterio._io:all_valid: False
DEBUG:rasterio._io:mask_flags: ([<MaskFlags.nodata: 8>],)
DEBUG:rasterio._io:Jump straight to _read()
DEBUG:rasterio._io:Window: Window(col_off=17028, row_off=8790, width=512, height=512)
DEBUG:rasterio._io:IO window xoff=17028.0 yoff=8790.0 width=512.0 height=512.0
DEBUG:rasterio.env:Exiting env context: <rasterio.env.Env object at 0x340aad760>
DEBUG:rasterio.env:Cleared existing <rasterio._env.GDALEnv object at 0x340aada30> options
DEBUG:rasterio._env:Stopped GDALEnv <rasterio._env.GDALEnv object at 0x340aada30>.
DEBUG:rasterio.env:Exiting outermost env
DEBUG:rasterio.env:Exited env context: <rasterio.env.Env object at 0x340aad760>
[DEBUG] Channel 'vh' image shape before transformations: (512, 512)
[DEBUG] Channel 'vh' image shape after individual_transforms: (512, 512)
[DEBUG] Channel 'vh' image shape after normalization: (512, 512)
[DEBUG] Channel 'vv' image shape before transformations: (512, 512)
[DEBUG] Channel 'vv' image shape after individual_transforms: (512, 512)
[DEBUG] Channel 'vv' image shape after normalization: (512, 512)
[DEBUG] Number of channels: 2
[DEBUG] Channel 0 shape: (512, 512)
[DEBUG] Channel 1 shape: (512, 512)
[DEBUG] Combined image shape after np.dstack: (512, 512, 2)
[DEBUG] Keypoints being passed to albumentations:
Error executing job with overrides: ['+config=configs/multilabel_circle_net.yaml']
Traceback (most recent call last):
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel.py", line 529, in <module>
    main()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/main.py", line 94, in decorated_main
    _run_hydra(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
    _run_app(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 457, in _run_app
    run_and_report(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 223, in run_and_report
    raise ex
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
    return func()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
    lambda: hydra.run(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/hydra.py", line 132, in run
    _ = ret.return_value
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/core/utils.py", line 260, in return_value
    raise self._return_value
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/core/utils.py", line 186, in run_job
    ret.return_value = task_function(task_cfg)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel.py", line 525, in main
    CenternetPipeline(cfg).train()
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel.py", line 416, in train
    model = self.build_model(config, self.loaders)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel.py", line 207, in build_model
    model = nas.search(config.nas.num_iterations, loaders['train'], loaders['valid'])
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/xview3/nas.py", line 36, in search
    scores = [self.evaluate_model(model, train_loader, val_loader) for model in population]
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/xview3/nas.py", line 36, in <listcomp>
    scores = [self.evaluate_model(model, train_loader, val_loader) for model in population]
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/xview3/nas.py", line 115, in evaluate_model
    outputs = model(inputs)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/xview3/centernet/models/centernet_unet.py", line 193, in forward
    neck_features = self.neck(decoded_features)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/xview3/centernet/models/necks.py", line 47, in forward
    return self.out_conv(fpn_output)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/conv.py", line 458, in forward
    return self._conv_forward(input, self.weight, self.bias)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/conv.py", line 454, in _conv_forward
    return F.conv2d(input, weight, bias, self.stride,
RuntimeError: Given groups=1, weight of size [256, 1024, 1, 1], expected input[8, 1280, 256, 256] to have 1024 channels, but got 1280 channels instead
(sam3) samerkharboush@Samers-MBP SAR-MPS-OptunaNasFbnYoloTransformers %