HYDRA_FULL_ERROR=1 python train_multilabel2.py +config=configs/multilabel_circle_net.yaml
DEBUG:hydra.core.utils:Setting JobRuntime:name=UNKNOWN_NAME
DEBUG:hydra.core.utils:Setting JobRuntime:name=train_multilabel2
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 913, in <module>
    main()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/main.py", line 94, in decorated_main
    _run_hydra(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
    _run_app(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 457, in _run_app
    run_and_report(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
    return func()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
    lambda: hydra.run(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/hydra.py", line 119, in run
    ret = run_job(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/core/utils.py", line 186, in run_job
    ret.return_value = task_function(task_cfg)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 910, in main
    CenternetPipeline(cfg).train()
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 82, in __init__
    self.scaler = GradScaler()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/amp/grad_scaler.py", line 132, in __init__
    warnings.warn(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/warnings.py", line 109, in _showwarnmsg
    sw(msg.message, msg.category, msg.filename, msg.lineno,
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 62, in warn_with_traceback
    traceback.print_stack(file=log)
/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/amp/grad_scaler.py:132: UserWarning:

torch.cuda.amp.GradScaler is enabled, but CUDA is not available.  Disabling.

Dataset valid_sar_cv
        ignore_low_confidence_detections False
        ignore_low_confidence_labels     False
        filter_low_confidence_objects    False
Train dataset
        Unknown vessels 90
        Unknown fishing 11242
Valid dataset
        Unknown vessels 20
        Unknown fishing 2778
Preparing training dataset: 100%|█████████████████████████████| 35/35 [00:00<00:00, 677.65it/s]
Preparing validation dataset: 100%|████████████████████████████| 7/7 [00:00<00:00, 2101.96it/s]
Train instances 13933 dataset 35 sampler 8192
Valid instances 3655 dataset 1039
Error executing job with overrides: ['+config=configs/multilabel_circle_net.yaml']
Traceback (most recent call last):
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 92, in _call_target
    return _target_(*args, **kwargs)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/xview3/centernet/models/centernet_unet.py", line 282, in from_config
    model_name = encoder_config.pop('model_name', 'tf_efficientnet_b5.ns_jft_in1k')
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1729, in __getattr__
    raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
AttributeError: 'CustomTimmEncoder' object has no attribute 'pop'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 913, in <module>
    main()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/main.py", line 94, in decorated_main
    _run_hydra(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
    _run_app(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 457, in _run_app
    run_and_report(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 223, in run_and_report
    raise ex
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
    return func()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
    lambda: hydra.run(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/hydra.py", line 132, in run
    _ = ret.return_value
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/core/utils.py", line 260, in return_value
    raise self._return_value
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/core/utils.py", line 186, in run_job
    ret.return_value = task_function(task_cfg)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 910, in main
    CenternetPipeline(cfg).train()
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 769, in train
    model = self.build_model(config, self.loaders)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 444, in build_model
    model = instantiate(config.model)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 226, in instantiate
    return instantiate_node(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 347, in instantiate_node
    return _call_target(_target_, partial, args, kwargs, full_key)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 97, in _call_target
    raise InstantiationException(msg) from e
hydra.errors.InstantiationException: Error in call to target 'xview3.centernet.models.centernet_unet.CenterNetUNetModel.from_config':
AttributeError("'CustomTimmEncoder' object has no attribute 'pop'")
full_key: model
(sam3) samerkharboush@Samers-MBP SAR-MPS-OptunaNasFbnYoloTransformers % 


here's my encoder
import timm
import torch.nn as nn
import torch
import torch.nn.functional as F
from pytorch_toolbelt.modules.decoders import UNetDecoder

class CustomTimmEncoder(nn.Module):
    def __init__(self, model_name, num_channels=2, features_only=True, pretrained=False):
        super(CustomTimmEncoder, self).__init__()
        self.encoder = timm.create_model(
            model_name,
            features_only=features_only,
            in_chans=num_channels,
            pretrained=pretrained
        )
        # Extract channels and strides
        self.channels = [f['num_chs'] for f in self.encoder.feature_info]
        self.strides = [f['reduction'] for f in self.encoder.feature_info]
        self.feature_info = self.encoder.feature_info
        
    def forward(self, x):
        features = self.encoder(x)
        for idx, feature in enumerate(features):
            print(f"[DEBUG] Feature {idx} shape: {feature.shape}")
        return features
    
class DebugUNetDecoder(nn.Module):
    def __init__(self, encoder_channels, decoder_channels, unet_block, upsample_block, center=False):
        super(DebugUNetDecoder, self).__init__()
        self.center = center
        self.num_stages = len(decoder_channels)
        self.up_samples = nn.ModuleList()
        self.unet_blocks = nn.ModuleList()

        if self.center:
            self.center_block = unet_block(encoder_channels[-1], encoder_channels[-1])
        else:
            self.center_block = nn.Identity()

        for i in range(self.num_stages):
            self.up_samples.append(upsample_block())

            if i == 0:
                in_channels = encoder_channels[-1] + encoder_channels[-2]
            else:
                in_channels = decoder_channels[i - 1] + encoder_channels[-(i + 2)]
            out_channels = decoder_channels[i]

            self.unet_blocks.append(unet_block(in_channels, out_channels))

    def forward(self, features):
        x = features[-1]
        x = self.center_block(x)
        num_features = len(features)
        print(f"[DEBUG] Number of encoder features: {num_features}")

        decoded_features = []
        for i in range(self.num_stages):
            print(f"[DEBUG] Decoder stage {i}")
            x = self.up_samples[i](x)

            encoder_index = num_features - i - 2
            if encoder_index >= 0:
                encoder_feature = features[encoder_index]
                print(f"Using encoder feature at index {encoder_index}")

                if x.size()[2:] != encoder_feature.size()[2:]:
                    encoder_feature = F.interpolate(encoder_feature, size=x.size()[2:], mode='nearest')
                    print(f"encoder_feature.shape after resizing at stage {i}: {encoder_feature.shape}")

                x = torch.cat([x, encoder_feature], dim=1)
                print(f"x.shape after concat at stage {i}: {x.shape}")
            else:
                print(f"No encoder feature for decoder stage {i}, skipping concatenation")

            x = self.unet_blocks[i](x)
            decoded_features.append(x)

        return decoded_features