(sam3) samerkharboush@Samers-MBP SAR-MPS-OptunaNasFbnYoloTransformers % HYDRA_FULL_ERROR=1 python train_multilabel2.py +config=configs/multilabel_circle_net.yaml
DEBUG:hydra.core.utils:Setting JobRuntime:name=UNKNOWN_NAME
DEBUG:hydra.core.utils:Setting JobRuntime:name=train_multilabel2
Dataset valid_sar_cv
        ignore_low_confidence_detections False
        ignore_low_confidence_labels     False
        filter_low_confidence_objects    False
Train dataset
        Unknown vessels 90
        Unknown fishing 11242
Valid dataset
        Unknown vessels 20
        Unknown fishing 2778
Preparing training dataset: 100%|███████████████| 35/35 [00:00<00:00, 758.15it/s]
Error executing job with overrides: ['+config=configs/multilabel_circle_net.yaml']
Traceback (most recent call last):
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 829, in <module>
    main()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/main.py", line 94, in decorated_main
    _run_hydra(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
    _run_app(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 457, in _run_app
    run_and_report(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 223, in run_and_report
    raise ex
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
    return func()
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
    lambda: hydra.run(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/_internal/hydra.py", line 132, in run
    _ = ret.return_value
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/core/utils.py", line 260, in return_value
    raise self._return_value
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/hydra/core/utils.py", line 186, in run_job
    ret.return_value = task_function(task_cfg)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 826, in main
    CenternetPipeline(cfg).train()
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 706, in train
    self.loaders, dataset_callbacks = self.build_loaders(config)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 431, in build_loaders
    train_ds, valid_ds, train_sampler, dataset_callbacks = self.build_datasets(config)
  File "/Users/samerkharboush/Downloads/SAR-MPS-OptunaNasFbnYoloTransformers/train_multilabel2.py", line 193, in build_datasets
    box_coder=config.box_coder,
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/dictconfig.py", line 359, in __getattr__
    self._format_and_raise(key=key, value=None, cause=e)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/base.py", line 231, in _format_and_raise
    format_and_raise(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/_utils.py", line 819, in format_and_raise
    _raise(ex, cause)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/_utils.py", line 797, in _raise
    raise ex.with_traceback(sys.exc_info()[2])  # set env var OC_CAUSE=1 for full trace
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/dictconfig.py", line 351, in __getattr__
    return self._get_impl(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/dictconfig.py", line 442, in _get_impl
    node = self._get_child(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/basecontainer.py", line 73, in _get_child
    child = self._get_node(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/dictconfig.py", line 475, in _get_node
    self._validate_get(key)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/dictconfig.py", line 164, in _validate_get
    self._format_and_raise(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/base.py", line 231, in _format_and_raise
    format_and_raise(
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/_utils.py", line 899, in format_and_raise
    _raise(ex, cause)
  File "/opt/anaconda3/envs/sam3/lib/python3.9/site-packages/omegaconf/_utils.py", line 797, in _raise
    raise ex.with_traceback(sys.exc_info()[2])  # set env var OC_CAUSE=1 for full trace
omegaconf.errors.ConfigAttributeError: Key 'box_coder' is not in struct
    full_key: box_coder
    object_type=dict
(sam3) samerkharboush@Samers-MBP SAR-MPS-OptunaNasFbnYoloTransformers % 


#config
defaults:
  - _self_
  - dataset: valid_sar_cv
  - augs: flips_light
  - sampler: balanced
  - normalization: default
  - loss: centernet/rfl_bce_mse
  - optimizer: adamw
  - scheduler: cos
  - train: default
  - runner: default
  - ema: disabled
  - torch: default

batch_size: 4
channels_last: False

local_rank: ${oc.env:LOCAL_RANK, 0}
world_size: ${oc.env:WORLD_SIZE, 0}

experiment: null
seed: 42
deterministic: False
detect_anomaly: False
find_unused: False

transfer: null
checkpoint: null

model_slug: center_net_yolo_model
dataset_slug: valid_sar_cv
loss_slug: centernet_rfl_bce_mse
augs_slug: flips_light

model:
  _target_: xview3.centernet.models.centernet_unet.CenterNetUNetModel.from_config
  encoder:
    model_name: swin_large_patch4_window7_224
    in_chans: 2
    features_only: true
    pretrained: false
  decoder:
    _target_: xview3.centernet.models.custom_encoder.DebugUNetDecoder
    encoder_channels: [32, 24, 40, 112, 320]  
    decoder_channels: [512, 384, 256, 128, 64]
    unet_block:
      _target_: xview3.centernet.models.unet_blocks.UnetBlock
      _partial_: true
    upsample_block:
      _target_: torch.nn.Upsample
      scale_factor: 2
      mode: bilinear
  nas:
    enabled: true
    num_iterations: 10
    search_space:
      num_layers: [2, 5]
      num_filters: [32, 128]
      kernel_size: [3, 5]
      activation: [relu, leaky_relu]
  num_channels: 2
  num_classes: 2
  anchors: [[10, 13], [16, 30], [33, 23]]
  input_upsample_factor: 1
  neck_out_channels: 256
  heads:
    - _target_: xview3.centernet.models.heads.decoupled_head.DecoupledHeadGroupNormLateShuffle
      channels: 256
      classifier_dim: 64
      dropout_rate: 0.1
    - _target_: xview3.centernet.models.heads.yolo_head.YOLOHead
      num_classes: 2
      anchors: [[10, 13], [16, 30], [33, 23]]
  box_coder:
    _target_: xview3.centernet.bboxer.multilabel_circle_coder.MultilabelCircleNetCoder
    image_size: [768, 768]
    output_stride: 32
    max_objects: 512
    labels_encoding: circle
    heatmap_encoding: umich
    fixed_radius: 2
    ignore_value: -1
    ignore_low_confidence_detections: false
    labels_radius: null
    nms_kernel_size: 3
    nms_method: centernet_heatmap_nms
  activation: silu
  num_extra_blocks: 0

yolo:
  enabled: true
  loss_weight: 1.0

transformer:
  enabled: true
  loss_weight: 1.0

train:
  slug: default_training 
  epochs: 4
  early_stopping: 2
  train_image_size: [768, 768]
  valid_image_size: [2048, 2048]
  main_metric: train_metrics_roc_auc  

torch:
  amp:
    enabled: false  
  cudnn:
    allow_tf32: True
  cuda_allow_tf32: True
  detect_anomaly: False
  deterministic: False
  benchmark: True

hydra:
  output_subdir: null
  run:
    dir: .
  sweep:
    dir: .
    subdir: .

augs:
  spatial:
    - albumentations.HorizontalFlip
    - albumentations.VerticalFlip
  individual:
    vv0:
      _target_: xview3.augmentations.unclipped_random_brightness.UnclippedRandomBrightnessContrast
      always_apply: false
      p: 0.5
  slug: flips_light 

normalization:
  channels:
    vv:
      _target_: xview3.SigmoidNormalization
      midpoint: -20.0
      temperature: 0.18
    vh:
      _target_: xview3.SigmoidNormalization
      midpoint: -20.0
      temperature: 0.18
    bathymetry:
      _target_: xview3.CubicRootNormalization
    wind_direction:
      _target_: albumentations.NoOp
    wind_speed:
      _target_: albumentations.NoOp
    mask:
      _target_: albumentations.NoOp

dataset:
  slug: valid_sar_cv  
  data_dir: ${oc.env:XVIEW3_DIR}/full_dataset
  splitter: valid_sar_cv
  fold: 0
  num_folds: 4