Add files using upload-large-folder tool

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/config.txt

@@ -0,0 +1,228 @@
+{'dataset': {'classname_feat_concat': 'only',
+             'classname_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/classname-clip-base/a_photo_of.pt',
+             'default_fps': 30,
+             'downsample_rate': 1,
+             'enable_temporal_jittering': False,
+             'feat_stride': 16.0,
+             'fix_video_frames': 0,
+             'input_txt_dim': 512,
+             'input_vid_dim': 256,
+             'json_file': 'ego4d_data/nlq_v1/ego4d_nlq_val_v1_lemma.jsonl',
+             'lavila_caption_dir': '/root/autodl-tmp/data/ego4d/nlq/lavila/narration',
+             'max_seq_len': 2560,
+             'num_classes': 1,
+             'num_frames': 16.0,
+             'object_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/co-detr/class-score0.6-minnum10-lmdb',
+             'object_feat_type': 'class-score',
+             'text_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/SnAG/nlq_v1_clip_token_features',
+             'train_jsonl_file': 'ego4d_data/nlq_v1/ego4d_nlq_train_v1_lemma_clean.jsonl',
+             'val_jsonl_file': 'ego4d_data/nlq_v1/ego4d_nlq_val_v1_lemma.jsonl',
+             'val_text_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/SnAG/nlq_v1_clip_token_features',
+             'video_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/egovlp_lmdb'},
+ 'dataset_name': 'ego4d_multitask',
+ 'devices': 'cuda:0',
+ 'init_rand_seed': 12345678,
+ 'loader': {'batch_size': 4, 'num_workers': 4},
+ 'model': {'backbone_arch': [2, 4, 4, 0, 6],
+           'backbone_type': 'ObjectMambaTransformer',
+           'embd_dim': 512,
+           'embd_kernel_size': 3,
+           'embd_with_ln': True,
+           'fpn_dim': 512,
+           'fpn_start_level': 0,
+           'fpn_type': 'identity',
+           'fpn_with_ln': True,
+           'generator': {'generator_type': 'point'},
+           'head_dim': 512,
+           'head_kernel_size': 3,
+           'head_num_layers': 3,
+           'head_with_ln': True,
+           'input_txt_dim': 512,
+           'input_vid_dim': 256,
+           'max_buffer_len_factor': 4.0,
+           'max_query': 230,
+           'max_seq_len': 2560,
+           'max_shot_num': 1800,
+           'multiscale_encoder_cfg': [{'layer_cfg': {'mha_win_size': 9,
+                                                     'n_ds_strides': [2, 2],
+                                                     'n_embd': 256,
+                                                     'n_head': 4,
+                                                     'path_pdrop': 0.1},
+                                       'layer_num': 6,
+                                       'layer_type': 'TransformerBlock'},
+                                      {'layer_cfg': {'in_channels': [256,
+                                                                     256,
+                                                                     256,
+                                                                     256,
+                                                                     256,
+                                                                     256,
+                                                                     256],
+                                                     'out_channel': 256},
+                                       'layer_num': 1,
+                                       'layer_type': 'FPNLayernorm'}],
+           'n_head': 4,
+           'n_mha_win_size': 9,
+           'nlq_heads_cfg': {'center_sample_radius': 1.5,
+                             'cls_head_cfg': {'empty_cls': [],
+                                              'feat_dim': 256,
+                                              'input_dim': 256,
+                                              'kernel_size': 3,
+                                              'num_classes': 1,
+                                              'num_layers': 3,
+                                              'prior_prob': 0.01,
+                                              'with_ln': True},
+                             'duration_thresh': 0.001,
+                             'iou_threshold': 0.1,
+                             'loss_normalizer': 200,
+                             'loss_normalizer_momentum': 0.9,
+                             'max_seg_num': 5,
+                             'min_score': 0.001,
+                             'pre_nms_thresh': 0.001,
+                             'pre_nms_topk': 2000,
+                             'reg_head_cfg': {'feat_dim': 256,
+                                              'fpn_levels': 7,
+                                              'input_dim': 256,
+                                              'kernel_size': 3,
+                                              'num_layers': 3,
+                                              'with_ln': True},
+                             'reg_loss_weight': 1.0,
+                             'train_label_smoothing': 0.1},
+           'num_classes': 1,
+           'obj_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                              'n_in': 512,
+                                              'n_out': 256,
+                                              'num_layer': 2},
+                                'layer_num': 1,
+                                'layer_type': 'MaskedConv1DLayer'},
+                               {'layer_cfg': {'n_embd': 256, 'path_pdrop': 0.1},
+                                'layer_num': 4,
+                                'layer_type': 'ObjectEncoderBlock'}],
+           'object_dim': 512,
+           'object_use_cross_model': True,
+           'object_win_size': 1,
+           'regression_range': [[0, 4],
+                                [2, 8],
+                                [4, 16],
+                                [8, 32],
+                                [16, 64],
+                                [32, 128],
+                                [64, 10000]],
+           'scale_factor': 2,
+           'tasks': ['NLQ', 'VTM'],
+           'test_cfg': {'duration_thresh': 0.001,
+                        'ext_score_file': None,
+                        'iou_threshold': 0.1,
+                        'max_seg_num': 5,
+                        'min_score': 0.001,
+                        'multiclass_nms': True,
+                        'nms_method': 'soft',
+                        'nms_sigma': 0.75,
+                        'pre_nms_thresh': 0.001,
+                        'pre_nms_topk': 2000,
+                        'test_num': 1,
+                        'test_start_epoch': 2,
+                        'voting_thresh': 0.9},
+           'text_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                               'n_in': 512,
+                                               'n_out': 256,
+                                               'num_layer': 2},
+                                 'layer_num': 1,
+                                 'layer_type': 'MaskedConv1DLayer'},
+                                {'layer_cfg': {'n_embd': 256,
+                                               'n_head': 4,
+                                               'path_pdrop': 0.1},
+                                 'layer_num': 4,
+                                 'layer_type': 'TransformerBlock'}],
+           'train_cfg': {'box_loss_weight': 5.0,
+                         'center_sample': 'radius',
+                         'center_sample_radius': 1.5,
+                         'clip_grad_l2norm': 1.0,
+                         'cls_prior_prob': 0.01,
+                         'dropout': 0.0,
+                         'droppath': 0.1,
+                         'head_empty_cls': [],
+                         'init_loss_norm': 200,
+                         'iou_loss_weight': 1.0,
+                         'label_smoothing': 0.1,
+                         'loss_weight': 1.0,
+                         'mamba_arch': ['bimamba1', 'mlp', 'obj'],
+                         'num_decoder_layer': 6},
+           'use_abs_pe': True,
+           'use_lmha_in_fpn': True,
+           'use_rel_pe': False,
+           'video_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                                'kernel_size': 3,
+                                                'n_hidden': 256,
+                                                'n_in': 256,
+                                                'n_out': 256,
+                                                'num_layer': 2},
+                                  'layer_num': 1,
+                                  'layer_type': 'MaskedConv1DLayer'},
+                                 {'layer_cfg': {'mamba_arch': ['bimamba1',
+                                                               'mlp',
+                                                               'obj'],
+                                                'n_embd': 256,
+                                                'n_head': 4,
+                                                'path_pdrop': 0.1},
+                                  'layer_num': 4,
+                                  'layer_type': 'ObjectMambaBlock'}],
+           'vtm_heads_cfg': {'loss_weight': 1.5,
+                             'multiscale': False,
+                             'shot_aggregator_cfg': {'layer_cfg': {'cross_mixer_cfg': {'block_cfg': {'n_embd': 256,
+                                                                                                     'n_head': 4},
+                                                                                       'block_type': 'MaskedMHA'},
+                                                                   'num_layer': 1,
+                                                                   'path_pdrop': 0.1,
+                                                                   'query_num': 5,
+                                                                   'self_mixer_cfg': {'block_cfg': {'n_embd': 256,
+                                                                                                    'n_head': 4},
+                                                                                      'block_type': 'MaskedMHCA'}},
+                                                     'layer_type': 'QFormerLayer'},
+                             'similarity_head_cfg': {'layer_cfg': {'con_dim': 256,
+                                                                   'x_dim': 256,
+                                                                   'y_dim': 256},
+                                                     'layer_type': 'Cosine'},
+                             'soft_label': False}},
+ 'model_name': 'MultiTaskArch',
+ 'opt': {'backbone_lr_weight': 1,
+         'epochs': 6,
+         'learning_rate': 0.0004,
+         'momentum': 0.9,
+         'schedule_gamma': 0.1,
+         'schedule_steps': [],
+         'schedule_type': 'cosine',
+         'type': 'AdamW',
+         'warmup': True,
+         'warmup_epochs': 4,
+         'weight_decay': 0.05},
+ 'output_folder': '/root/autodl-tmp/model/GroundNLQ/ckpt/',
+ 'test_cfg': {'duration_thresh': 0.001,
+              'ext_score_file': None,
+              'iou_threshold': 0.1,
+              'max_seg_num': 5,
+              'min_score': 0.001,
+              'multiclass_nms': True,
+              'nms_method': 'soft',
+              'nms_sigma': 0.75,
+              'pre_nms_thresh': 0.001,
+              'pre_nms_topk': 2000,
+              'test_num': 1,
+              'test_start_epoch': 2,
+              'voting_thresh': 0.9},
+ 'track': 'goal_step',
+ 'train_cfg': {'box_loss_weight': 5.0,
+               'center_sample': 'radius',
+               'center_sample_radius': 1.5,
+               'clip_grad_l2norm': 1.0,
+               'cls_prior_prob': 0.01,
+               'dropout': 0.0,
+               'droppath': 0.1,
+               'head_empty_cls': [],
+               'init_loss_norm': 200,
+               'iou_loss_weight': 1.0,
+               'label_smoothing': 0.1,
+               'loss_weight': 1.0,
+               'mamba_arch': ['bimamba1', 'mlp', 'obj'],
+               'num_decoder_layer': 6},
+ 'train_split': ['training'],
+ 'val_split': ['validation']}

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/eval_results.txt

@@ -0,0 +1,110 @@
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|   8.00   |   4.31   |  23.26   |  12.13   |  23.26   |  12.13   |
++----------+----------+----------+----------+----------+----------+avgiou=8.002065
+epoch0
+	final_loss 0.67 (1.66)
+	cls_loss 0.24 (0.91)
+	reg_loss 0.17 (0.39)
+	vtm_loss 0.18 (0.24)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  20.03   |  13.29   |  41.35   |  29.04   |  41.35   |  29.04   |
++----------+----------+----------+----------+----------+----------+avgiou=20.030976
+epoch1
+	final_loss 0.71 (1.61)
+	cls_loss 0.29 (0.90)
+	reg_loss 0.15 (0.37)
+	vtm_loss 0.18 (0.23)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  21.35   |  14.61   |  43.78   |  32.11   |  43.78   |  32.11   |
++----------+----------+----------+----------+----------+----------+avgiou=21.347445
+epoch2
+	final_loss 0.63 (1.59)
+	cls_loss 0.22 (0.89)
+	reg_loss 0.15 (0.36)
+	vtm_loss 0.18 (0.23)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  21.97   |  15.20   |  44.61   |  32.96   |  44.61   |  32.96   |
++----------+----------+----------+----------+----------+----------+avgiou=21.966959
+epoch3
+	final_loss 0.54 (1.60)
+	cls_loss 0.16 (0.90)
+	reg_loss 0.11 (0.36)
+	vtm_loss 0.18 (0.23)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  21.55   |  14.79   |  42.82   |  31.62   |  42.82   |  31.62   |
++----------+----------+----------+----------+----------+----------+avgiou=21.553949
+epoch4
+	final_loss 0.68 (1.64)
+	cls_loss 0.26 (0.93)
+	reg_loss 0.16 (0.36)
+	vtm_loss 0.17 (0.23)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  20.55   |  14.09   |  41.40   |  30.23   |  41.40   |  30.23   |
++----------+----------+----------+----------+----------+----------+avgiou=20.547238
+epoch5
+	final_loss 0.61 (1.69)
+	cls_loss 0.20 (0.98)
+	reg_loss 0.13 (0.36)
+	vtm_loss 0.19 (0.23)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  19.90   |  13.42   |  39.03   |  28.32   |  39.03   |  28.32   |
++----------+----------+----------+----------+----------+----------+avgiou=19.901910
+epoch6
+	final_loss 0.72 (1.78)
+	cls_loss 0.23 (1.04)
+	reg_loss 0.13 (0.37)
+	vtm_loss 0.24 (0.25)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  18.61   |  12.39   |  36.91   |  27.00   |  36.91   |  27.00   |
++----------+----------+----------+----------+----------+----------+avgiou=18.611255
+epoch7
+	final_loss 0.64 (1.85)
+	cls_loss 0.20 (1.08)
+	reg_loss 0.10 (0.38)
+	vtm_loss 0.23 (0.26)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  17.73   |  11.51   |  35.26   |  25.27   |  35.26   |  25.27   |
++----------+----------+----------+----------+----------+----------+avgiou=17.733609
+epoch8
+	final_loss 0.74 (1.92)
+	cls_loss 0.25 (1.12)
+	reg_loss 0.13 (0.38)
+	vtm_loss 0.23 (0.28)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  17.42   |  11.51   |  33.66   |  24.26   |  33.66   |  24.26   |
++----------+----------+----------+----------+----------+----------+avgiou=17.423851
+epoch9
+	final_loss 0.70 (1.95)
+	cls_loss 0.25 (1.14)
+	reg_loss 0.11 (0.38)
+	vtm_loss 0.23 (0.29)

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/logs/EVAL_GPU_0_all_losses_cls_loss/events.out.tfevents.1729874043.autodl-container-b3ec4da47b-bc5fbea1.663163.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65f92dbc196d204a5bbf192b730190cda2c485c7914f7e4e2dcd701038c89a6d
+size 5486

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/logs/EVAL_GPU_0_all_losses_reg_loss/events.out.tfevents.1729874043.autodl-container-b3ec4da47b-bc5fbea1.663163.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ee9efa1cd173238395f72430ea8fd85b71d712a988b1191c069ddec5230161c
+size 5486

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/logs/EVAL_GPU_0_all_losses_vtm_loss/events.out.tfevents.1729874043.autodl-container-b3ec4da47b-bc5fbea1.663163.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d610e01810c519a7e41dc31e8ecf599103f3e00022a6fc343e597d52875fddad
+size 5486

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/logs/events.out.tfevents.1729871205.autodl-container-b3ec4da47b-bc5fbea1.663163.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4045b4b0d93a6cdee08d58dce600856cb25a359a55874c68b4e03b29e86e87d
+size 20484

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/logs/events.out.tfevents.1729871205.autodl-container-b3ec4da47b-bc5fbea1.663164.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44b680fd20310c4773deaddbb8c93562931ca0921fef7900c2a67c03ce628aa6
+size 88

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/logs/train_GPU_0_all_losses_cls_loss/events.out.tfevents.1729871427.autodl-container-b3ec4da47b-bc5fbea1.663163.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53de0e531f68e19336dfb0fe941113e31478e4918067f9435f9efb3b0069cee7
+size 7407

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/logs/train_GPU_0_all_losses_reg_loss/events.out.tfevents.1729871427.autodl-container-b3ec4da47b-bc5fbea1.663163.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b50081741ae2873022666decf314f8cb1a5c2ab4af4d7fdf103d21877b1b481d
+size 7407

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/logs/train_GPU_0_all_losses_vtm_loss/events.out.tfevents.1729871427.autodl-container-b3ec4da47b-bc5fbea1.663163.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:995c1012741d37f3f379da79a4ef7399703746711bb50f4160d4b5e41ed92a7f
+size 7407

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/model_3_21.966959215281364.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:941a926ec2879bd0cc2a245a2eaa5ac4c03cbbed2deebc26c86ae15ed59c340b
+size 122083183

ego4d_nlq_v1/egovlp/finetuned/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4_objectmambafinetune173/run.sh

@@ -0,0 +1 @@
+bash tools/train_ego4d_finetune_head_twogpu.sh configs/ego4d_nlq_v1_multitask_egovlp_256_finetune_2e-4.yaml /root/autodl-tmp/model/GroundNLQ/ckpt/save/model_5_pretrain_egovlp.pth.tar objectmambafinetune173 0,1

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/config.txt

@@ -0,0 +1,226 @@
+{'dataset': {'classname_feat_concat': 'only',
+             'classname_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/classname-clip-base/a_photo_of.pt',
+             'default_fps': 30,
+             'downsample_rate': 1,
+             'enable_temporal_jittering': False,
+             'feat_stride': 16.0,
+             'fix_video_frames': 0,
+             'input_txt_dim': 512,
+             'input_vid_dim': 2304,
+             'json_file': './ego4d_data/ego4d_nlq_v2_ori_data/nlq_val.json',
+             'lavila_caption_dir': '/root/autodl-tmp/data/ego4d/nlq/lavila/narration',
+             'max_seq_len': 2560,
+             'num_classes': 1,
+             'num_frames': 16.0,
+             'object_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/co-detr/class-score0.6-minnum10-lmdb',
+             'object_feat_type': 'class-score',
+             'text_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/nlq_v2_clip_token_features',
+             'train_jsonl_file': './ego4d_data/ego4d_nlq_train_v2_label_lemma.jsonl',
+             'val_jsonl_file': './ego4d_data/ego4d_nlq_val_v2_label_lemma.jsonl',
+             'val_text_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/nlq_v2_clip_token_features',
+             'video_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/em_egovlp+internvideo_visual_features_1.87fps'},
+ 'dataset_name': 'ego4d_multitask',
+ 'devices': 'cuda:0',
+ 'init_rand_seed': 12345678,
+ 'loader': {'batch_size': 4, 'num_workers': 4},
+ 'model': {'backbone_arch': [2, 4, 4, 0, 6],
+           'backbone_type': 'ObjectMambaTransformer',
+           'embd_dim': 512,
+           'embd_kernel_size': 3,
+           'embd_with_ln': True,
+           'fpn_dim': 512,
+           'fpn_start_level': 0,
+           'fpn_type': 'identity',
+           'fpn_with_ln': True,
+           'generator': {'generator_type': 'point'},
+           'head_dim': 512,
+           'head_kernel_size': 3,
+           'head_num_layers': 3,
+           'head_with_ln': True,
+           'input_txt_dim': 512,
+           'input_vid_dim': 2304,
+           'max_buffer_len_factor': 4.0,
+           'max_query': 230,
+           'max_seq_len': 2560,
+           'max_shot_num': 1800,
+           'multiscale_encoder_cfg': [{'layer_cfg': {'mha_win_size': 9,
+                                                     'n_ds_strides': [2, 2],
+                                                     'n_embd': 512,
+                                                     'n_head': 4,
+                                                     'path_pdrop': 0.1},
+                                       'layer_num': 6,
+                                       'layer_type': 'TransformerBlock'},
+                                      {'layer_cfg': {'in_channels': [512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512],
+                                                     'out_channel': 512},
+                                       'layer_num': 1,
+                                       'layer_type': 'FPNLayernorm'}],
+           'n_head': 4,
+           'n_mha_win_size': 9,
+           'nlq_heads_cfg': {'center_sample_radius': 1.5,
+                             'cls_head_cfg': {'empty_cls': [],
+                                              'feat_dim': 512,
+                                              'input_dim': 512,
+                                              'kernel_size': 3,
+                                              'num_classes': 1,
+                                              'num_layers': 3,
+                                              'prior_prob': 0.01,
+                                              'with_ln': True},
+                             'duration_thresh': 0.001,
+                             'iou_threshold': 0.1,
+                             'loss_normalizer': 200,
+                             'loss_normalizer_momentum': 0.9,
+                             'max_seg_num': 5,
+                             'min_score': 0.001,
+                             'pre_nms_thresh': 0.001,
+                             'pre_nms_topk': 2000,
+                             'reg_head_cfg': {'feat_dim': 512,
+                                              'fpn_levels': 7,
+                                              'input_dim': 512,
+                                              'kernel_size': 3,
+                                              'num_layers': 3,
+                                              'with_ln': True},
+                             'reg_loss_weight': 1.0,
+                             'train_label_smoothing': 0.1},
+           'num_classes': 1,
+           'obj_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                              'n_in': 512,
+                                              'num_layer': 2},
+                                'layer_num': 1,
+                                'layer_type': 'MaskedConv1DLayer'},
+                               {'layer_cfg': {'n_embd': 512, 'path_pdrop': 0.1},
+                                'layer_num': 4,
+                                'layer_type': 'ObjectEncoderBlock'}],
+           'object_dim': 512,
+           'object_use_cross_model': True,
+           'object_win_size': 1,
+           'regression_range': [[0, 4],
+                                [2, 8],
+                                [4, 16],
+                                [8, 32],
+                                [16, 64],
+                                [32, 128],
+                                [64, 10000]],
+           'scale_factor': 2,
+           'tasks': ['NLQ', 'VTM'],
+           'test_cfg': {'duration_thresh': 0.001,
+                        'ext_score_file': None,
+                        'iou_threshold': 0.1,
+                        'max_seg_num': 5,
+                        'min_score': 0.001,
+                        'multiclass_nms': True,
+                        'nms_method': 'soft',
+                        'nms_sigma': 0.75,
+                        'pre_nms_thresh': 0.001,
+                        'pre_nms_topk': 2000,
+                        'test_num': 1,
+                        'test_start_epoch': 2,
+                        'voting_thresh': 0.9},
+           'text_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                               'n_in': 512,
+                                               'num_layer': 2},
+                                 'layer_num': 1,
+                                 'layer_type': 'MaskedConv1DLayer'},
+                                {'layer_cfg': {'n_embd': 512,
+                                               'n_head': 4,
+                                               'path_pdrop': 0.1},
+                                 'layer_num': 4,
+                                 'layer_type': 'TransformerBlock'}],
+           'train_cfg': {'box_loss_weight': 5.0,
+                         'center_sample': 'radius',
+                         'center_sample_radius': 1.5,
+                         'clip_grad_l2norm': 1.0,
+                         'cls_prior_prob': 0.01,
+                         'dropout': 0.0,
+                         'droppath': 0.1,
+                         'head_empty_cls': [],
+                         'init_loss_norm': 200,
+                         'iou_loss_weight': 1.0,
+                         'label_smoothing': 0.1,
+                         'loss_weight': 1.0,
+                         'mamba_arch': ['bimamba1', 'mlp', 'obj'],
+                         'num_decoder_layer': 6},
+           'use_abs_pe': True,
+           'use_lmha_in_fpn': True,
+           'use_rel_pe': False,
+           'video_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                                'kernel_size': 3,
+                                                'n_hidden': 512,
+                                                'n_in': 2304,
+                                                'n_out': 512,
+                                                'num_layer': 2},
+                                  'layer_num': 1,
+                                  'layer_type': 'MaskedConv1DLayer'},
+                                 {'layer_cfg': {'mamba_arch': ['bimamba1',
+                                                               'mlp',
+                                                               'obj'],
+                                                'n_embd': 512,
+                                                'n_head': 4,
+                                                'path_pdrop': 0.1},
+                                  'layer_num': 4,
+                                  'layer_type': 'ObjectMambaBlock'}],
+           'vtm_heads_cfg': {'loss_weight': 1.5,
+                             'multiscale': False,
+                             'shot_aggregator_cfg': {'layer_cfg': {'cross_mixer_cfg': {'block_cfg': {'n_embd': 512,
+                                                                                                     'n_head': 4},
+                                                                                       'block_type': 'MaskedMHA'},
+                                                                   'num_layer': 1,
+                                                                   'path_pdrop': 0.1,
+                                                                   'query_num': 5,
+                                                                   'self_mixer_cfg': {'block_cfg': {'n_embd': 512,
+                                                                                                    'n_head': 4},
+                                                                                      'block_type': 'MaskedMHCA'}},
+                                                     'layer_type': 'QFormerLayer'},
+                             'similarity_head_cfg': {'layer_cfg': {'con_dim': 512,
+                                                                   'x_dim': 512,
+                                                                   'y_dim': 512},
+                                                     'layer_type': 'Cosine'},
+                             'soft_label': False}},
+ 'model_name': 'MultiTaskArch',
+ 'opt': {'backbone_lr_weight': 1,
+         'epochs': 6,
+         'learning_rate': 0.0004,
+         'momentum': 0.9,
+         'schedule_gamma': 0.1,
+         'schedule_steps': [],
+         'schedule_type': 'cosine',
+         'type': 'AdamW',
+         'warmup': True,
+         'warmup_epochs': 4,
+         'weight_decay': 0.05},
+ 'output_folder': '/root/autodl-tmp/model/GroundNLQ/ckpt/',
+ 'test_cfg': {'duration_thresh': 0.001,
+              'ext_score_file': None,
+              'iou_threshold': 0.1,
+              'max_seg_num': 5,
+              'min_score': 0.001,
+              'multiclass_nms': True,
+              'nms_method': 'soft',
+              'nms_sigma': 0.75,
+              'pre_nms_thresh': 0.001,
+              'pre_nms_topk': 2000,
+              'test_num': 1,
+              'test_start_epoch': 2,
+              'voting_thresh': 0.9},
+ 'track': 'nlq',
+ 'train_cfg': {'box_loss_weight': 5.0,
+               'center_sample': 'radius',
+               'center_sample_radius': 1.5,
+               'clip_grad_l2norm': 1.0,
+               'cls_prior_prob': 0.01,
+               'dropout': 0.0,
+               'droppath': 0.1,
+               'head_empty_cls': [],
+               'init_loss_norm': 200,
+               'iou_loss_weight': 1.0,
+               'label_smoothing': 0.1,
+               'loss_weight': 1.0,
+               'mamba_arch': ['bimamba1', 'mlp', 'obj'],
+               'num_decoder_layer': 6},
+ 'train_split': ['training'],
+ 'val_split': ['validation']}

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/eval_results.txt

@@ -0,0 +1,44 @@
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  22.28   |  13.51   |  47.47   |  30.82   |  47.47   |  30.82   |
++----------+----------+----------+----------+----------+----------+avgiou=17.893234
+epoch0
+	final_loss 1.49 (1.50)
+	cls_loss 0.80 (0.82)
+	reg_loss 0.41 (0.37)
+	vtm_loss 0.18 (0.21)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  30.80   |  21.22   |  57.93   |  43.91   |  57.93   |  43.91   |
++----------+----------+----------+----------+----------+----------+avgiou=26.010545
+epoch1
+	final_loss 1.32 (1.47)
+	cls_loss 0.76 (0.81)
+	reg_loss 0.29 (0.34)
+	vtm_loss 0.18 (0.21)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  31.63   |  22.03   |  57.91   |  45.19   |  57.91   |  45.19   |
++----------+----------+----------+----------+----------+----------+avgiou=26.834359
+epoch2
+	final_loss 1.39 (1.48)
+	cls_loss 0.82 (0.84)
+	reg_loss 0.31 (0.34)
+	vtm_loss 0.17 (0.20)
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  30.82   |  20.98   |  56.81   |  43.83   |  56.81   |  43.83   |
++----------+----------+----------+----------+----------+----------+avgiou=25.900703
+epoch3
+	final_loss 1.45 (1.50)
+	cls_loss 0.80 (0.86)
+	reg_loss 0.40 (0.34)
+	vtm_loss 0.17 (0.20)

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/EVAL_GPU_0_all_losses_cls_loss/events.out.tfevents.1728637028.autodl-container-b3ec4da47b-bc5fbea1.33573.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:204898598341fc7043ef1565613b7d8f4f72c38b4f00cc21786f8cbaff7fd1cb
+size 2726

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/EVAL_GPU_0_all_losses_reg_loss/events.out.tfevents.1728637028.autodl-container-b3ec4da47b-bc5fbea1.33573.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:902bc45d32111db78cc6e77989359e1a7a6a84cd731e3cbd7ebd78db976394a7
+size 2726

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/EVAL_GPU_0_all_losses_vtm_loss/events.out.tfevents.1728637028.autodl-container-b3ec4da47b-bc5fbea1.33573.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2fb79a698b30e5224fea25ec682384c511beaf6adedc29c26457aa1e286dd94
+size 2726

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/events.out.tfevents.1728632235.autodl-container-b3ec4da47b-bc5fbea1.31554.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a939c2d48a6aea6f129586ee4d0fa6ba50f075927233c253ca3544769e15d79
+size 88

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/events.out.tfevents.1728632235.autodl-container-b3ec4da47b-bc5fbea1.31555.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5435840bf4ee16117de08cd1862d324adee13f0f8474bf803994ae794ad6b182
+size 88

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/events.out.tfevents.1728632793.autodl-container-b3ec4da47b-bc5fbea1.33573.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2835cf0277a91e459c72e8eed594345f5b97e35298e453e192332afa8cd47de3
+size 12974

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/events.out.tfevents.1728632793.autodl-container-b3ec4da47b-bc5fbea1.33574.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46d617f8b7b1e0cf8f671677c63e908bd78be53699e32afa09e17e4536bfbcfc
+size 88

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/train_GPU_0_all_losses_cls_loss/events.out.tfevents.1728633033.autodl-container-b3ec4da47b-bc5fbea1.33573.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:418ff6da494fef248a78f29ec7bd762321dd4fcdd73bca21d8a00876b14fcb8a
+size 5089

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/train_GPU_0_all_losses_reg_loss/events.out.tfevents.1728633033.autodl-container-b3ec4da47b-bc5fbea1.33573.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30a8ce87fb2738bb17843b94845afcfb532659e2426c7ff0cb7871f6dcc89bcf
+size 5089

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/logs/train_GPU_0_all_losses_vtm_loss/events.out.tfevents.1728633033.autodl-container-b3ec4da47b-bc5fbea1.33573.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44a0fc1c4df109b6a12877a31941824addd99bfdf6ec269d0ed190ea7a024a6b
+size 5089

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/model_2_26.834358523725836.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:382435ff1efe5a1167474e66d3bb31c1f33d80414bdd9c532ddeb08d39ccd139
+size 486572975

ego4d_nlq_v2/internvideo/finetune/ego4d_nlq_v2_multitask_finetune_2e-4_objectmambafinetune144/run.sh

@@ -0,0 +1 @@
+bash tools/train_ego4d_finetune_head_twogpu.sh configs/ego4d_nlq_v2_multitask_finetune_2e-4.yaml /root/autodl-tmp/model/GroundNLQ/ckpt/save/model_7_pretrain.pth.tar objectmambafinetune144 0,1

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/config.txt

@@ -0,0 +1,223 @@
+{'dataset': {'classname_feat_concat': 'only',
+             'classname_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/classname-clip-base/a_photo_of.pt',
+             'default_fps': 30,
+             'downsample_rate': 1,
+             'enable_temporal_jittering': False,
+             'feat_stride': 16.0,
+             'fix_video_frames': 0,
+             'input_txt_dim': 512,
+             'input_vid_dim': 2304,
+             'json_file': 'ego4d_data/goalstep_data/ego4d_goal_step_val_v2_lemma.jsonl',
+             'lavila_caption_dir': '/root/autodl-tmp/data/ego4d/goalstep/lavila-64/',
+             'max_seq_len': 9216,
+             'num_classes': 1,
+             'num_frames': 16.0,
+             'object_feat_dir': '/root/autodl-tmp/data/ego4d/goalstep/co-detr/clip-class-lmdb',
+             'object_feat_type': 'class-score',
+             'text_feat_dir': '/root/autodl-tmp/data/ego4d/goalstep/clip_query_lmdb',
+             'train_jsonl_file': 'ego4d_data/goalstep_data/clip/ego4d_goal_step_train_v2.jsonl',
+             'val_jsonl_file': 'ego4d_data/goalstep_data/ego4d_goal_step_val_v2_lemma.jsonl',
+             'val_text_feat_dir': '/root/autodl-tmp/data/ego4d/goalstep/clip_query_lmdb',
+             'video_feat_dir': '/root/autodl-tmp/data/ego4d/goalstep/video_feature/internvideo_clip_lmdb'},
+ 'dataset_name': 'ego4d_multitask',
+ 'devices': 'cuda:0',
+ 'init_rand_seed': 12345678,
+ 'loader': {'batch_size': 1, 'num_workers': 1},
+ 'model': {'backbone_arch': (2, 2, 2, 0, 6),
+           'backbone_type': 'convTransformer',
+           'embd_dim': 512,
+           'embd_kernel_size': 3,
+           'embd_with_ln': True,
+           'fpn_dim': 512,
+           'fpn_start_level': 0,
+           'fpn_type': 'identity',
+           'fpn_with_ln': True,
+           'generator': {'generator_type': 'point'},
+           'head_dim': 512,
+           'head_kernel_size': 3,
+           'head_num_layers': 3,
+           'head_with_ln': True,
+           'input_txt_dim': 512,
+           'input_vid_dim': 2304,
+           'max_buffer_len_factor': 4.0,
+           'max_query': 560,
+           'max_seq_len': 9216,
+           'max_shot_num': 3400,
+           'multiscale_encoder_cfg': [{'layer_cfg': {'mha_win_size': 9,
+                                                     'n_ds_strides': [2, 2],
+                                                     'n_embd': 512,
+                                                     'n_head': 4,
+                                                     'path_pdrop': 0.1},
+                                       'layer_num': 6,
+                                       'layer_type': 'TransformerBlock'},
+                                      {'layer_cfg': {'in_channels': [512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512],
+                                                     'out_channel': 512},
+                                       'layer_num': 1,
+                                       'layer_type': 'FPNLayernorm'}],
+           'n_head': 4,
+           'n_mha_win_size': -1,
+           'nlq_heads_cfg': {'center_sample_radius': 1.5,
+                             'cls_head_cfg': {'empty_cls': [],
+                                              'feat_dim': 512,
+                                              'input_dim': 512,
+                                              'kernel_size': 3,
+                                              'num_classes': 1,
+                                              'num_layers': 3,
+                                              'prior_prob': 0.01,
+                                              'with_ln': True},
+                             'duration_thresh': 0.001,
+                             'iou_threshold': 0.1,
+                             'loss_normalizer': 200,
+                             'loss_normalizer_momentum': 0.9,
+                             'max_seg_num': 5,
+                             'min_score': 0.001,
+                             'pre_nms_thresh': 0.001,
+                             'pre_nms_topk': 2000,
+                             'reg_head_cfg': {'feat_dim': 512,
+                                              'fpn_levels': 7,
+                                              'input_dim': 512,
+                                              'kernel_size': 3,
+                                              'num_layers': 3,
+                                              'with_ln': True},
+                             'reg_loss_weight': 1.0,
+                             'train_label_smoothing': 0.1},
+           'num_classes': 1,
+           'obj_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                              'n_in': 512,
+                                              'num_layer': 2},
+                                'layer_num': 1,
+                                'layer_type': 'MaskedConv1DLayer'},
+                               {'layer_cfg': {'n_embd': 512, 'path_pdrop': 0.1},
+                                'layer_num': 4,
+                                'layer_type': 'ObjectEncoderBlock'}],
+           'regression_range': [[0, 4],
+                                [2, 8],
+                                [4, 16],
+                                [8, 32],
+                                [16, 64],
+                                [32, 128],
+                                [64, 10000]],
+           'scale_factor': 2,
+           'tasks': ['NLQ', 'VTM'],
+           'test_cfg': {'duration_thresh': 0.001,
+                        'ext_score_file': None,
+                        'iou_threshold': 0.1,
+                        'max_seg_num': 5,
+                        'min_score': 0.001,
+                        'multiclass_nms': True,
+                        'nms_method': 'soft',
+                        'nms_sigma': 0.75,
+                        'pre_nms_thresh': 0.001,
+                        'pre_nms_topk': 2000,
+                        'test_num': 1,
+                        'test_start_epoch': 2,
+                        'voting_thresh': 0.9},
+           'text_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                               'n_in': 512,
+                                               'num_layer': 2},
+                                 'layer_num': 1,
+                                 'layer_type': 'MaskedConv1DLayer'},
+                                {'layer_cfg': {'n_embd': 512,
+                                               'n_head': 4,
+                                               'path_pdrop': 0.1},
+                                 'layer_num': 4,
+                                 'layer_type': 'TransformerBlock'}],
+           'train_cfg': {'box_loss_weight': 5.0,
+                         'center_sample': 'radius',
+                         'center_sample_radius': 1.5,
+                         'clip_grad_l2norm': 1.0,
+                         'cls_prior_prob': 0.01,
+                         'dropout': 0.0,
+                         'droppath': 0.1,
+                         'head_empty_cls': [],
+                         'init_loss_norm': 200,
+                         'iou_loss_weight': 1.0,
+                         'label_smoothing': 0.1,
+                         'loss_weight': 1.0,
+                         'mamba_arch': ['bimamba1', 'mlp', 'obj'],
+                         'num_decoder_layer': 6},
+           'use_abs_pe': True,
+           'use_lmha_in_fpn': True,
+           'use_rel_pe': False,
+           'video_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                                'kernel_size': 3,
+                                                'n_hidden': 512,
+                                                'n_in': 2304,
+                                                'n_out': 512,
+                                                'num_layer': 2},
+                                  'layer_num': 1,
+                                  'layer_type': 'MaskedConv1DLayer'},
+                                 {'layer_cfg': {'mamba_arch': ['bimamba1',
+                                                               'mlp',
+                                                               'obj'],
+                                                'n_embd': 512,
+                                                'n_head': 4,
+                                                'path_pdrop': 0.1},
+                                  'layer_num': 4,
+                                  'layer_type': 'ObjectMambaBlock'}],
+           'vtm_heads_cfg': {'loss_weight': 1.5,
+                             'multiscale': False,
+                             'shot_aggregator_cfg': {'layer_cfg': {'cross_mixer_cfg': {'block_cfg': {'n_embd': 512,
+                                                                                                     'n_head': 4},
+                                                                                       'block_type': 'MaskedMHA'},
+                                                                   'num_layer': 1,
+                                                                   'path_pdrop': 0.1,
+                                                                   'query_num': 5,
+                                                                   'self_mixer_cfg': {'block_cfg': {'n_embd': 512,
+                                                                                                    'n_head': 4},
+                                                                                      'block_type': 'MaskedMHCA'}},
+                                                     'layer_type': 'QFormerLayer'},
+                             'similarity_head_cfg': {'layer_cfg': {'con_dim': 512,
+                                                                   'x_dim': 512,
+                                                                   'y_dim': 512},
+                                                     'layer_type': 'Cosine'},
+                             'soft_label': False}},
+ 'model_name': 'MultiTaskArch',
+ 'opt': {'backbone_lr_weight': 1,
+         'epochs': 6,
+         'learning_rate': 0.0008,
+         'momentum': 0.9,
+         'schedule_gamma': 0.1,
+         'schedule_steps': [],
+         'schedule_type': 'cosine',
+         'type': 'AdamW',
+         'warmup': True,
+         'warmup_epochs': 4,
+         'weight_decay': 0.05},
+ 'output_folder': '/root/autodl-tmp/model/GroundNLQ/goalstep/',
+ 'test_cfg': {'duration_thresh': 0.001,
+              'ext_score_file': None,
+              'iou_threshold': 0.1,
+              'max_seg_num': 5,
+              'min_score': 0.001,
+              'multiclass_nms': True,
+              'nms_method': 'soft',
+              'nms_sigma': 0.75,
+              'pre_nms_thresh': 0.001,
+              'pre_nms_topk': 2000,
+              'test_num': 1,
+              'test_start_epoch': 2,
+              'voting_thresh': 0.9},
+ 'track': 'goal_step',
+ 'train_cfg': {'box_loss_weight': 5.0,
+               'center_sample': 'radius',
+               'center_sample_radius': 1.5,
+               'clip_grad_l2norm': 1.0,
+               'cls_prior_prob': 0.01,
+               'dropout': 0.0,
+               'droppath': 0.1,
+               'head_empty_cls': [],
+               'init_loss_norm': 200,
+               'iou_loss_weight': 1.0,
+               'label_smoothing': 0.1,
+               'loss_weight': 1.0,
+               'mamba_arch': ['bimamba1', 'mlp', 'obj'],
+               'num_decoder_layer': 6},
+ 'train_split': ['training'],
+ 'val_split': ['validation']}

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/eval_results.txt

@@ -0,0 +1,56 @@
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  26.13   |  21.61   |  56.73   |  49.30   |  56.73   |  49.30   |
++----------+----------+----------+----------+----------+----------+avgiou=26.130457
+epoch0
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  27.99   |  23.83   |  58.72   |  51.57   |  58.72   |  51.57   |
++----------+----------+----------+----------+----------+----------+avgiou=27.988565
+epoch1
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  28.42   |  23.79   |  59.50   |  52.39   |  59.50   |  52.39   |
++----------+----------+----------+----------+----------+----------+avgiou=28.417360
+epoch2
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  28.17   |  24.06   |  58.74   |  51.74   |  58.74   |  51.74   |
++----------+----------+----------+----------+----------+----------+avgiou=28.170478
+epoch3
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  28.74   |  24.27   |  59.16   |  52.27   |  59.16   |  52.27   |
++----------+----------+----------+----------+----------+----------+avgiou=28.742204
+epoch4
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  28.60   |  24.44   |  59.65   |  53.13   |  59.65   |  53.13   |
++----------+----------+----------+----------+----------+----------+avgiou=28.599272
+epoch5
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  29.61   |  24.94   |  59.51   |  52.48   |  59.51   |  52.48   |
++----------+----------+----------+----------+----------+----------+avgiou=29.612786
+epoch6
++----------+----------+----------+----------+----------+----------+
+|  Rank@1  |  Rank@1  |  Rank@5  |  Rank@5  | Rank@10  | Rank@10  |
+| mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 | mIoU@0.3 | mIoU@0.5 |
++----------+----------+----------+----------+----------+----------+
+|  28.92   |  24.57   |  59.17   |  52.43   |  59.17   |  52.43   |
++----------+----------+----------+----------+----------+----------+avgiou=28.924116
+epoch7

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/events.out.tfevents.1726904924.autodl-container-b3ec4da47b-bc5fbea1.458057.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecca71d48639775ce1f6c217bfe928294fc08b01b5568b66109cb367ea1b9f63
+size 88

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/events.out.tfevents.1726904924.autodl-container-b3ec4da47b-bc5fbea1.458059.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00aaeab73622930454886f391bb37812060334fc4e955468ec0796ca6f992f96
+size 88

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/events.out.tfevents.1726904924.autodl-container-b3ec4da47b-bc5fbea1.458060.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64ac5b1566c4058ef555d97b101936cee9aeb90cfcb2885b648f348e1b6fd8d9
+size 88

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/events.out.tfevents.1726906438.autodl-container-b3ec4da47b-bc5fbea1.460310.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d096f2b7006f2deb0a4c10eaa54f0a69f89c4ff41eaa521de174b436e93169b
+size 88

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/events.out.tfevents.1726906439.autodl-container-b3ec4da47b-bc5fbea1.460307.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4e28bd158929f7db87bfc69169443fa99dcb61a8e47c4fd235ea86cfebe403a
+size 79899

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/events.out.tfevents.1726906439.autodl-container-b3ec4da47b-bc5fbea1.460308.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91a45dc73a5396e958bb5b8b0ed845bf85c3d7625c807f712686c3b41b9af2cd
+size 88

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/events.out.tfevents.1726906439.autodl-container-b3ec4da47b-bc5fbea1.460309.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b33462cec9f03325622b13d84818afb653ab1aea89bfe5f3449bc670b0ea288
+size 88

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/train_GPU_0_all_losses_cls_loss/events.out.tfevents.1726906838.autodl-container-b3ec4da47b-bc5fbea1.460307.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:695ef39d29320f4820ae3af21b36d103b3b88a547c56db757caf66deae948f36
+size 39047

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/train_GPU_0_all_losses_reg_loss/events.out.tfevents.1726906838.autodl-container-b3ec4da47b-bc5fbea1.460307.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74c8ee73938de8a9ebbf72aa4ed3401902ef7ad497b49ca30a5042dccf8a001b
+size 39047

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/logs/train_GPU_0_all_losses_vtm_loss/events.out.tfevents.1726906838.autodl-container-b3ec4da47b-bc5fbea1.460307.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:692dfc9b278fc5fa5ebcb849f351b4c6c14636f88c16f4c3b6de74ba5ac0d545
+size 39047

goalstep/internvideo/finetuned/ego4d_goalstep_v2_baseline_2e-4_objectmambafinetune135/run.sh

@@ -0,0 +1 @@
+bash tools/train_ego4d_finetune_head_4gpu_noeval.sh configs/ego4d_goalstep_v2_baseline_2e-4.yaml /root/autodl-tmp/model/GroundNLQ/ckpt/save/model_7_pretrain.pth.tar objectmambafinetune135 0,1,2,3

pretrain_naq/egovlp/model_5_pretrain_egovlp.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0582a3fc28fd33b0c1a183014ca70eaa39faffad9c5955f97ce460f01324b04d
+size 122080780

pretrain_naq/internvideo/model_7_pretrain.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55ea828749455f900e0d7bad3556d6d4035d31a7fc247a1e3d6b697a86ec5ed1
+size 425493745

tacos/c3d/scratch/tacos_c3d_glove_weight1_5e-5_objectmambafinetune150/config.txt

@@ -0,0 +1,228 @@
+{'dataset': {'classname_feat_concat': 'only',
+             'classname_feat_dir': '/root/autodl-tmp/data/ego4d/nlq/classname-clip-base/a_photo_of.pt',
+             'default_fps': 29.4,
+             'downsample_rate': 1,
+             'enable_temporal_jittering': False,
+             'feat_stride': 16.0,
+             'fix_video_frames': 0,
+             'input_txt_dim': 512,
+             'input_vid_dim': 4096,
+             'json_file': 'ego4d_data/tacos/test_lemma.jsonl',
+             'lavila_caption_dir': '/root/autodl-tmp/data/tacos/lavila',
+             'max_seq_len': 3072,
+             'num_classes': 1,
+             'num_frames': 16.0,
+             'object_feat_dir': '/root/autodl-tmp/data/tacos/class-score0.6-minnum10-lmdb',
+             'object_feat_type': 'class-score',
+             'text_feat_dir': '/root/autodl-tmp/data/tacos/glove_clip_token_features',
+             'train_jsonl_file': 'ego4d_data/tacos/train_lemma.jsonl',
+             'val_jsonl_file': 'ego4d_data/tacos/test_lemma.jsonl',
+             'val_text_feat_dir': '/root/autodl-tmp/data/tacos/glove_clip_token_features',
+             'video_feat_dir': '/root/autodl-tmp/data/tacos/c3d_lmdb'},
+ 'dataset_name': 'ego4d_multitask',
+ 'devices': 'cuda:0',
+ 'init_rand_seed': 12345678,
+ 'loader': {'batch_size': 2, 'num_workers': 2},
+ 'model': {'backbone_arch': [2, 4, 4, 0, 6],
+           'backbone_type': 'ObjectMambaTransformer',
+           'embd_dim': 512,
+           'embd_kernel_size': 3,
+           'embd_with_ln': True,
+           'fpn_dim': 512,
+           'fpn_start_level': 0,
+           'fpn_type': 'identity',
+           'fpn_with_ln': True,
+           'generator': {'generator_type': 'point'},
+           'head_dim': 512,
+           'head_kernel_size': 3,
+           'head_num_layers': 3,
+           'head_with_ln': True,
+           'input_txt_dim': 512,
+           'input_vid_dim': 4096,
+           'max_buffer_len_factor': 4.0,
+           'max_query': 500,
+           'max_seq_len': 3072,
+           'max_shot_num': 100,
+           'multiscale_encoder_cfg': [{'layer_cfg': {'mha_win_size': 9,
+                                                     'n_ds_strides': [2, 2],
+                                                     'n_embd': 512,
+                                                     'n_head': 4,
+                                                     'path_pdrop': 0.1},
+                                       'layer_num': 6,
+                                       'layer_type': 'TransformerBlock'},
+                                      {'layer_cfg': {'in_channels': [512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512,
+                                                                     512],
+                                                     'out_channel': 512},
+                                       'layer_num': 1,
+                                       'layer_type': 'FPNLayernorm'}],
+           'n_head': 4,
+           'n_mha_win_size': 9,
+           'nlq_heads_cfg': {'center_sample_radius': 1.5,
+                             'cls_head_cfg': {'empty_cls': [],
+                                              'feat_dim': 512,
+                                              'input_dim': 512,
+                                              'kernel_size': 3,
+                                              'num_classes': 1,
+                                              'num_layers': 3,
+                                              'prior_prob': 0.01,
+                                              'with_ln': True},
+                             'duration_thresh': 0.001,
+                             'iou_threshold': 0.1,
+                             'loss_normalizer': 200,
+                             'loss_normalizer_momentum': 0.9,
+                             'max_seg_num': 5,
+                             'min_score': 0.001,
+                             'pre_nms_thresh': 0.001,
+                             'pre_nms_topk': 2000,
+                             'reg_head_cfg': {'feat_dim': 512,
+                                              'fpn_levels': 7,
+                                              'input_dim': 512,
+                                              'kernel_size': 3,
+                                              'num_layers': 3,
+                                              'with_ln': True},
+                             'reg_loss_weight': 1.0,
+                             'train_label_smoothing': 0.1},
+           'num_classes': 1,
+           'obj_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                              'n_in': 512,
+                                              'num_layer': 2},
+                                'layer_num': 1,
+                                'layer_type': 'MaskedConv1DLayer'},
+                               {'layer_cfg': {'n_embd': 512, 'path_pdrop': 0.1},
+                                'layer_num': 4,
+                                'layer_type': 'ObjectEncoderBlock'}],
+           'object_dim': 512,
+           'object_use_cross_model': True,
+           'object_win_size': 1,
+           'regression_range': [[0, 4],
+                                [2, 8],
+                                [4, 16],
+                                [8, 32],
+                                [16, 64],
+                                [32, 128],
+                                [64, 10000]],
+           'scale_factor': 2,
+           'tasks': ['NLQ', 'VTM'],
+           'test_cfg': {'duration_thresh': 0.001,
+                        'ext_score_file': None,
+                        'iou_threshold': 0.1,
+                        'max_seg_num': 5,
+                        'min_score': 0.001,
+                        'multiclass_nms': True,
+                        'nms_method': 'soft',
+                        'nms_sigma': 0.75,
+                        'pre_nms_thresh': 0.001,
+                        'pre_nms_topk': 2000,
+                        'test_num': 1,
+                        'test_start_epoch': 2,
+                        'voting_thresh': 0.9},
+           'text_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                               'n_in': 300,
+                                               'n_out': 512,
+                                               'num_layer': 2},
+                                 'layer_num': 1,
+                                 'layer_type': 'MaskedConv1DLayer'},
+                                {'layer_cfg': {'n_embd': 512,
+                                               'n_head': 4,
+                                               'path_pdrop': 0.1},
+                                 'layer_num': 4,
+                                 'layer_type': 'TransformerBlock',
+                                 'use_abs_pe': True}],
+           'train_cfg': {'box_loss_weight': 5.0,
+                         'center_sample': 'radius',
+                         'center_sample_radius': 1.5,
+                         'clip_grad_l2norm': 1.0,
+                         'cls_prior_prob': 0.01,
+                         'dropout': 0.0,
+                         'droppath': 0.1,
+                         'head_empty_cls': [],
+                         'init_loss_norm': 200,
+                         'iou_loss_weight': 1.0,
+                         'label_smoothing': 0.1,
+                         'loss_weight': 1.0,
+                         'mamba_arch': ['bimamba1', 'mlp', 'obj'],
+                         'num_decoder_layer': 6},
+           'use_abs_pe': True,
+           'use_lmha_in_fpn': True,
+           'use_rel_pe': False,
+           'video_encoder_cfg': [{'layer_cfg': {'act': 'relu',
+                                                'kernel_size': 3,
+                                                'n_hidden': 512,
+                                                'n_in': 4096,
+                                                'n_out': 512,
+                                                'num_layer': 2},
+                                  'layer_num': 1,
+                                  'layer_type': 'MaskedConv1DLayer'},
+                                 {'layer_cfg': {'mamba_arch': ['bimamba1',
+                                                               'mlp',
+                                                               'obj'],
+                                                'n_embd': 512,
+                                                'n_head': 4,
+                                                'path_pdrop': 0.1},
+                                  'layer_num': 4,
+                                  'layer_type': 'ObjectMambaBlock'}],
+           'vtm_heads_cfg': {'loss_weight': 1.0,
+                             'multiscale': False,
+                             'shot_aggregator_cfg': {'layer_cfg': {'cross_mixer_cfg': {'block_cfg': {'n_embd': 512,
+                                                                                                     'n_head': 4},
+                                                                                       'block_type': 'MaskedMHA'},
+                                                                   'num_layer': 1,
+                                                                   'path_pdrop': 0.1,
+                                                                   'query_num': 5,
+                                                                   'self_mixer_cfg': {'block_cfg': {'n_embd': 512,
+                                                                                                    'n_head': 4},
+                                                                                      'block_type': 'MaskedMHCA'}},
+                                                     'layer_type': 'QFormerLayer'},
+                             'similarity_head_cfg': {'layer_cfg': {'con_dim': 512,
+                                                                   'x_dim': 512,
+                                                                   'y_dim': 512},
+                                                     'layer_type': 'Cosine'},
+                             'soft_label': False}},
+ 'model_name': 'MultiTaskArch',
+ 'opt': {'backbone_lr_weight': 1,
+         'epochs': 6,
+         'learning_rate': 0.0002,
+         'momentum': 0.9,
+         'schedule_gamma': 0.1,
+         'schedule_steps': [],
+         'schedule_type': 'cosine',
+         'type': 'AdamW',
+         'warmup': True,
+         'warmup_epochs': 4,
+         'weight_decay': 0.05},
+ 'output_folder': '/root/autodl-tmp/model/GroundNLQ/tacos/',
+ 'test_cfg': {'duration_thresh': 0.001,
+              'ext_score_file': None,
+              'iou_threshold': 0.1,
+              'max_seg_num': 5,
+              'min_score': 0.001,
+              'multiclass_nms': True,
+              'nms_method': 'soft',
+              'nms_sigma': 0.75,
+              'pre_nms_thresh': 0.001,
+              'pre_nms_topk': 2000,
+              'test_num': 1,
+              'test_start_epoch': 2,
+              'voting_thresh': 0.9},
+ 'track': 'goal_step',
+ 'train_cfg': {'box_loss_weight': 5.0,
+               'center_sample': 'radius',
+               'center_sample_radius': 1.5,
+               'clip_grad_l2norm': 1.0,
+               'cls_prior_prob': 0.01,
+               'dropout': 0.0,
+               'droppath': 0.1,
+               'head_empty_cls': [],
+               'init_loss_norm': 200,
+               'iou_loss_weight': 1.0,
+               'label_smoothing': 0.1,
+               'loss_weight': 1.0,
+               'mamba_arch': ['bimamba1', 'mlp', 'obj'],
+               'num_decoder_layer': 6},
+ 'train_split': ['training'],
+ 'val_split': ['validation']}