initial commit

DOCKERFILE

@@ -0,0 +1,29 @@
+FROM continuumio/anaconda3:main
+
+WORKDIR /code
+COPY ./environment_docker.yml /code/environment_docker.yml
+
+# Create the environment using the environment.yml file
+RUN conda env create -f /code/environment_docker.yml
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PYTHONPATH=$HOME/app \
+    PYTHONUNBUFFERED=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_THEME=huggingface \
+    SYSTEM=spaces
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+CMD ["./run.sh"]

README.md

@@ -1,11 +1,8 @@
 ---
-title: MASA GroundingDINO
-emoji: 🌍
-colorFrom: red
-colorTo: pink
+title: MASA + GroundingDINO Space
+emoji: 🐳
+colorFrom: purple
+colorTo: gray
 sdk: docker
-pinned: false
-license: mit
+app_port: 7860
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,71 @@
+import gradio as gr
+import os
+import tempfile
+import subprocess
+
+# Define the function to call the command line script
+def process_video(uploaded_video_path, texts):
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as tmpfile:
+        output_video_path = tmpfile.name
+
+    command = [
+        "python", "demo/video_demo_with_text.py", uploaded_video_path,
+        "--out", output_video_path,
+        "--masa_config", "configs/masa-gdino/masa_gdino_swinb_inference.py",
+        "--masa_checkpoint", "saved_models/masa_models/gdino_masa.pth",
+        "--texts", texts,
+        "--score-thr", "0.2",
+        "--unified",
+        "--show_fps"
+    ]
+
+    subprocess.run(command, check=True)
+
+    # Ensure the video is in a compatible format using ffmpeg
+    converted_output_path = output_video_path.replace('.mp4', '_converted.mp4')
+    ffmpeg_command = [
+        "ffmpeg", "-i", output_video_path, "-c:v", "mpeg4", 
+        "-c:a", "aac", "-b:a", "128k", "-movflags", "+faststart", converted_output_path
+    ]
+    subprocess.run(ffmpeg_command, check=True)
+
+    return converted_output_path
+
+css = """
+#img-display-container {
+    max-height: 100vh;
+    }
+#img-display-input {
+    max-height: 80vh;
+    }
+#img-display-output {
+    max-height: 80vh;
+    }
+"""
+
+title = "# MASA Track Everything Demo"
+description = """ MASA + GroundingDINO on your video files!
+Please refer to our [paper](https://arxiv.org/abs/2406.04221), [project page](https://matchinganything.github.io/), or [github](https://github.com/siyuanliii/masa/tree/main?tab=readme-ov-file) for more details."""
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown(title)
+    gr.Markdown(description)
+    gr.Markdown("### Video Object Tracking demo")
+
+    with gr.Row():
+        input_video = gr.Video(label="Input Video")
+        input_texts = gr.Textbox(label="Input Texts")
+
+    submit = gr.Button("Submit")
+    processed_video = gr.Video(label="Processed Video")
+
+    submit.click(process_video, inputs=[input_video, input_texts], outputs=processed_video)
+
+    example_files = os.listdir('assets/examples_video')
+    example_files.sort()
+    example_files = [os.path.join('assets/examples_video', filename) for filename in example_files]
+    examples = gr.Examples(examples=example_files, inputs=[input_video, input_texts], outputs=processed_video, fn=process_video, cache_examples=True)
+
+if __name__ == '__main__':
+    demo.queue().launch()
+

configs/datasets/bdd/bdd_dataset.py

@@ -0,0 +1,44 @@
+# dataset settings
+img_norm_cfg = dict(
+    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+
+
+test_dataset_tpye = 'BDDVideoDataset'
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='Resize', scale=(1333, 800), keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+val_dataloader = dict(
+    batch_size=1,
+    num_workers=2,
+    persistent_workers=True,
+    sampler=dict(type='TrackImgSampler'),
+    dataset=dict(
+        type=test_dataset_tpye,
+        ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+        data_prefix=dict(img_path='data/bdd/bdd100k/images/track/val/'),
+        test_mode=True,
+        pipeline=test_pipeline
+    ))
+
+test_dataloader = val_dataloader
+
+# evaluator
+val_evaluator = dict(
+    type='BDDTETAMetric',
+    dataset_type=test_dataset_tpye,
+    format_only=False,
+    ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/box_track_20/val/',
+    metric=['TETA'])
+test_evaluator = val_evaluator
+
+

configs/datasets/tao/tao_dataset_v05.py

@@ -0,0 +1,43 @@
+# data pipeline
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='Resize', scale=(1333, 800), keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# dataloader
+
+test_dataset_tpye = 'Taov05Dataset'
+
+val_dataloader = dict(
+    batch_size=1,
+    num_workers=2,
+    persistent_workers=True,
+    # Now we support two ways to test, image_based and video_based
+    # if you want to use video_based sampling, you can use as follows
+    sampler=dict(type='TrackImgSampler'),  # image-based sampling
+    dataset=dict(
+        type=test_dataset_tpye,
+        ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+        data_prefix=dict(img_path='data/tao/frames/'),
+        test_mode=True,
+        pipeline=test_pipeline
+    ))
+test_dataloader = val_dataloader
+
+# evaluator
+val_evaluator = dict(
+    type='TaoTETAMetric',
+    dataset_type=test_dataset_tpye,
+    format_only=False,
+    ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    metric=['TETA'])
+test_evaluator = val_evaluator
+
+

configs/datasets/tao/tao_dataset_v1.py

@@ -0,0 +1,44 @@
+# data pipeline
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='Resize', scale=(1333, 800), keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# dataloader
+
+test_dataset_tpye = 'Taov1Dataset'
+
+val_dataloader = dict(
+    batch_size=1,
+    num_workers=2,
+    persistent_workers=True,
+    # Now we support two ways to test, image_based and video_based
+    # if you want to use video_based sampling, you can use as follows
+    sampler=dict(type='TrackImgSampler'),  # image-based sampling
+    dataset=dict(
+        type=test_dataset_tpye,
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        data_prefix=dict(img_path='data/tao/frames/'),
+        test_mode=True,
+        pipeline=test_pipeline
+    ))
+
+test_dataloader = val_dataloader
+
+# evaluator
+val_evaluator = dict(
+    type='TaoTETAMetric',
+    dataset_type=test_dataset_tpye,
+    format_only=False,
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    metric=['TETA'])
+test_evaluator = val_evaluator
+
+

configs/default_runtime.py

@@ -0,0 +1,23 @@
+default_scope = 'mmdet'
+default_hooks = dict(
+    timer=dict(type='IterTimerHook'),
+    logger=dict(type='LoggerHook', interval=50),
+    param_scheduler=dict(type='ParamSchedulerHook'),
+    checkpoint=dict(type='CheckpointHook', interval=1),
+    sampler_seed=dict(type='DistSamplerSeedHook'),
+    visualization=dict(type='DetVisualizationHook'))
+
+env_cfg = dict(
+    cudnn_benchmark=False,
+    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0),
+    dist_cfg=dict(backend='nccl'),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='DetLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+log_processor = dict(type='LogProcessor', window_size=50, by_epoch=True)
+
+log_level = 'INFO'
+load_from = None
+resume = False

configs/masa-detic/bdd_test/masa_detic_bdd_mot_test.py

@@ -0,0 +1,224 @@
+_base_ = [
+    '../../../projects/Detic_new/configs/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-masa.py',
+    '../../datasets/bdd/bdd_dataset.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-ec91245d.pth'
+    # noqa: E501
+)
+detector['type'] = 'DeticMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'bdd',
+    public_det_path = 'results/public_dets/bdd_mot_yolox_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+    checkpoint=dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/box_track_20/val/',
+    outfile_prefix='results/detic_masa_trained_bdd_demo',
+    metric=['TETA', 'HOTA', 'CLEAR']
+)
+test_evaluator = val_evaluator

configs/masa-detic/bdd_test/masa_detic_bdd_mots_test.py

@@ -0,0 +1,227 @@
+_base_ = [
+    '../../projects/Detic_new/configs/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-masa.py',
+    '../datasets/bdd/bdd_dataset.py',
+    '../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-ec91245d.pth'
+    # noqa: E501
+)
+detector['type'] = 'DeticMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    with_segm=True,
+    benchmark = 'bdd',
+    public_det_path = 'results/public_dets/bdd_mots_val_uninext_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/seg_track_20/val/',
+    outfile_prefix='results/masa_results/masa-groundingdino-release-bdd-mots-test',
+    metric=['TETA', 'HOTA', 'CLEAR'],
+    with_mask=True,
+)
+test_evaluator = val_evaluator

configs/masa-detic/open_vocabulary_mot_test/masa_detic_swinb_open_vocabulary_test.py

@@ -0,0 +1,236 @@
+_base_ = [
+    '../../../projects/Detic_new/configs/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-masa.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-ec91245d.pth'
+    # noqa: E501
+)
+detector['type'] = 'DeticMasa'
+detector['test_cfg'] =dict(
+        rpn=dict(
+            score_thr=0.0001,
+            nms_pre=1000,
+            max_per_img=256,
+            nms=dict(type='nms', iou_threshold=0.9),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+    )
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = False,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.8,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+# custom hooks
+custom_hooks = [
+    # Synchronize model buffers such as running_mean and running_var in BN
+    # at the end of each epoch
+    dict(type='SyncBuffersHook')
+]
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    outfile_prefix='results/masa_results/masa-detic-release-ovmot-test',
+    open_vocabulary=True,
+)
+test_evaluator = val_evaluator

configs/masa-detic/tao_teta_test/masa_detic_swinb_tao_test_detic_dets.py

@@ -0,0 +1,219 @@
+_base_ = [
+    '../../../projects/Detic_new/configs/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-masa.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-ec91245d.pth'
+    # noqa: E501
+)
+detector['type'] = 'DeticMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.8,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+# custom hooks
+custom_hooks = [
+    # Synchronize model buffers such as running_mean and running_var in BN
+    # at the end of each epoch
+    dict(type='SyncBuffersHook')
+]
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    outfile_prefix='results/masa_results/masa-detic-release-detic-dets-tao-test',
+)
+test_evaluator = val_evaluator

configs/masa-detic/tao_teta_test/masa_detic_swinb_tao_test_teter_swinT_dets.py

@@ -0,0 +1,219 @@
+_base_ = [
+    '../../../projects/Detic_new/configs/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-masa.py',
+    '../../datasets/tao/tao_dataset_v05.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/detic_centernet2_swin-b_fpn_4x_lvis-base_in21k-lvis-ec91245d.pth'
+    # noqa: E501
+)
+detector['type'] = 'DeticMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark='tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/teter_swinT_tao_val_internms_50/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.8,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+# custom hooks
+custom_hooks = [
+    # Synchronize model buffers such as running_mean and running_var in BN
+    # at the end of each epoch
+    dict(type='SyncBuffersHook')
+]
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json'
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    outfile_prefix='results/masa_results/masa-detic-release-test',
+)
+test_evaluator = val_evaluator

configs/masa-gdino/bdd_test/masa_gdino_bdd_mot_test.py

@@ -0,0 +1,226 @@
+_base_ = [
+    '../../../projects/grounding_dino/grounding_dino_swin-b_pretrain_mixeddata_masa.py',
+    '../../datasets/bdd/bdd_dataset.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+# detector.backbone.update(dict(out_indices=(1, 2, 3)))
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/groundingdino_swinb_cogcoor_mmdet-55949c9c.pth'
+    # noqa: E501
+)
+detector['type'] = 'GroundingDINOMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'bdd',
+    public_det_path = 'results/public_dets/bdd_mot_yolox_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024, # Padding the image to multiples of 32
+    ),
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/box_track_20/val/',
+    outfile_prefix='results/detic_masa_trained_bdd_demo',
+metric=['TETA', 'HOTA', 'CLEAR']
+)
+test_evaluator = val_evaluator

configs/masa-gdino/bdd_test/masa_gdino_bdd_mots_test.py

@@ -0,0 +1,227 @@
+_base_ = [
+    '../../../projects/grounding_dino/grounding_dino_swin-b_pretrain_mixeddata_masa.py',
+    '../../datasets/bdd/bdd_dataset.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/groundingdino_swinb_cogcoor_mmdet-55949c9c.pth'
+    # noqa: E501
+)
+detector['type'] = 'GroundingDINOMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    with_segm=True,
+    benchmark = 'bdd',
+    public_det_path = 'results/public_dets/bdd_mots_val_uninext_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024, # Padding the image to multiples of 32
+    ),
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/seg_track_20/val/',
+    outfile_prefix='results/masa_results/masa-groundingdino-release-bdd-mots-test',
+    metric=['TETA', 'HOTA', 'CLEAR'],
+    with_mask=True,
+)
+test_evaluator = val_evaluator

configs/masa-gdino/masa_gdino_swinb_inference.py

@@ -0,0 +1,216 @@
+_base_ = [
+    '../../projects/grounding_dino/grounding_dino_swin-b_pretrain_mixeddata_masa.py',
+    '../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/groundingdino_swinb_cogcoor_mmdet-55949c9c.pth'
+    # noqa: E501
+)
+detector['type'] = 'GroundingDINOMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = False,
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.1,
+        obj_score_thr=0.01,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=100,
+        fps=30,
+        )
+)
+
+inference_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(
+                type='Resize',
+                scale=(1333, 800),
+                keep_ratio=True),
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+

configs/masa-gdino/masa_gdino_swinb_plug_and_play.py

@@ -0,0 +1,218 @@
+_base_ = [
+    '../../projects/grounding_dino/grounding_dino_swin-b_pretrain_mixeddata_masa.py',
+    '../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/groundingdino_swinb_cogcoor_mmdet-55949c9c.pth'
+    # noqa: E501
+)
+detector['type'] = 'GroundingDINOMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = False,
+    given_dets = True,
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.1,
+        obj_score_thr=0.01,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=100,
+        fps=30,
+        )
+)
+
+inference_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(
+                type='Resize',
+                scale=(1333, 800),
+                keep_ratio=True),
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+
+# runtime settings
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+

configs/masa-gdino/open_vocabulary_mot_test/masa_gdino_swinb_open_vocabulary_test.py

@@ -0,0 +1,236 @@
+_base_ = [
+    '../../../projects/grounding_dino/grounding_dino_swin-b_pretrain_mixeddata_masa.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+# detector.backbone.update(dict(out_indices=(1, 2, 3)))
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/groundingdino_swinb_cogcoor_mmdet-55949c9c.pth'
+    # noqa: E501
+)
+detector['type'] = 'GroundingDINOMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024, # Padding the image to multiples of 32
+    ),
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    outfile_prefix='results/masa_results/masa-groundingdino-release-ovmot-test',
+    open_vocabulary=True,
+)

configs/masa-gdino/tao_teta_test/masa_gdino_swinb_tao_test_detic_dets.py

@@ -0,0 +1,235 @@
+_base_ = [
+    '../../../projects/grounding_dino/grounding_dino_swin-b_pretrain_mixeddata_masa.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+# detector.backbone.update(dict(out_indices=(1, 2, 3)))
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/groundingdino_swinb_cogcoor_mmdet-55949c9c.pth'
+    # noqa: E501
+)
+detector['type'] = 'GroundingDINOMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024, # Padding the image to multiples of 32
+    ),
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+outfile_prefix='results/masa_results/masa-groundingdino-release_detic_dets-test',
+)

configs/masa-gdino/tao_teta_test/masa_gdino_swinb_tao_test_teter_swinT_dets.py

@@ -0,0 +1,240 @@
+_base_ = [
+    '../../../projects/grounding_dino/grounding_dino_swin-b_pretrain_mixeddata_masa.py',
+    '../../datasets/tao/tao_dataset_v05.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+# detector.backbone.update(dict(out_indices=(1, 2, 3)))
+detector.pop('data_preprocessor')
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/tsa_models/groundingdino_swinb_cogcoor_mmdet-55949c9c.pth'
+    # noqa: E501
+)
+detector['type'] = 'GroundingDINOMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/teter_swinT_tao_val_internms_50/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024, # Padding the image to multiples of 32
+    ),
+    detector=detector,
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[8, 16, 32, 64, 128]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+# custom hooks
+custom_hooks = [
+    # Synchronize model buffers such as running_mean and running_var in BN
+    # at the end of each epoch
+    dict(type='SyncBuffersHook')
+]
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    outfile_prefix='results/masa_results/masa-groundingdino-release-tao-teter-test',
+)
+test_evaluator = val_evaluator

configs/masa-one/bdd_test/masa_r50_bdd_mot_test.py

@@ -0,0 +1,235 @@
+_base_ = [
+    '../../default_runtime.py',
+    '../../datasets/bdd/bdd_dataset.py',
+]
+default_scope = 'mmdet'
+
+model = dict(
+    type='MASA',
+    unified_backbone=False,
+    load_public_dets = True,
+    use_masa_backbone = True,
+    benchmark='bdd',
+    public_det_path='results/public_dets/bdd_mot_yolox_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='SyncBN', requires_grad=True),
+        norm_eval=True,
+        style='caffe',),
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024, 2048],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared4Conv1FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/box_track_20/val/',
+    outfile_prefix='results/masa_results/masa-r50-release-bdd-mot-test',
+    metric=['TETA', 'HOTA', 'CLEAR']
+)
+test_evaluator = val_evaluator

configs/masa-one/bdd_test/masa_r50_bdd_mots_test.py

@@ -0,0 +1,238 @@
+_base_ = [
+    '../../default_runtime.py',
+    '../../datasets/bdd/bdd_dataset.py',
+]
+default_scope = 'mmdet'
+
+model = dict(
+    type='MASA',
+    unified_backbone=False,
+    load_public_dets = True,
+    use_masa_backbone = True,
+    benchmark='bdd',
+    with_segm=True,
+    public_det_path = 'results/public_dets/bdd_mots_val_uninext_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='SyncBN', requires_grad=True),
+        norm_eval=True,
+        style='caffe',),
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024, 2048],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared4Conv1FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+        pipeline=test_pipeline,
+    )
+)
+
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/seg_track_20/val/',
+    outfile_prefix='results/masa_results/masa-r50-release-bdd-mots-test',
+    metric=['TETA', 'HOTA', 'CLEAR'],
+    with_mask=True,
+)
+test_evaluator = val_evaluator

configs/masa-one/masa_r50_plug_and_play.py

@@ -0,0 +1,214 @@
+_base_ = [
+    '../default_runtime.py'
+]
+default_scope = 'mmdet'
+
+model = dict(
+    type='MASA',
+    unified_backbone=False,
+    load_public_dets = False,
+    use_masa_backbone = True,
+    given_dets = True,
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    # detector=detector,
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='SyncBN', requires_grad=True),
+        norm_eval=True,
+        style='caffe',),
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024, 2048],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared4Conv1FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='QuasiDenseTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.1,
+        obj_score_thr=0.01,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=100,
+        fps=30,
+        )
+)
+
+inference_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+    checkpoint=dict(type='CheckpointHook', interval=12),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/masa-one/open_vocabulary_mot_test/masa_r50_open_vocabulary_test.py

@@ -0,0 +1,231 @@
+_base_ = [
+    '../../default_runtime.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+]
+default_scope = 'mmdet'
+
+model = dict(
+    type='MASA',
+    unified_backbone=False,
+    load_public_dets = True,
+    use_masa_backbone = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='SyncBN', requires_grad=True),
+        norm_eval=True,
+        style='caffe',),
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024, 2048],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared4Conv1FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    outfile_prefix='results/masa_results/masa-r50-release-ovmot-test',
+    open_vocabulary=True,
+)

configs/masa-one/tao_teta_test/masa_r50_tao_test_detic_dets.py

@@ -0,0 +1,230 @@
+_base_ = [
+    '../../default_runtime.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+]
+default_scope = 'mmdet'
+
+model = dict(
+    type='MASA',
+    unified_backbone=False,
+    load_public_dets = True,
+    use_masa_backbone = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='SyncBN', requires_grad=True),
+        norm_eval=True,
+        style='caffe',),
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024, 2048],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared4Conv1FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    outfile_prefix='results/masa_results/masa-r50-release_detic_dets-test',
+)

configs/masa-one/tao_teta_test/masa_r50_tao_test_teter_swinT_dets.py

@@ -0,0 +1,230 @@
+_base_ = [
+    '../../default_runtime.py',
+    '../../datasets/tao/tao_dataset_v05.py',
+]
+default_scope = 'mmdet'
+
+model = dict(
+    type='MASA',
+    unified_backbone=False,
+    load_public_dets = True,
+    use_masa_backbone = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/teter_swinT_tao_val_internms_50/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=True,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=32),  # Padding the image to multiples of 32
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='SyncBN', requires_grad=True),
+        norm_eval=True,
+        style='caffe',),
+    masa_adapter=[
+        dict(
+            type='FPN',
+            in_channels=[256, 512, 1024, 2048],
+            out_channels=256,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            num_outs=5),
+        dict(
+        type='DeformFusion',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared4Conv1FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            norm_cfg=dict(type='SyncBN', requires_grad=True),
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    outfile_prefix='results/masa_results/masa-r50-release-tao-teter-test',
+)

configs/masa-sam/bdd_test/masa_sam_vitb_bdd_mot_test.py

@@ -0,0 +1,245 @@
+_base_ = [
+    '../sam-vitb.py',
+    '../../datasets/bdd/bdd_dataset.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_b_01ec64_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'bdd',
+    public_det_path = 'results/public_dets/bdd_mot_yolox_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[768, 768, 768, 768],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+    checkpoint=dict(type='CheckpointHook', interval=12),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+# custom hooks
+custom_hooks = [
+    # Synchronize model buffers such as running_mean and running_var in BN
+    # at the end of each epoch
+    dict(type='SyncBuffersHook')
+]
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/box_track_20/val/',
+    outfile_prefix='results/masa_results/masa-sam-vitb-bdd-mot-test',
+    metric=['TETA', 'HOTA', 'CLEAR']
+)
+test_evaluator = val_evaluator

configs/masa-sam/bdd_test/masa_sam_vitb_bdd_mots_test.py

@@ -0,0 +1,241 @@
+_base_ = [
+    '../sam-vitb.py',
+    '../../datasets/bdd/bdd_dataset.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_b_01ec64_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    with_segm=True,
+    benchmark = 'bdd',
+    public_det_path = 'results/public_dets/bdd_mots_val_uninext_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[768, 768, 768, 768],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+        pipeline=test_pipeline,
+    )
+)
+
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/seg_track_20/val/',
+    outfile_prefix='results/masa_results/masa-sam-vitb-bdd-mots-test',
+    metric=['TETA', 'HOTA', 'CLEAR'],
+    with_mask=True,
+)
+test_evaluator = val_evaluator

configs/masa-sam/bdd_test/masa_sam_vith_bdd_mot_test.py

@@ -0,0 +1,246 @@
+_base_ = [
+    '../sam-vith.py',
+    '../../datasets/bdd/bdd_dataset.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_h_4b8939_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'bdd',
+    public_det_path = 'results/public_dets/bdd_mot_yolox_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[1280, 1280, 1280, 1280],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+    checkpoint=dict(type='CheckpointHook', interval=12),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+# custom hooks
+custom_hooks = [
+    # Synchronize model buffers such as running_mean and running_var in BN
+    # at the end of each epoch
+    dict(type='SyncBuffersHook')
+]
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/bdd/annotations/box_track_20/box_track_val_cocofmt.json',
+    scalabel_gt='data/bdd/annotations/scalabel_gt/box_track_20/val/',
+    outfile_prefix='results/masa_results/masa-sam-vith-bdd-mot-test',
+metric=['TETA', 'HOTA', 'CLEAR']
+)
+test_evaluator = val_evaluator

configs/masa-sam/bdd_test/masa_sam_vith_bdd_mots_test.py

@@ -0,0 +1,240 @@
+_base_ = [
+    '../sam-vith.py',
+    '../../datasets/bdd/bdd_dataset.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_h_4b8939_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    with_segm=True,
+    benchmark = 'bdd',
+    public_det_path = 'results/public_dets/bdd_mots_val_uninext_dets/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[1280, 1280, 1280, 1280],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaBDDTracker',
+        init_score_thr=0.5,
+        obj_score_thr=0.3,
+        match_score_thr=0.6,
+        memo_tracklet_frames=10,
+        memo_backdrop_frames=1,
+        memo_momentum=0.8,
+        nms_conf_thr=0.5,
+        nms_backdrop_iou_thr=0.3,
+        nms_class_iou_thr=0.7,
+        with_cats=False,
+        match_metric='bisoftmax')
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False),
+checkpoint = dict(type='CheckpointHook', interval=1),
+)
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/bdd/annotations/seg_track_val_cocofmt.json',
+        pipeline=test_pipeline,
+    )
+)
+
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    outfile_prefix='results/masa_results/masa-sam-vith-bdd-mots-test',
+    metric=['TETA'],
+    with_mask=True,
+)
+test_evaluator = val_evaluator

configs/masa-sam/open_vocabulary_mot_test/masa_sam_vitb_open_vocabulary_test.py

@@ -0,0 +1,233 @@
+_base_ = [
+    '../sam-vitb.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_b_01ec64_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[768, 768, 768, 768],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    outfile_prefix='results/masa_results/masa-sam-b-release-ovmot-test',
+    open_vocabulary=True,
+)

configs/masa-sam/open_vocabulary_mot_test/masa_sam_vith_open_vocabulary_test.py

@@ -0,0 +1,234 @@
+_base_ = [
+    '../sam-vith.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_h_4b8939_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[1280, 1280, 1280, 1280],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.8,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    outfile_prefix='results/masa_results/masa-sam-h-release-ovmot-test',
+    open_vocabulary=True,
+)

configs/masa-sam/sam-vitb.py

@@ -0,0 +1,30 @@
+prompt_embed_dim=256
+model = dict(
+    type='SamMasa',
+    backbone=dict(
+        type='ImageEncoderViT',
+        depth=12,
+        embed_dim=768,
+        img_size=1024,
+        mlp_ratio=4,
+        num_heads=12,
+        patch_size=16,
+        qkv_bias=True,
+        use_rel_pos=True,
+        global_attn_indexes=[2, 5, 8, 11],
+        window_size=14,
+        out_chans=prompt_embed_dim,
+        out_indices=[2, 5, 8, 11]),
+    mask_decoder=dict(
+        type='MaskDecoder',
+        num_multimask_outputs=3,
+        transformer_dim=prompt_embed_dim,
+        iou_head_depth=3,
+        iou_head_hidden_dim=256),
+    prompt_encoder=dict(
+        type='PromptEncoder',
+        embed_dim=prompt_embed_dim,
+        image_embedding_size=(64, 64),
+        input_image_size=(1024, 1024),
+        mask_in_chans=16),
+)

configs/masa-sam/sam-vith.py

@@ -0,0 +1,30 @@
+prompt_embed_dim=256
+model = dict(
+    type='SamMasa',
+    backbone=dict(
+        type='ImageEncoderViT',
+        depth=32,
+        embed_dim=1280,
+        img_size=1024,
+        mlp_ratio=4,
+        num_heads=16,
+        patch_size=16,
+        qkv_bias=True,
+        use_rel_pos=True,
+        global_attn_indexes=[7, 15, 23, 31],
+        window_size=14,
+        out_chans=prompt_embed_dim,
+        out_indices=[7, 15, 23, 31]),
+    mask_decoder=dict(
+        type='MaskDecoder',
+        num_multimask_outputs=3,
+        transformer_dim=prompt_embed_dim,
+        iou_head_depth=3,
+        iou_head_hidden_dim=256),
+    prompt_encoder=dict(
+        type='PromptEncoder',
+        embed_dim=prompt_embed_dim,
+        image_embedding_size=(64, 64),
+        input_image_size=(1024, 1024),
+        mask_in_chans=16),
+)

configs/masa-sam/tao_teta_test/masa_sam_vitb_tao_test_detic_dets.py

@@ -0,0 +1,232 @@
+_base_ = [
+    '../sam-vitb.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_b_01ec64_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[768, 768, 768, 768],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+# runtime settings
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+outfile_prefix='results/masa_results/masa-sam-vitb-tao-test-detic-dets',
+)

configs/masa-sam/tao_teta_test/masa_sam_vitb_tao_test_teter_swinT_dets.py

@@ -0,0 +1,238 @@
+_base_ = [
+    '../sam-vitb.py',
+    '../../datasets/tao/tao_dataset_v05.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_b_01ec64_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/teter_swinT_tao_val_internms_50/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[768, 768, 768, 768],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.5,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+# custom hooks
+custom_hooks = [
+    # Synchronize model buffers such as running_mean and running_var in BN
+    # at the end of each epoch
+    dict(type='SyncBuffersHook')
+]
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+outfile_prefix='results/masa_results/masa-sam-vitb-tao-test-teter-swinT-dets',
+)
+test_evaluator = val_evaluator

configs/masa-sam/tao_teta_test/masa_sam_vith_tao_test_detic_dets.py

@@ -0,0 +1,233 @@
+_base_ = [
+    '../sam-vith.py',
+    '../../datasets/tao/tao_dataset_v1.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_h_4b8939_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    benchmark = 'tao',
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/detic_tao_val_det/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[1280, 1280, 1280, 1280],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.8,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+        pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+test_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v1_classes.json',
+    outfile_prefix='results/masa_results/masa-sam-vith-tao-test-detic-dets',
+)

configs/masa-sam/tao_teta_test/masa_sam_vith_tao_test_teter_swinT_dets.py

@@ -0,0 +1,239 @@
+_base_ = [
+    '../sam-vith.py',
+    '../../datasets/tao/tao_dataset_v05.py',
+    '../../default_runtime.py'
+]
+default_scope = 'mmdet'
+detector = _base_.model
+detector['init_cfg'] = dict(
+    type='Pretrained',
+    checkpoint= 'saved_models/pretrain_weights/sam_vit_h_4b8939_mmdet.pth'
+    # noqa: E501
+)
+detector['type'] = 'SamMasa'
+
+del _base_.model
+
+model = dict(
+    type='MASA',
+    freeze_detector=True,
+    unified_backbone=True,
+    load_public_dets = True,
+    public_det_path = 'results/public_dets/tao_val_dets/teta_50_internms/teter_swinT_tao_val_internms_50/',
+    data_preprocessor=dict(
+        type='TrackDataPreprocessor',
+        # Image normalization parameters
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        # Image padding parameters
+        pad_mask=False,  # In instance segmentation, the mask needs to be padded
+        pad_size_divisor=1024),  # Padding the image to multiples of 32
+    detector=detector,
+    masa_adapter=[
+        dict(
+        type='SimpleFPN',
+        in_channels=[1280, 1280, 1280, 1280],
+        out_channels=256,
+        use_residual=True,
+        num_outs=5),
+        dict(
+        type='DyHead',
+         in_channels=256,
+        out_channels=256,
+        num_blocks=3)
+    ],
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)
+    ),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=1,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=True,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.02,
+            # nms=dict(type='nms', iou_threshold=0.5),
+            nms=dict(type='nms',
+                     iou_threshold=0.5,
+                     class_agnostic=True,
+                     split_thr=100000),
+            max_per_img=50,
+            mask_thr_binary=0.5)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ),
+    track_head=dict(
+        type='MasaTrackHead',
+        roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        embed_head=dict(
+            type='QuasiDenseEmbedHead',
+            num_convs=4,
+            num_fcs=1,
+            embed_channels=256,
+            norm_cfg=dict(type='GN', num_groups=32),
+            loss_track=dict(type='UnbiasedContrastLoss', loss_weight=0.25),
+            loss_track_aux=dict(
+                type='MarginL2Loss',
+                neg_pos_ub=3,
+                pos_margin=0,
+                neg_margin=0.1,
+                hard_mining=True,
+                loss_weight=1.0)),
+        # loss_bbox=dict(type='L1Loss', loss_weight=1.0),
+        train_cfg=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='CombinedSampler',
+                num=512,
+                pos_fraction=0.8,
+                neg_pos_ub=3,
+                add_gt_as_proposals=True,
+                pos_sampler=dict(type='InstanceBalancedPosSampler'),
+                neg_sampler=dict(type='RandomSampler')))),
+    tracker=dict(
+        type='MasaTaoTracker',
+        init_score_thr=0.0001,
+        obj_score_thr=0.0001,
+        match_score_thr=0.5,
+        memo_tracklet_frames=10,
+        memo_momentum=0.8,
+        with_cats=False,
+        max_distance=-1,
+        fps=1,
+        )
+)
+
+test_pipeline = [
+    dict(
+        type='TransformBroadcaster',
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(
+                type='Resize',
+                scale=(1024, 1024),
+                keep_ratio=True),
+            dict(type='LoadTrackAnnotations')
+        ]),
+    dict(type='PackTrackInputs')
+]
+
+
+train_dataloader = None
+train_cfg = None
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+default_hooks = dict(
+    logger=dict(type='LoggerHook', interval=50),
+    visualization=dict(type='TrackVisualizationHook', draw=False))
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='MasaTrackLocalVisualizer', vis_backends=vis_backends, name='visualizer')
+
+# custom hooks
+custom_hooks = [
+    # Synchronize model buffers such as running_mean and running_var in BN
+    # at the end of each epoch
+    dict(type='SyncBuffersHook')
+]
+auto_scale_lr = dict(enable=False, base_batch_size=16)
+val_dataloader = dict(
+    dataset=dict(
+        ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    pipeline=test_pipeline,
+    )
+)
+test_dataloader = val_dataloader
+val_evaluator = dict(
+    ann_file='data/tao/annotations/tao_val_lvis_v05_classes.json',
+    outfile_prefix='results/masa_results/masa-sam-vith-tao-test-teter-swinT-dets',
+)
+test_evaluator = val_evaluator

environment_docker.yml

@@ -0,0 +1,302 @@
+name: masaenv
+channels:
+  - pytorch
+  - nvidia
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - aom=3.9.1=hac33072_0
+  - blas=1.0=mkl
+  - brotli-python=1.0.9=py311h6a678d5_8
+  - bzip2=1.0.8=h5eee18b_6
+  - ca-certificates=2024.6.2=hbcca054_0
+  - cairo=1.18.0=h3faef2a_0
+  - certifi=2024.6.2=pyhd8ed1ab_0
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - cuda-cudart=11.8.89=0
+  - cuda-cupti=11.8.87=0
+  - cuda-libraries=11.8.0=0
+  - cuda-nvrtc=11.8.89=0
+  - cuda-nvtx=11.8.86=0
+  - cuda-runtime=11.8.0=0
+  - cudatoolkit=11.8.0=h6a678d5_0
+  - dav1d=1.2.1=hd590300_0
+  - expat=2.6.2=h59595ed_0
+  - ffmpeg=7.0.1=gpl_hb399a10_100
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_2
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - freetype=2.12.1=h4a9f257_0
+  - fribidi=1.0.10=h36c2ea0_0
+  - gmp=6.3.0=h59595ed_1
+  - gmpy2=2.1.2=py311hc9b5ff0_0
+  - gnutls=3.7.9=hb077bed_0
+  - graphite2=1.3.13=h59595ed_1003
+  - harfbuzz=8.5.0=hfac3d4d_0
+  - icu=73.2=h59595ed_0
+  - idna=3.7=py311h06a4308_0
+  - intel-openmp=2023.1.0=hdb19cb5_46306
+  - jinja2=3.1.4=py311h06a4308_0
+  - jpeg=9e=h5eee18b_1
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libabseil=20240116.2=cxx17_h59595ed_0
+  - libass=0.17.1=h8fe9dca_1
+  - libcublas=11.11.3.6=0
+  - libcufft=10.9.0.58=0
+  - libcufile=1.9.1.3=0
+  - libcurand=10.3.5.147=0
+  - libcusolver=11.4.1.48=0
+  - libcusparse=11.7.5.86=0
+  - libdeflate=1.17=h5eee18b_1
+  - libdrm=2.4.120=hd590300_0
+  - libexpat=2.6.2=h59595ed_0
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=13.2.0=h77fa898_10
+  - libglib=2.80.2=hf974151_0
+  - libgomp=13.2.0=h77fa898_10
+  - libhwloc=2.10.0=default_h5622ce7_1001
+  - libiconv=1.17=hd590300_2
+  - libidn2=2.3.4=h5eee18b_0
+  - libjpeg-turbo=2.0.0=h9bf148f_0
+  - libnpp=11.8.0.86=0
+  - libnsl=2.0.1=hd590300_0
+  - libnvjpeg=11.9.0.86=0
+  - libopenvino=2024.1.0=h2da1b83_7
+  - libopenvino-auto-batch-plugin=2024.1.0=hb045406_7
+  - libopenvino-auto-plugin=2024.1.0=hb045406_7
+  - libopenvino-hetero-plugin=2024.1.0=h5c03a75_7
+  - libopenvino-intel-cpu-plugin=2024.1.0=h2da1b83_7
+  - libopenvino-intel-gpu-plugin=2024.1.0=h2da1b83_7
+  - libopenvino-intel-npu-plugin=2024.1.0=he02047a_7
+  - libopenvino-ir-frontend=2024.1.0=h5c03a75_7
+  - libopenvino-onnx-frontend=2024.1.0=h07e8aee_7
+  - libopenvino-paddle-frontend=2024.1.0=h07e8aee_7
+  - libopenvino-pytorch-frontend=2024.1.0=he02047a_7
+  - libopenvino-tensorflow-frontend=2024.1.0=h39126c6_7
+  - libopenvino-tensorflow-lite-frontend=2024.1.0=he02047a_7
+  - libopus=1.3.1=h7f98852_1
+  - libpciaccess=0.18=hd590300_0
+  - libpng=1.6.39=h5eee18b_0
+  - libprotobuf=4.25.3=h08a7969_0
+  - libsqlite=3.46.0=hde9e2c9_0
+  - libstdcxx-ng=13.2.0=hc0a3c3a_10
+  - libtasn1=4.19.0=h5eee18b_0
+  - libtiff=4.5.1=h6a678d5_0
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libva=2.21.0=h4ab18f5_2
+  - libvpx=1.14.1=hac33072_0
+  - libwebp-base=1.3.2=h5eee18b_0
+  - libxcb=1.15=h0b41bf4_0
+  - libxcrypt=4.4.36=hd590300_1
+  - libxml2=2.12.7=hc051c1a_1
+  - libzlib=1.2.13=h4ab18f5_6
+  - llvm-openmp=14.0.6=h9e868ea_0
+  - lz4-c=1.9.4=h6a678d5_1
+  - markupsafe=2.1.3=py311h5eee18b_0
+  - mkl=2023.1.0=h213fc3f_46344
+  - mkl-service=2.4.0=py311h5eee18b_1
+  - mkl_fft=1.3.8=py311h5eee18b_0
+  - mkl_random=1.2.4=py311hdb19cb5_0
+  - mpc=1.1.0=h10f8cd9_1
+  - mpfr=4.0.2=hb69a4c5_1
+  - mpmath=1.3.0=py311h06a4308_0
+  - ncurses=6.4=h6a678d5_0
+  - nettle=3.9.1=h7ab15ed_0
+  - networkx=3.2.1=py311h06a4308_0
+  - numpy=1.26.4=py311h08b1b3b_0
+  - numpy-base=1.26.4=py311hf175353_0
+  - ocl-icd=2.3.2=hd590300_1
+  - openh264=2.4.1=h59595ed_0
+  - openjpeg=2.4.0=h3ad879b_0
+  - openssl=3.3.1=h4ab18f5_0
+  - p11-kit=0.24.1=hc5aa10d_0
+  - pcre2=10.43=hcad00b1_0
+  - pillow=10.3.0=py311h5eee18b_0
+  - pip=24.0=py311h06a4308_0
+  - pixman=0.43.2=h59595ed_0
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - pugixml=1.14=h59595ed_0
+  - pysocks=1.7.1=py311h06a4308_0
+  - python=3.11.8=hab00c5b_0_cpython
+  - pytorch=2.1.2=py3.11_cuda11.8_cudnn8.7.0_0
+  - pytorch-cuda=11.8=h7e8668a_5
+  - pytorch-mutex=1.0=cuda
+  - pyyaml=6.0.1=py311h5eee18b_0
+  - readline=8.2=h5eee18b_0
+  - snappy=1.2.0=hdb0a2a9_1
+  - sqlite=3.45.3=h5eee18b_0
+  - svt-av1=2.1.0=hac33072_0
+  - sympy=1.12=py311h06a4308_0
+  - tbb=2021.12.0=h297d8ca_1
+  - tk=8.6.14=h39e8969_0
+  - torchaudio=2.1.2=py311_cu118
+  - torchtriton=2.1.0=py311
+  - torchvision=0.16.2=py311_cu118
+  - typing_extensions=4.11.0=py311h06a4308_0
+  - wheel=0.43.0=py311h06a4308_0
+  - x264=1!164.3095=h166bdaf_2
+  - x265=3.5=h924138e_3
+  - xorg-fixesproto=5.0=h7f98852_1002
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.9=h8ee46fc_0
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxfixes=5.0.3=h7f98852_1004
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xz=5.4.6=h5eee18b_1
+  - yaml=0.2.5=h7b6447c_0
+  - zlib=1.2.13=h4ab18f5_6
+  - zstd=1.5.5=hc292b87_2
+  - pip:
+      - addict==2.4.0
+      - aiofiles==23.2.1
+      - aliyun-python-sdk-core==2.15.1
+      - aliyun-python-sdk-kms==2.16.3
+      - altair==5.3.0
+      - annotated-types==0.7.0
+      - anyio==4.4.0
+      - attrs==23.2.0
+      - boto3==1.34.128
+      - botocore==1.34.128
+      - cffi==1.16.0
+      - click==8.1.7
+      - clip==1.0
+      - colorama==0.4.6
+      - contourpy==1.2.1
+      - crcmod==1.7
+      - cryptography==42.0.8
+      - cycler==0.12.1
+      - cython==3.0.10
+      - decorator==4.4.2
+      - defusedxml==0.7.1
+      - dnspython==2.6.1
+      - einops==0.8.0
+      - email-validator==2.1.2
+      - fairscale==0.4.13
+      - fastapi==0.111.0
+      - fastapi-cli==0.0.4
+      - ffmpy==0.3.2
+      - filelock==3.14.0
+      - fonttools==4.53.0
+      - fsspec==2024.6.0
+      - ftfy==6.2.0
+      - gradio==4.36.1
+      - gradio-client==1.0.1
+      - h11==0.14.0
+      - h5py==3.11.0
+      - httpcore==1.0.5
+      - httptools==0.6.1
+      - httpx==0.27.0
+      - huggingface-hub==0.23.4
+      - imageio==2.34.1
+      - importlib-metadata==7.1.0
+      - importlib-resources==6.4.0
+      - jmespath==0.10.0
+      - joblib==1.4.2
+      - jsonschema==4.22.0
+      - jsonschema-specifications==2023.12.1
+      - kiwisolver==1.4.5
+      - llvmlite==0.43.0
+      - lvis==0.5.3
+      - markdown==3.6
+      - markdown-it-py==3.0.0
+      - matplotlib==3.9.0
+      - mdurl==0.1.2
+      - mmcv==2.1.0
+      - mmdet==3.3.0
+      - mmengine==0.10.4
+      - model-index==0.1.11
+      - motmetrics==1.4.0
+      - moviepy==0.2.3.5
+      - nanoid==2.0.0
+      - natsort==8.4.0
+      - nltk==3.8.1
+      - numba==0.60.0
+      - opencv-python==4.10.0.84
+      - opencv-python-headless==4.10.0.84
+      - opendatalab==0.0.10
+      - openmim==0.3.9
+      - openxlab==0.1.0
+      - ordered-set==4.1.0
+      - orjson==3.10.5
+      - oss2==2.17.0
+      - packaging==24.1
+      - pandas==2.2.2
+      - platformdirs==4.2.2
+      - plyfile==1.0.3
+      - psutil==5.9.8
+      - pycocotools==2.0.8
+      - pycparser==2.22
+      - pycryptodome==3.20.0
+      - pydantic==2.7.4
+      - pydantic-core==2.18.4
+      - pydub==0.25.1
+      - pygments==2.18.0
+      - pyparsing==3.1.2
+      - python-dateutil==2.9.0.post0
+      - python-dotenv==1.0.1
+      - python-multipart==0.0.9
+      - pytz==2023.4
+      - referencing==0.35.1
+      - regex==2024.5.15
+      - requests==2.32.3
+      - rich==13.4.2
+      - rpds-py==0.18.1
+      - ruff==0.4.9
+      - s3transfer==0.10.1
+      - safetensors==0.4.3
+      - scalabel==0.3.0
+      - scipy==1.13.1
+      - script-utils==0.0.1
+      - seaborn==0.13.2
+      - semantic-version==2.10.0
+      - setuptools==60.2.0
+      - shapely==2.0.4
+      - shellingham==1.5.4
+      - six==1.16.0
+      - sniffio==1.3.1
+      - starlette==0.37.2
+      - supervision==0.21.0
+      - tabulate==0.9.0
+      - tao==0.1.0
+      - termcolor==2.4.0
+      - terminaltables==3.1.10
+      - teta==0.1.0
+      - tokenizers==0.15.2
+      - toml==0.10.2
+      - tomli==2.0.1
+      - tomlkit==0.12.0
+      - toolz==0.12.1
+      - tqdm==4.65.2
+      - trackeval==1.0.dev1
+      - transformers==4.38.2
+      - typer==0.12.3
+      - tzdata==2024.1
+      - ujson==5.10.0
+      - urllib3==2.2.2
+      - uvicorn==0.30.1
+      - uvloop==0.19.0
+      - watchfiles==0.22.0
+      - wcwidth==0.2.13
+      - websockets==11.0.3
+      - xmltodict==0.13.0
+      - yacs==0.1.8
+      - yapf==0.40.2
+      - youtube-dl==2021.12.17
+      - zipp==3.19.2

masa/__init__.py

@@ -0,0 +1,3 @@
+from .datasets import *  # noqa
+from .models import *  # noqa
+from .visualization import *  # noqa

masa/apis/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .masa_inference import (build_test_pipeline, inference_detector,
+                             inference_masa, init_masa)
+
+__all__ = [
+    "inference_masa",
+    "init_masa",
+    "inference_detector",
+    "build_test_pipeline",
+]

masa/apis/masa_inference.py

@@ -0,0 +1,297 @@
+import copy
+import time
+import warnings
+from pathlib import Path
+from typing import Optional, Sequence, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+from mmcv.ops import RoIPool
+from mmcv.transforms import Compose
+from mmdet.evaluation import get_classes
+from mmdet.registry import MODELS
+from mmdet.structures import DetDataSample, SampleList
+from mmdet.utils import ConfigType, get_test_pipeline_cfg
+from mmengine.config import Config
+from mmengine.dataset import default_collate
+from mmengine.model.utils import revert_sync_batchnorm
+from mmengine.registry import init_default_scope
+from mmengine.runner import autocast, load_checkpoint
+
+ImagesType = Union[str, np.ndarray, Sequence[str], Sequence[np.ndarray]]
+
+
+def init_masa(
+    config: Union[str, Path, Config],
+    checkpoint: Optional[str] = None,
+    palette: str = "none",
+    device: str = "cuda:0",
+    cfg_options: Optional[dict] = None,
+) -> nn.Module:
+    """Initialize a unified masa detector from config file.
+
+    Args:
+        config (str, :obj:`Path`, or :obj:`mmengine.Config`): Config file path,
+            :obj:`Path`, or the config object.
+        checkpoint (str, optional): Checkpoint path. If left as None, the model
+            will not load any weights.
+        palette (str): Color palette used for visualization. If palette
+            is stored in checkpoint, use checkpoint's palette first, otherwise
+            use externally passed palette. Currently, supports 'coco', 'voc',
+            'citys' and 'random'. Defaults to none.
+        device (str): The device where the anchors will be put on.
+            Defaults to cuda:0.
+        cfg_options (dict, optional): Options to override some settings in
+            the used config.
+
+    Returns:
+        nn.Module: The constructed detector.
+    """
+    if isinstance(config, (str, Path)):
+        config = Config.fromfile(config)
+    elif not isinstance(config, Config):
+        raise TypeError(
+            "config must be a filename or Config object, " f"but got {type(config)}"
+        )
+
+    with_backbone = config.model.get("backbone", False)
+    if with_backbone:
+        if cfg_options is not None:
+            config.merge_from_dict(cfg_options)
+        elif "init_cfg" in config.model.backbone:
+            config.model.backbone.init_cfg = None
+    else:
+        if cfg_options is not None:
+            config.merge_from_dict(cfg_options)
+        elif "init_cfg" in config.model.detector.backbone:
+            config.model.detector.backbone.init_cfg = None
+
+    scope = config.get("default_scope", "mmdet")
+    if scope is not None:
+        init_default_scope(config.get("default_scope", "mmdet"))
+
+    model = MODELS.build(config.model)
+    model = revert_sync_batchnorm(model)
+    if checkpoint is None:
+        warnings.simplefilter("once")
+        warnings.warn("checkpoint is None, use COCO classes by default.")
+        model.dataset_meta = {"classes": get_classes("coco")}
+    else:
+        checkpoint = load_checkpoint(model, checkpoint, map_location="cpu")
+        # Weights converted from elsewhere may not have meta fields.
+        checkpoint_meta = checkpoint.get("meta", {})
+
+        # save the dataset_meta in the model for convenience
+        if "dataset_meta" in checkpoint_meta:
+            # mmdet 3.x, all keys should be lowercase
+            model.dataset_meta = {
+                k.lower(): v for k, v in checkpoint_meta["dataset_meta"].items()
+            }
+        elif "CLASSES" in checkpoint_meta:
+            # < mmdet 3.x
+            classes = checkpoint_meta["CLASSES"]
+            model.dataset_meta = {"classes": classes}
+        else:
+            warnings.simplefilter("once")
+            warnings.warn(
+                "dataset_meta or class names are not saved in the "
+                "checkpoint's meta data, use COCO classes by default."
+            )
+            model.dataset_meta = {"classes": get_classes("coco")}
+
+    # Priority:  args.palette -> config -> checkpoint
+    if palette != "none":
+        model.dataset_meta["palette"] = palette
+    else:
+        if "palette" not in model.dataset_meta:
+            warnings.warn(
+                "palette does not exist, random is used by default. "
+                "You can also set the palette to customize."
+            )
+            model.dataset_meta["palette"] = "random"
+
+    model.cfg = config  # save the config in the model for convenience
+    model.to(device)
+    model.eval()
+    return model
+
+
+def inference_detector(
+    model: nn.Module,
+    imgs: ImagesType,
+    test_pipeline: Optional[Compose] = None,
+    text_prompt: Optional[str] = None,
+    custom_entities: bool = False,
+    fp16: bool = False,
+) -> Union[DetDataSample, SampleList]:
+    """Inference image(s) with the detector.
+
+    Args:
+        model (nn.Module): The loaded detector.
+        imgs (str, ndarray, Sequence[str/ndarray]):
+           Either image files or loaded images.
+        test_pipeline (:obj:`Compose`): Test pipeline.
+
+    Returns:
+        :obj:`DetDataSample` or list[:obj:`DetDataSample`]:
+        If imgs is a list or tuple, the same length list type results
+        will be returned, otherwise return the detection results directly.
+    """
+
+    if isinstance(imgs, (list, tuple)):
+        is_batch = True
+    else:
+        imgs = [imgs]
+        is_batch = False
+
+    cfg = model.cfg
+
+    if test_pipeline is None:
+        cfg = cfg.copy()
+        test_pipeline = get_test_pipeline_cfg(cfg)
+        if isinstance(imgs[0], np.ndarray):
+            # Calling this method across libraries will result
+            # in module unregistered error if not prefixed with mmdet.
+            test_pipeline[0].type = "mmdet.LoadImageFromNDArray"
+
+        test_pipeline = Compose(test_pipeline)
+
+    if model.data_preprocessor.device.type == "cpu":
+        for m in model.modules():
+            assert not isinstance(
+                m, RoIPool
+            ), "CPU inference with RoIPool is not supported currently."
+
+    result_list = []
+    for i, img in enumerate(imgs):
+        # prepare data
+        if isinstance(img, np.ndarray):
+            # TODO: remove img_id.
+            data_ = dict(img=img, img_id=0)
+        else:
+            # TODO: remove img_id.
+            data_ = dict(img_path=img, img_id=0)
+
+        if text_prompt:
+            data_["text"] = text_prompt
+            data_["custom_entities"] = custom_entities
+
+        # build the data pipeline
+        data_ = test_pipeline(data_)
+
+        data_["inputs"] = [data_["inputs"]]
+        data_["data_samples"] = [data_["data_samples"]]
+
+        # forward the model
+        with torch.no_grad():
+            with autocast(enabled=fp16):
+                results = model.test_step(data_)[0]
+
+        result_list.append(results)
+
+    if not is_batch:
+        return result_list[0]
+    else:
+        return result_list
+
+
+def inference_masa(
+    model: nn.Module,
+    img: np.ndarray,
+    frame_id: int,
+    video_len: int,
+    test_pipeline: Optional[Compose] = None,
+    text_prompt=None,
+    custom_entities: bool = False,
+    det_bboxes=None,
+    det_labels=None,
+    fp16=False,
+    detector_type="mmdet",
+    show_fps=False,
+) -> SampleList:
+    """Inference image(s) with the masa model.
+
+    Args:
+        model (nn.Module): The loaded mot model.
+        img (np.ndarray): Loaded image.
+        frame_id (int): frame id.
+        video_len (int): demo video length
+    Returns:
+        SampleList: The tracking data samples.
+    """
+    data = dict(
+        img=[img.astype(np.float32)],
+        # img=[img.astype(np.uint8)],
+        frame_id=[frame_id],
+        ori_shape=[img.shape[:2]],
+        img_id=[frame_id + 1],
+        ori_video_length=[video_len],
+    )
+
+    if text_prompt is not None:
+        if detector_type == "mmdet":
+            data["text"] = [text_prompt]
+            data["custom_entities"] = [custom_entities]
+        elif detector_type == "yolo-world":
+            data["texts"] = [text_prompt]
+            data["custom_entities"] = [custom_entities]
+
+    data = test_pipeline(data)
+
+    # forward the model
+    with torch.no_grad():
+        data = default_collate([data])
+        if det_bboxes is not None:
+            data["data_samples"][0].video_data_samples[0].det_bboxes = det_bboxes
+            data["data_samples"][0].video_data_samples[0].det_labels = det_labels
+        # measure FPS ##
+        if show_fps:
+            start = time.time()
+            with autocast(enabled=fp16):
+                result = model.test_step(data)[0]
+            end = time.time()
+            fps = 1 / (end - start)
+            return result, fps
+
+        else:
+            with autocast(enabled=fp16):
+                result = model.test_step(data)[0]
+            return result
+
+
+def build_test_pipeline(
+    cfg: ConfigType, with_text=False, detector_type="mmdet"
+) -> ConfigType:
+    """Build test_pipeline for mot/vis demo. In mot/vis infer, original
+    test_pipeline should remove the "LoadImageFromFile" and
+    "LoadTrackAnnotations".
+
+    Args:
+         cfg (ConfigDict): The loaded config.
+    Returns:
+         ConfigType: new test_pipeline
+    """
+    # remove the "LoadImageFromFile" and "LoadTrackAnnotations" in pipeline
+    transform_broadcaster = cfg.inference_pipeline[0].copy()
+    if detector_type == "yolo-world":
+        kept_transform = []
+        for transform in transform_broadcaster["transforms"]:
+            if (
+                transform["type"] == "mmyolo.YOLOv5KeepRatioResize"
+                or transform["type"] == "mmyolo.LetterResize"
+            ):
+                kept_transform.append(transform)
+        transform_broadcaster["transforms"] = kept_transform
+        pack_track_inputs = cfg.test_dataloader.dataset.pipeline[-1].copy()
+        test_pipeline = Compose([transform_broadcaster, pack_track_inputs])
+    else:
+        for transform in transform_broadcaster["transforms"]:
+            if "Resize" in transform["type"]:
+                transform_broadcaster["transforms"] = transform
+        pack_track_inputs = cfg.inference_pipeline[-1].copy()
+        if with_text:
+            pack_track_inputs["meta_keys"] = ("text", "custom_entities")
+        test_pipeline = Compose([transform_broadcaster, pack_track_inputs])
+
+    return test_pipeline

masa/datasets/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) Tencent Inc. All rights reserved.
+from .bdd_masa_dataset import BDDVideoDataset
+from .dataset_wrappers import SeqMultiImageMixDataset
+from .evaluation import *  # NOQA
+from .masa_dataset import MASADataset
+from .pipelines import *  # NOQA
+from .rsconcat_dataset import RandomSampleConcatDataset
+from .tao_masa_dataset import Taov1Dataset, Taov05Dataset
+from .utils import yolow_collate
+
+__all__ = [
+    "yolow_collate",
+    "RandomSampleConcatDataset",
+    "MASADataset",
+    "SeqMultiImageMixDataset",
+    "Taov05Dataset",
+    "Taov1Dataset",
+    "BDDVideoDataset",
+]