Duplicate from bodhicitta/sam3

Co-authored-by: Hu <bodhicitta@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,51 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+ 
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+ 
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.gguf* filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.llamafile* filter=lfs diff=lfs merge=lfs -text
+*.pt2 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+sam3.pt filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,61 @@
+SAM License
+Last Updated: November 19, 2025
+
+“Agreement” means the terms and conditions for use, reproduction, distribution and modification of the SAM Materials set forth herein.
+
+
+“SAM Materials” means, collectively, Documentation and the models, software and algorithms, including machine-learning model code, trained model weights, inference-enabling code, training-enabling code, fine-tuning enabling code, and other elements of the foregoing distributed by Meta and made available under this Agreement.
+
+“Documentation” means the specifications, manuals and documentation accompanying
+SAM Materials distributed by Meta.
+
+
+“Licensee” or “you” means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity’s behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf.
+
+
+“Meta” or “we” means Meta Platforms Ireland Limited (if you are located in or, if you are an entity, your principal place of business is in the EEA or Switzerland) or Meta Platforms, Inc. (if you are located outside of the EEA or Switzerland).
+
+
+“Sanctions” means any economic or trade sanctions or restrictions administered or enforced by the United States (including the Office of Foreign Assets Control of the U.S. Department of the Treasury (“OFAC”), the U.S. Department of State and the U.S. Department of Commerce), the United Nations, the European Union, or the United Kingdom.
+
+
+“Trade Controls” means any of the following: Sanctions and applicable export and import controls.
+
+By using or distributing any portion or element of the SAM Materials, you agree to be bound by this Agreement.
+
+
+1. License Rights and Redistribution.
+
+
+a. Grant of Rights. You are granted a non-exclusive, worldwide, non-transferable and royalty-free limited license under Meta’s intellectual property or other rights owned by Meta embodied in the SAM Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the SAM Materials.
+
+b. Redistribution and Use.
+i. Distribution of SAM Materials, and any derivative works thereof, are subject to the terms of this Agreement. If you distribute or make the SAM Materials, or any derivative works thereof, available to a third party, you may only do so under the terms of this Agreement and you shall provide a copy of this Agreement with any such SAM Materials.
+
+
+ii.  If you submit for publication the results of research you perform on, using, or otherwise in connection with SAM Materials, you must acknowledge the use of SAM Materials in your publication.
+
+
+iii. Your use of the SAM Materials must comply with applicable laws and regulations, including Trade Control Laws and applicable privacy and data protection laws.
+iv. Your use of the SAM Materials will not involve or encourage others to reverse engineer, decompile or discover the underlying components of the SAM Materials.
+v. You are not the target of Trade Controls and your use of SAM Materials must comply with Trade Controls. You agree not to use, or permit others to use, SAM Materials for any activities subject to the International Traffic in Arms Regulations (ITAR) or end uses prohibited by Trade Controls, including those related to military or warfare purposes, nuclear industries or applications, espionage, or the development or use of guns or illegal weapons.
+2. User Support. Your use of the SAM Materials is done at your own discretion; Meta does not process any information nor provide any service in relation to such use.  Meta is under no obligation to provide any support services for the SAM Materials. Any support provided is “as is”, “with all faults”, and without warranty of any kind.
+
+
+3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE SAM MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, AND META DISCLAIMS ALL WARRANTIES OF ANY KIND, BOTH EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE SAM MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE SAM MATERIALS AND ANY OUTPUT AND RESULTS.
+
+4. Limitation of Liability. IN NO EVENT WILL META OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY DIRECT OR INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF META OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING.
+
+5. Intellectual Property.
+
+
+a. Subject to Meta’s ownership of SAM Materials and derivatives made by or for Meta, with respect to any derivative works and modifications of the SAM Materials that are made by you, as between you and Meta, you are and will be the owner of such derivative works and modifications.
+
+b. If you institute litigation or other proceedings against Meta or any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the SAM Materials, outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Meta from and against any claim by any third party arising out of or related to your use or distribution of the SAM Materials.
+
+6. Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the SAM Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Meta may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of the SAM Materials. Sections 3, 4 and 7 shall survive the termination of this Agreement.
+
+7. Governing Law and Jurisdiction. This Agreement will be governed and construed under the laws of the State of California without regard to choice of law principles, and the UN Convention on Contracts for the International Sale of Goods does not apply to this Agreement. The courts of California shall have exclusive jurisdiction of any dispute arising out of this Agreement.
+
+
+8. Modifications and Amendments. Meta may modify this Agreement from time to time; provided that they are similar in spirit to the current version of the Agreement, but may differ in detail to address new problems or concerns. All such changes will be effective immediately. Your continued use of the SAM Materials after any modification to this Agreement constitutes your agreement to such modification. Except as provided in this Agreement, no modification or addition to any provision of this Agreement will be binding unless it is in writing and signed by an authorized representative of both you and Meta.

README.md

@@ -0,0 +1,723 @@
+---
+license: other
+extra_gated_fields:
+  First Name: text
+  Last Name: text
+  Date of birth: date_picker
+  Country: country
+  Affiliation: text
+  Job title:
+    type: select
+    options:
+    - Student
+    - Research Graduate
+    - AI researcher
+    - AI developer/engineer
+    - Reporter
+    - Other
+  geo: ip_location
+  By clicking Submit below I accept the terms of the license and acknowledge that the information I provide will be collected stored processed and shared in accordance with the Meta Privacy Policy: checkbox
+extra_gated_description: >-
+  The information you provide will be collected, stored, processed and shared in
+  accordance with the [Meta Privacy
+  Policy](https://www.facebook.com/privacy/policy/).
+extra_gated_button_content: Submit
+language:
+- en
+pipeline_tag: mask-generation
+library_name: transformers
+tags:
+- sam3
+---
+
+** 💛💛💛 南無阿彌陀佛 💛 ** <br/>
+** 💛💛💛 NamoAhmituofo 💛 **
+> 讓我們以慈心善待一切衆生，拒絕肉食、喫素放生，阿彌陀佛 💛 <br/>
+> Let's be kind to all sentinent beings, being vegetarians and stop eatting meats, away from all sufferrings, Ahmituofo 💛. <br/>
+> 如此，我們將獲得無量無邊利益，因爲衆生一體，因果相續。
+> By doing so, we shall gain immeasurable benefits and peace for ourselves and all beings. For we are ONE, you get what you pay.
+
+SAM 3 is a unified foundation model for promptable segmentation in images and videos. It can detect, segment, and track objects using text or visual prompts such as points, boxes, and masks. Compared to its predecessor [SAM 2](https://github.com/facebookresearch/sam2), SAM 3 introduces the ability to exhaustively segment all instances of an open-vocabulary concept specified by a short text phrase or exemplars. Unlike prior work, SAM 3 can handle a vastly larger set of open-vocabulary prompts. It achieves 75-80% of human performance on our new [SA-CO benchmark](https://github.com/facebookresearch/sam3/edit/main_readme/README.md#sa-co-dataset) which contains 270K unique concepts, over 50 times more than existing benchmarks.
+
+[Hugging Face 🤗  app](https://huggingface.co/spaces/akhaliq/sam3)
+
+### Basic Usage
+
+```python
+import torch
+#################################### For Image ####################################
+from PIL import Image
+from sam3.model_builder import build_sam3_image_model
+from sam3.model.sam3_image_processor import Sam3Processor
+# Load the model
+model = build_sam3_image_model()
+processor = Sam3Processor(model)
+# Load an image
+image = Image.open("<YOUR_IMAGE_PATH.jpg>")
+inference_state = processor.set_image(image)
+# Prompt the model with text
+output = processor.set_text_prompt(state=inference_state, prompt="<YOUR_TEXT_PROMPT>")
+
+# Get the masks, bounding boxes, and scores
+masks, boxes, scores = output["masks"], output["boxes"], output["scores"]
+
+#################################### For Video ####################################
+
+from sam3.model_builder import build_sam3_video_predictor
+
+video_predictor = build_sam3_video_predictor()
+video_path = "<YOUR_VIDEO_PATH>" # a JPEG folder or an MP4 video file
+# Start a session
+response = video_predictor.handle_request(
+    request=dict(
+        type="start_session",
+        resource_path=video_path,
+    )
+)
+response = video_predictor.handle_request(
+    request=dict(
+        type="add_prompt",
+        session_id=response["session_id"],
+        frame_index=0, # Arbitrary frame index
+        text="<YOUR_TEXT_PROMPT>",
+    )
+)
+output = response["outputs"]
+```
+
+The official code is publicly released in the [sam3 repo](https://github.com/facebookresearch/sam3).
+
+
+## Usage with 🤗 Transformers
+
+### SAM3 - Promptable Concept Segmentation (PCS) for Images
+
+SAM3 performs Promptable Concept Segmentation (PCS) on images, taking text and/or image exemplars as prompts and returning segmentation masks for **all matching object instances** in the image.
+
+#### Text-Only Prompts
+
+```python
+>>> from transformers import Sam3Processor, Sam3Model
+>>> import torch
+>>> from PIL import Image
+>>> import requests
+
+>>> device = "cuda" if torch.cuda.is_available() else "cpu"
+
+>>> model = Sam3Model.from_pretrained("facebook/sam3").to(device)
+>>> processor = Sam3Processor.from_pretrained("facebook/sam3")
+
+>>> # Load image
+>>> image_url = "http://images.cocodataset.org/val2017/000000077595.jpg"
+>>> image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")
+
+>>> # Segment using text prompt
+>>> inputs = processor(images=image, text="ear", return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> # Post-process results
+>>> results = processor.post_process_instance_segmentation(
+...     outputs,
+...     threshold=0.5,
+...     mask_threshold=0.5,
+...     target_sizes=inputs.get("original_sizes").tolist()
+... )[0]
+
+>>> print(f"Found {len(results['masks'])} objects")
+>>> # Results contain:
+>>> # - masks: Binary masks resized to original image size
+>>> # - boxes: Bounding boxes in absolute pixel coordinates (xyxy format)
+>>> # - scores: Confidence scores
+```
+
+You can display masks using a simple helper like the following:
+
+```python
+import numpy as np
+import matplotlib
+
+def overlay_masks(image, masks):
+    image = image.convert("RGBA")
+    masks = 255 * masks.cpu().numpy().astype(np.uint8)
+    
+    n_masks = masks.shape[0]
+    cmap = matplotlib.colormaps.get_cmap("rainbow").resampled(n_masks)
+    colors = [
+        tuple(int(c * 255) for c in cmap(i)[:3])
+        for i in range(n_masks)
+    ]
+
+    for mask, color in zip(masks, colors):
+        mask = Image.fromarray(mask)
+        overlay = Image.new("RGBA", image.size, color + (0,))
+        alpha = mask.point(lambda v: int(v * 0.5))
+        overlay.putalpha(alpha)
+        image = Image.alpha_composite(image, overlay)
+    return image
+```
+
+Then you can save the resulting composite image or display it in a notebook:
+
+```python
+>>> overlay_masks(image, results["masks"])
+```
+
+#### Single Bounding Box Prompt
+
+Segment objects using a bounding box:
+
+```python
+>>> # Box in xyxy format: [x1, y1, x2, y2] in pixel coordinates
+>>> # Example: laptop region
+>>> box_xyxy = [100, 150, 500, 450]
+>>> input_boxes = [[box_xyxy]]  # [batch, num_boxes, 4]
+>>> input_boxes_labels = [[1]]  # 1 = positive box
+
+>>> inputs = processor(
+...     images=image,
+...     input_boxes=input_boxes,
+...     input_boxes_labels=input_boxes_labels,
+...     return_tensors="pt"
+... ).to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> # Post-process results
+>>> results = processor.post_process_instance_segmentation(
+...     outputs,
+...     threshold=0.5,
+...     mask_threshold=0.5,
+...     target_sizes=inputs.get("original_sizes").tolist()
+... )[0]
+```
+
+#### Multiple Box Prompts (Positive and Negative)
+
+Use multiple boxes with positive and negative labels to refine the concept:
+
+```python
+>>> # Load kitchen image
+>>> kitchen_url = "http://images.cocodataset.org/val2017/000000136466.jpg"
+>>> kitchen_image = Image.open(requests.get(kitchen_url, stream=True).raw).convert("RGB")
+
+>>> # Define two positive boxes (e.g., dial and button on oven)
+>>> # Boxes are in xyxy format [x1, y1, x2, y2] in pixel coordinates
+>>> box1_xyxy = [59, 144, 76, 163]  # Dial box
+>>> box2_xyxy = [87, 148, 104, 159]  # Button box
+>>> input_boxes = [[box1_xyxy, box2_xyxy]]
+>>> input_boxes_labels = [[1, 1]]  # Both positive
+
+>>> inputs = processor(
+...     images=kitchen_image,
+...     input_boxes=input_boxes,
+...     input_boxes_labels=input_boxes_labels,
+...     return_tensors="pt"
+... ).to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> # Post-process results
+>>> results = processor.post_process_instance_segmentation(
+...     outputs,
+...     threshold=0.5,
+...     mask_threshold=0.5,
+...     target_sizes=inputs.get("original_sizes").tolist()
+... )[0]
+>>> overlay_masks(kitchen_image, results["masks"])
+```
+
+#### Combined Prompts (Text + Negative Box)
+
+Use text prompts with negative visual prompts to refine the concept:
+
+```python
+>>> # Segment "handle" but exclude the oven handle using a negative box
+>>> text = "handle"
+>>> # Negative box covering oven handle area (xyxy): [40, 183, 318, 204]
+>>> oven_handle_box = [40, 183, 318, 204]
+>>> input_boxes = [[oven_handle_box]]
+
+>>> inputs = processor(
+...     images=kitchen_image,
+...     text=text,
+...     input_boxes=input_boxes,
+...     input_boxes_labels=[[0]],  # 0 = negative (exclude this region)
+...     return_tensors="pt"
+... ).to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> # Post-process results
+>>> results = processor.post_process_instance_segmentation(
+...     outputs,
+...     threshold=0.5,
+...     mask_threshold=0.5,
+...     target_sizes=inputs.get("original_sizes").tolist()
+... )[0]
+>>> # This will segment pot handles but exclude the oven handle
+```
+
+#### Batched Inference with Text Prompts
+
+Process multiple images with different text prompts by batch:
+
+```python
+>>> cat_url = "http://images.cocodataset.org/val2017/000000077595.jpg"
+>>> kitchen_url = "http://images.cocodataset.org/val2017/000000136466.jpg"
+>>> images = [
+...     Image.open(requests.get(cat_url, stream=True).raw).convert("RGB"),
+...     Image.open(requests.get(kitchen_url, stream=True).raw).convert("RGB")
+... ]
+
+>>> text_prompts = ["ear", "dial"]
+
+>>> inputs = processor(images=images, text=text_prompts, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> # Post-process results for both images
+>>> results = processor.post_process_instance_segmentation(
+...     outputs,
+...     threshold=0.5,
+...     mask_threshold=0.5,
+...     target_sizes=inputs.get("original_sizes").tolist()
+... )
+
+>>> print(f"Image 1: {len(results[0]['masks'])} objects found")
+>>> print(f"Image 2: {len(results[1]['masks'])} objects found")
+```
+
+#### Batched Mixed Prompts
+
+Use different prompt types for different images in the same batch:
+
+```python
+>>> # Image 1: text prompt "laptop"
+>>> # Image 2: visual prompt (dial box)
+>>> box2_xyxy = [59, 144, 76, 163]
+
+>>> inputs = processor(
+...     images=images,
+...     text=["laptop", None],  # Only first image has text
+...     input_boxes=[None, [box2_xyxy]],  # Only second image has box
+...     input_boxes_labels=[None, [1]],  # Positive box for second image
+...     return_tensors="pt"
+... ).to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> # Post-process results for both images
+>>> results = processor.post_process_instance_segmentation(
+...     outputs,
+...     threshold=0.5,
+...     mask_threshold=0.5,
+...     target_sizes=inputs.get("original_sizes").tolist()
+... )
+>>> # Both images processed in single forward pass
+```
+
+#### Semantic Segmentation Output
+
+SAM3 also provides semantic segmentation alongside instance masks:
+
+```python
+>>> inputs = processor(images=image, text="ear", return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> # Instance segmentation masks
+>>> instance_masks = torch.sigmoid(outputs.pred_masks)  # [batch, num_queries, H, W]
+
+>>> # Semantic segmentation (single channel)
+>>> semantic_seg = outputs.semantic_seg  # [batch, 1, H, W]
+
+>>> print(f"Instance masks: {instance_masks.shape}")
+>>> print(f"Semantic segmentation: {semantic_seg.shape}")
+```
+
+### SAM3 Video - Promptable Concept Segmentation (PCS) for Videos
+
+SAM3 Video performs Promptable Concept Segmentation (PCS) on videos, taking text as prompts and detecting and tracking **all matching object instances** across video frames.
+
+#### Pre-loaded Video Inference
+
+Process a video with all frames already available using text prompts:
+
+```python
+>>> from transformers import Sam3VideoModel, Sam3VideoProcessor
+>>> from accelerate import Accelerator
+>>> import torch
+
+>>> device = Accelerator().device
+>>> model = Sam3VideoModel.from_pretrained("facebook/sam3").to(device, dtype=torch.bfloat16)
+>>> processor = Sam3VideoProcessor.from_pretrained("facebook/sam3")
+
+>>> # Load video frames
+>>> from transformers.video_utils import load_video
+>>> video_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/bedroom.mp4"
+>>> video_frames, _ = load_video(video_url)
+
+>>> # Initialize video inference session
+>>> inference_session = processor.init_video_session(
+...     video=video_frames,
+...     inference_device=device,
+...     processing_device="cpu",
+...     video_storage_device="cpu",
+...     dtype=torch.bfloat16,
+... )
+
+>>> # Add text prompt to detect and track objects
+>>> text = "person"
+>>> inference_session = processor.add_text_prompt(
+...     inference_session=inference_session,
+...     text=text,
+... )
+
+>>> # Process all frames in the video
+>>> outputs_per_frame = {}
+>>> for model_outputs in model.propagate_in_video_iterator(
+...     inference_session=inference_session, max_frame_num_to_track=50
+... ):
+...     processed_outputs = processor.postprocess_outputs(inference_session, model_outputs)
+...     outputs_per_frame[model_outputs.frame_idx] = processed_outputs
+
+>>> print(f"Processed {len(outputs_per_frame)} frames")
+Processed 51 frames
+
+>>> # Access results for a specific frame
+>>> frame_0_outputs = outputs_per_frame[0]
+>>> print(f"Detected {len(frame_0_outputs['object_ids'])} objects")
+>>> print(f"Object IDs: {frame_0_outputs['object_ids'].tolist()}")
+>>> print(f"Scores: {frame_0_outputs['scores'].tolist()}")
+>>> print(f"Boxes shape (XYXY format, absolute coordinates): {frame_0_outputs['boxes'].shape}")
+>>> print(f"Masks shape: {frame_0_outputs['masks'].shape}")
+```
+
+#### Streaming Video Inference
+
+For real-time applications, the Transformers implementation of SAM3 Video supports processing video frames as they arrive:
+
+```python
+>>> # Initialize session for streaming
+>>> streaming_inference_session = processor.init_video_session(
+...     inference_device=device,
+...     processing_device="cpu",
+...     video_storage_device="cpu",
+...     dtype=torch.bfloat16,
+... )
+
+>>> # Add text prompt
+>>> text = "person"
+>>> streaming_inference_session = processor.add_text_prompt(
+...     inference_session=streaming_inference_session,
+...     text=text,
+... )
+
+>>> # Process frames one by one (streaming mode)
+>>> streaming_outputs_per_frame = {}
+>>> for frame_idx, frame in enumerate(video_frames[:50]):  # Process first 50 frames
+...     # First, process the frame using the processor
+...     inputs = processor(images=frame, device=device, return_tensors="pt")
+...
+...     # Process frame using streaming inference - pass the processed pixel_values
+...     model_outputs = model(
+...         inference_session=streaming_inference_session,
+...         frame=inputs.pixel_values[0],  # Provide processed frame - this enables streaming mode
+...         reverse=False,
+...     )
+...
+...     # Post-process outputs with original_sizes for proper resolution handling
+...     processed_outputs = processor.postprocess_outputs(
+...         streaming_inference_session,
+...         model_outputs,
+...         original_sizes=inputs.original_sizes,  # Required for streaming inference
+...     )
+...     streaming_outputs_per_frame[frame_idx] = processed_outputs
+...
+...     if (frame_idx + 1) % 10 == 0:
+...         print(f"Processed {frame_idx + 1} frames...")
+
+>>> print(f"✓ Streaming inference complete! Processed {len(streaming_outputs_per_frame)} frames")
+✓ Streaming inference complete! Processed 50 frames
+
+>>> # Access results
+>>> frame_0_outputs = streaming_outputs_per_frame[0]
+>>> print(f"Detected {len(frame_0_outputs['object_ids'])} objects in first frame")
+>>> print(f"Boxes are in XYXY format (absolute pixel coordinates): {frame_0_outputs['boxes'].shape}")
+>>> print(f"Masks are at original video resolution: {frame_0_outputs['masks'].shape}")
+```
+
+<div class="warning">
+⚠️ **Note on Streaming Inference Quality**: Streaming inference disables hotstart heuristics that remove unmatched and duplicate objects, as these require access to future frames to make informed decisions. This may result in more false positive detections and duplicate object tracks compared to pre-loaded video inference. For best results, use pre-loaded video inference when all frames are available.
+</div>
+
+### SAM3 Tracker - Promptable Visual Segmentation (PVS) for Images
+
+Sam3Tracker performs Promptable Visual Segmentation (PVS) on images, taking interactive visual prompts (points, boxes, masks) to segment a **specific object instance** per prompt. It is an updated version of SAM2 that maintains the same API while providing improved performance, making it a drop-in replacement for SAM2 workflows.
+
+#### Automatic Mask Generation with Pipeline
+
+```python
+>>> from transformers import pipeline
+
+>>> generator = pipeline("mask-generation", model="facebook/sam3", device=0)
+>>> image_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg"
+>>> outputs = generator(image_url, points_per_batch=64)
+
+>>> len(outputs["masks"])  # Number of masks generated
+```
+
+#### Basic Image Segmentation
+
+##### Single Point Click
+
+```python
+>>> from transformers import Sam3TrackerProcessor, Sam3TrackerModel
+>>> from accelerate import Accelerator
+>>> import torch
+>>> from PIL import Image
+>>> import requests
+
+>>> device = Accelerator().device
+
+>>> model = Sam3TrackerModel.from_pretrained("facebook/sam3").to(device)
+>>> processor = Sam3TrackerProcessor.from_pretrained("facebook/sam3")
+
+>>> image_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg"
+>>> raw_image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")
+
+>>> input_points = [[[[500, 375]]]]  # Single point click, 4 dimensions (image_dim, object_dim, point_per_object_dim, coordinates)
+>>> input_labels = [[[1]]]  # 1 for positive click, 0 for negative click, 3 dimensions (image_dim, object_dim, point_label)
+
+>>> inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(model.device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])[0]
+
+>>> # The model outputs multiple mask predictions ranked by quality score
+>>> print(f"Generated {masks.shape[1]} masks with shape {masks.shape}")
+```
+
+##### Multiple Points for Refinement
+
+```python
+>>> # Add both positive and negative points to refine the mask
+>>> input_points = [[[[500, 375], [1125, 625]]]]  # Multiple points for refinement
+>>> input_labels = [[[1, 1]]]  # Both positive clicks
+
+>>> inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])[0]
+```
+
+##### Bounding Box Input
+
+```python
+>>> # Define bounding box as [x_min, y_min, x_max, y_max]
+>>> input_boxes = [[[75, 275, 1725, 850]]]
+
+>>> inputs = processor(images=raw_image, input_boxes=input_boxes, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])[0]
+```
+
+##### Multiple Objects Segmentation
+
+```python
+>>> # Define points for two different objects
+>>> input_points = [[[[500, 375]], [[650, 750]]]]  # Points for two objects in same image
+>>> input_labels = [[[1], [1]]]  # Positive clicks for both objects
+
+>>> inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(model.device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs, multimask_output=False)
+
+>>> # Each object gets its own mask
+>>> masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])[0]
+>>> print(f"Generated masks for {masks.shape[0]} objects")
+Generated masks for 2 objects
+```
+
+#### Batch Inference
+
+
+```python
+>>> # Load multiple images
+>>> image_urls = [
+...     "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg",
+...     "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/dog-sam.png"
+... ]
+>>> raw_images = [Image.open(requests.get(url, stream=True).raw).convert("RGB") for url in image_urls]
+
+>>> # Single point per image
+>>> input_points = [[[[500, 375]]], [[[770, 200]]]]  # One point for each image
+>>> input_labels = [[[1]], [[1]]]  # Positive clicks for both images
+
+>>> inputs = processor(images=raw_images, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(model.device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs, multimask_output=False)
+
+>>> # Post-process masks for each image
+>>> all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])
+>>> print(f"Processed {len(all_masks)} images, each with {all_masks[0].shape[0]} objects")
+```
+
+### SAM3 Tracker Video - Promptable Visual Segmentation (PVS) for Videos
+
+Sam3TrackerVideo performs Promptable Visual Segmentation (PVS) on videos, taking interactive visual prompts (points, boxes, masks) to track a **specific object instance** per prompt across video frames. It is an updated version of SAM2 Video that maintains the same API while providing improved performance, making it a drop-in replacement for SAM2 Video workflows.
+
+#### Basic Video Tracking
+
+```python
+>>> from transformers import Sam3TrackerVideoModel, Sam3TrackerVideoProcessor
+>>> from accelerate import Accelerator
+>>> import torch
+
+>>> device = Accelerator().device
+>>> model = Sam3TrackerVideoModel.from_pretrained("facebook/sam3").to(device, dtype=torch.bfloat16)
+>>> processor = Sam3TrackerVideoProcessor.from_pretrained("facebook/sam3")
+
+>>> # Load video frames
+>>> from transformers.video_utils import load_video
+>>> video_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/bedroom.mp4"
+>>> video_frames, _ = load_video(video_url)
+
+>>> # Initialize video inference session
+>>> inference_session = processor.init_video_session(
+...     video=video_frames,
+...     inference_device=device,
+...     dtype=torch.bfloat16,
+... )
+
+>>> # Add click on first frame to select object
+>>> ann_frame_idx = 0
+>>> ann_obj_id = 1
+>>> points = [[[[210, 350]]]]
+>>> labels = [[[1]]]
+
+>>> processor.add_inputs_to_inference_session(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+...     obj_ids=ann_obj_id,
+...     input_points=points,
+...     input_labels=labels,
+... )
+
+>>> # Segment the object on the first frame (optional, you can also propagate the masks through the video directly)
+>>> outputs = model(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+... )
+>>> video_res_masks = processor.post_process_masks(
+...     [outputs.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+... )[0]
+>>> print(f"Segmentation shape: {video_res_masks.shape}")
+Segmentation shape: torch.Size([1, 1, 480, 854])
+
+>>> # Propagate through the entire video
+>>> video_segments = {}
+>>> for sam3_tracker_video_output in model.propagate_in_video_iterator(inference_session):
+...     video_res_masks = processor.post_process_masks(
+...         [sam3_tracker_video_output.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+...     )[0]
+...     video_segments[sam3_tracker_video_output.frame_idx] = video_res_masks
+
+>>> print(f"Tracked object through {len(video_segments)} frames")
+Tracked object through 180 frames
+```
+
+#### Multi-Object Video Tracking
+
+Track multiple objects simultaneously across video frames:
+
+```python
+>>> # Reset for new tracking session
+>>> inference_session.reset_inference_session()
+
+>>> # Add multiple objects on the first frame
+>>> ann_frame_idx = 0
+>>> obj_ids = [2, 3]
+>>> input_points = [[[[200, 300]], [[400, 150]]]]  # Points for two objects (batched)
+>>> input_labels = [[[1], [1]]]
+
+>>> processor.add_inputs_to_inference_session(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+...     obj_ids=obj_ids,
+...     input_points=input_points,
+...     input_labels=input_labels,
+... )
+
+>>> # Get masks for both objects on first frame (optional, you can also propagate the masks through the video directly)
+>>> outputs = model(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+... )
+
+>>> # Propagate both objects through video
+>>> video_segments = {}
+>>> for sam3_tracker_video_output in model.propagate_in_video_iterator(inference_session):
+...     video_res_masks = processor.post_process_masks(
+...         [sam3_tracker_video_output.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+...     )[0]
+...     video_segments[sam3_tracker_video_output.frame_idx] = {
+...         obj_id: video_res_masks[i]
+...         for i, obj_id in enumerate(inference_session.obj_ids)
+...     }
+
+>>> print(f"Tracked {len(inference_session.obj_ids)} objects through {len(video_segments)} frames")
+Tracked 2 objects through 180 frames
+```
+
+#### Streaming Video Inference
+
+For real-time applications, Sam3TrackerVideo supports processing video frames as they arrive:
+
+```python
+>>> # Initialize session for streaming
+>>> inference_session = processor.init_video_session(
+...     inference_device=device,
+...     dtype=torch.bfloat16,
+... )
+
+>>> # Process frames one by one
+>>> for frame_idx, frame in enumerate(video_frames[:10]):  # Process first 10 frames
+...     inputs = processor(images=frame, device=device, return_tensors="pt")
+...
+...     if frame_idx == 0:
+...         # Add point input on first frame
+...         processor.add_inputs_to_inference_session(
+...             inference_session=inference_session,
+...             frame_idx=0,
+...             obj_ids=1,
+...             input_points=[[[[210, 350], [250, 220]]]],
+...             input_labels=[[[1, 1]]],
+...             original_size=inputs.original_sizes[0], # need to be provided when using streaming video inference
+...         )
+...
+...     # Process current frame
+...     sam3_tracker_video_output = model(inference_session=inference_session, frame=inputs.pixel_values[0])
+...
+...     video_res_masks = processor.post_process_masks(
+...         [sam3_tracker_video_output.pred_masks], original_sizes=inputs.original_sizes, binarize=False
+...     )[0]
+...     print(f"Frame {frame_idx}: mask shape {video_res_masks.shape}")
+```

config.json

@@ -0,0 +1,896 @@
+{
+  "architectures": [
+    "Sam3VideoModel"
+  ],
+  "assoc_iou_thresh": 0.1,
+  "decrease_trk_keep_alive_for_empty_masklets": false,
+  "det_nms_thresh": 0.1,
+  "detector_config": {
+    "detr_decoder_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "box_rpb_mode": "log",
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dropout": 0.1,
+      "dtype": null,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "relu",
+      "hidden_dropout": 0.0,
+      "hidden_size": 256,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_range": 0.02,
+      "intermediate_size": 2048,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "sam3_detr_decoder",
+      "no_repeat_ngram_size": 0,
+      "num_attention_heads": 8,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_layers": 6,
+      "num_queries": 200,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0,
+      "use_presence_token": true
+    },
+    "detr_encoder_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dropout": 0.1,
+      "dtype": null,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "relu",
+      "hidden_dropout": 0.0,
+      "hidden_size": 256,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_range": 0.02,
+      "intermediate_size": 2048,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "sam3_detr_encoder",
+      "no_repeat_ngram_size": 0,
+      "num_attention_heads": 8,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_layers": 6,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0
+    },
+    "geometry_encoder_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dropout": 0.1,
+      "dtype": null,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "relu",
+      "hidden_dropout": 0.0,
+      "hidden_size": 256,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_range": 0.02,
+      "intermediate_size": 2048,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "sam3_geometry_encoder",
+      "no_repeat_ngram_size": 0,
+      "num_attention_heads": 8,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_layers": 3,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "roi_size": 7,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0
+    },
+    "initializer_range": 0.02,
+    "mask_decoder_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dropout": 0.0,
+      "dtype": null,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_size": 256,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_range": 0.02,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "sam3_mask_decoder",
+      "no_repeat_ngram_size": 0,
+      "num_attention_heads": 8,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_return_sequences": 1,
+      "num_upsampling_stages": 3,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0
+    },
+    "model_type": "sam3",
+    "text_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": 49406,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dtype": null,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": 49407,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "gelu",
+      "hidden_size": 1024,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_factor": 1.0,
+      "initializer_range": 0.02,
+      "intermediate_size": 4096,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-05,
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "max_position_embeddings": 32,
+      "min_length": 0,
+      "model_type": "clip_text_model",
+      "no_repeat_ngram_size": 0,
+      "num_attention_heads": 16,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_hidden_layers": 24,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": 1,
+      "prefix": null,
+      "problem_type": null,
+      "projection_dim": 512,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0,
+      "vocab_size": 49408
+    },
+    "vision_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "backbone_config": {
+        "_name_or_path": "",
+        "add_cross_attention": false,
+        "architectures": null,
+        "attention_dropout": 0.0,
+        "bad_words_ids": null,
+        "begin_suppress_tokens": null,
+        "bos_token_id": null,
+        "chunk_size_feed_forward": 0,
+        "cross_attention_hidden_size": null,
+        "decoder_start_token_id": null,
+        "diversity_penalty": 0.0,
+        "do_sample": false,
+        "dtype": null,
+        "early_stopping": false,
+        "encoder_no_repeat_ngram_size": 0,
+        "eos_token_id": null,
+        "exponential_decay_length_penalty": null,
+        "finetuning_task": null,
+        "forced_bos_token_id": null,
+        "forced_eos_token_id": null,
+        "global_attn_indexes": [
+          7,
+          15,
+          23,
+          31
+        ],
+        "hidden_act": "gelu",
+        "hidden_dropout": 0.0,
+        "hidden_size": 1024,
+        "id2label": {
+          "0": "LABEL_0",
+          "1": "LABEL_1"
+        },
+        "image_size": 1008,
+        "initializer_range": 0.02,
+        "intermediate_size": 4736,
+        "is_decoder": false,
+        "is_encoder_decoder": false,
+        "label2id": {
+          "LABEL_0": 0,
+          "LABEL_1": 1
+        },
+        "layer_norm_eps": 1e-06,
+        "layer_scale_init_value": null,
+        "length_penalty": 1.0,
+        "max_length": 20,
+        "min_length": 0,
+        "model_type": "sam3_vit_model",
+        "no_repeat_ngram_size": 0,
+        "num_attention_heads": 16,
+        "num_beam_groups": 1,
+        "num_beams": 1,
+        "num_channels": 3,
+        "num_hidden_layers": 32,
+        "num_return_sequences": 1,
+        "output_attentions": false,
+        "output_hidden_states": false,
+        "output_scores": false,
+        "pad_token_id": null,
+        "patch_size": 14,
+        "prefix": null,
+        "pretrain_image_size": 336,
+        "problem_type": null,
+        "qkv_bias": true,
+        "remove_invalid_values": false,
+        "repetition_penalty": 1.0,
+        "return_dict": true,
+        "return_dict_in_generate": false,
+        "rope_theta": 10000.0,
+        "sep_token_id": null,
+        "suppress_tokens": null,
+        "task_specific_params": null,
+        "temperature": 1.0,
+        "tie_encoder_decoder": false,
+        "tie_word_embeddings": true,
+        "tokenizer_class": null,
+        "top_k": 50,
+        "top_p": 1.0,
+        "typical_p": 1.0,
+        "window_size": 24
+      },
+      "backbone_feature_sizes": [
+        [
+          288,
+          288
+        ],
+        [
+          144,
+          144
+        ],
+        [
+          72,
+          72
+        ]
+      ],
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dtype": null,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "fpn_hidden_size": 256,
+      "fpn_kernel_size": 2,
+      "fpn_stride": 2,
+      "hidden_act": "gelu",
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_range": 0.02,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "sam3_vision_model",
+      "no_repeat_ngram_size": 0,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_feature_levels": 3,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "scale_factors": [
+        4.0,
+        2.0,
+        1.0,
+        0.5
+      ],
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0
+    }
+  },
+  "dtype": "float32",
+  "fill_hole_area": 16,
+  "high_conf_thresh": 0.8,
+  "high_iou_thresh": 0.8,
+  "hotstart_delay": 15,
+  "hotstart_dup_thresh": 8,
+  "hotstart_unmatch_thresh": 8,
+  "init_trk_keep_alive": 30,
+  "initializer_range": 0.02,
+  "low_res_mask_size": 288,
+  "max_num_objects": 10000,
+  "max_trk_keep_alive": 30,
+  "min_trk_keep_alive": -1,
+  "model_type": "sam3_video",
+  "new_det_thresh": 0.7,
+  "recondition_every_nth_frame": 16,
+  "recondition_on_trk_masks": false,
+  "score_threshold_detection": 0.5,
+  "suppress_overlapping_based_on_recent_occlusion_threshold": 0.7,
+  "suppress_unmatched_only_within_hotstart": true,
+  "tracker_config": {
+    "enable_occlusion_spatial_embedding": true,
+    "enable_temporal_pos_encoding_for_object_pointers": true,
+    "image_size": 1008,
+    "initializer_range": 0.02,
+    "mask_decoder_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "attention_downsample_rate": 2,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dtype": null,
+      "dynamic_multimask_stability_delta": 0.05,
+      "dynamic_multimask_stability_thresh": 0.98,
+      "dynamic_multimask_via_stability": true,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "gelu",
+      "hidden_size": 256,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "iou_head_depth": 3,
+      "iou_head_hidden_dim": 256,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "mlp_dim": 2048,
+      "model_type": "",
+      "no_repeat_ngram_size": 0,
+      "num_attention_heads": 8,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_hidden_layers": 2,
+      "num_multimask_outputs": 3,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0
+    },
+    "mask_downsampler_embed_dim": 256,
+    "mask_downsampler_hidden_act": "gelu",
+    "mask_downsampler_kernel_size": 3,
+    "mask_downsampler_padding": 1,
+    "mask_downsampler_stride": 2,
+    "mask_downsampler_total_stride": 16,
+    "max_cond_frame_num": 4,
+    "max_object_pointers_in_encoder": 16,
+    "memory_attention_downsample_rate": 1,
+    "memory_attention_dropout": 0.1,
+    "memory_attention_feed_forward_hidden_act": "relu",
+    "memory_attention_feed_forward_hidden_size": 2048,
+    "memory_attention_hidden_size": 256,
+    "memory_attention_num_attention_heads": 1,
+    "memory_attention_num_layers": 4,
+    "memory_attention_rope_dropout": 0.1,
+    "memory_attention_rope_feat_sizes": [
+      72,
+      72
+    ],
+    "memory_attention_rope_theta": 10000,
+    "memory_encoder_hidden_size": 256,
+    "memory_encoder_output_channels": 64,
+    "memory_fuser_embed_dim": 256,
+    "memory_fuser_hidden_act": "gelu",
+    "memory_fuser_intermediate_dim": 1024,
+    "memory_fuser_kernel_size": 7,
+    "memory_fuser_layer_scale_init_value": 1e-06,
+    "memory_fuser_num_layers": 2,
+    "memory_fuser_padding": 3,
+    "model_type": "sam3_tracker_video",
+    "multimask_max_pt_num": 1,
+    "multimask_min_pt_num": 0,
+    "multimask_output_for_tracking": true,
+    "multimask_output_in_sam": true,
+    "num_maskmem": 7,
+    "prompt_encoder_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dtype": null,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "gelu",
+      "hidden_size": 256,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "image_size": 1008,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "length_penalty": 1.0,
+      "mask_input_channels": 16,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "",
+      "no_repeat_ngram_size": 0,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_point_embeddings": 4,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "patch_size": 14,
+      "prefix": null,
+      "problem_type": null,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "scale": 1,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0
+    },
+    "sigmoid_bias_for_mem_enc": -10.0,
+    "sigmoid_scale_for_mem_enc": 20.0,
+    "vision_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "backbone_config": {
+        "_name_or_path": "",
+        "add_cross_attention": false,
+        "architectures": null,
+        "attention_dropout": 0.0,
+        "bad_words_ids": null,
+        "begin_suppress_tokens": null,
+        "bos_token_id": null,
+        "chunk_size_feed_forward": 0,
+        "cross_attention_hidden_size": null,
+        "decoder_start_token_id": null,
+        "diversity_penalty": 0.0,
+        "do_sample": false,
+        "dtype": null,
+        "early_stopping": false,
+        "encoder_no_repeat_ngram_size": 0,
+        "eos_token_id": null,
+        "exponential_decay_length_penalty": null,
+        "finetuning_task": null,
+        "forced_bos_token_id": null,
+        "forced_eos_token_id": null,
+        "global_attn_indexes": [
+          7,
+          15,
+          23,
+          31
+        ],
+        "hidden_act": "gelu",
+        "hidden_dropout": 0.0,
+        "hidden_size": 1024,
+        "id2label": {
+          "0": "LABEL_0",
+          "1": "LABEL_1"
+        },
+        "image_size": 1008,
+        "initializer_range": 0.02,
+        "intermediate_size": 4736,
+        "is_decoder": false,
+        "is_encoder_decoder": false,
+        "label2id": {
+          "LABEL_0": 0,
+          "LABEL_1": 1
+        },
+        "layer_norm_eps": 1e-06,
+        "layer_scale_init_value": null,
+        "length_penalty": 1.0,
+        "max_length": 20,
+        "min_length": 0,
+        "model_type": "sam3_vit_model",
+        "no_repeat_ngram_size": 0,
+        "num_attention_heads": 16,
+        "num_beam_groups": 1,
+        "num_beams": 1,
+        "num_channels": 3,
+        "num_hidden_layers": 32,
+        "num_return_sequences": 1,
+        "output_attentions": false,
+        "output_hidden_states": false,
+        "output_scores": false,
+        "pad_token_id": null,
+        "patch_size": 14,
+        "prefix": null,
+        "pretrain_image_size": 336,
+        "problem_type": null,
+        "qkv_bias": true,
+        "remove_invalid_values": false,
+        "repetition_penalty": 1.0,
+        "return_dict": true,
+        "return_dict_in_generate": false,
+        "rope_theta": 10000.0,
+        "sep_token_id": null,
+        "suppress_tokens": null,
+        "task_specific_params": null,
+        "temperature": 1.0,
+        "tie_encoder_decoder": false,
+        "tie_word_embeddings": true,
+        "tokenizer_class": null,
+        "top_k": 50,
+        "top_p": 1.0,
+        "typical_p": 1.0,
+        "window_size": 24
+      },
+      "backbone_feature_sizes": [
+        [
+          288,
+          288
+        ],
+        [
+          144,
+          144
+        ],
+        [
+          72,
+          72
+        ]
+      ],
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dtype": null,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "fpn_hidden_size": 256,
+      "fpn_kernel_size": 2,
+      "fpn_stride": 2,
+      "hidden_act": "gelu",
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_range": 0.02,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "sam3_vision_model",
+      "no_repeat_ngram_size": 0,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_feature_levels": 3,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "scale_factors": [
+        4.0,
+        2.0,
+        1.0,
+        0.5
+      ],
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "typical_p": 1.0
+    }
+  },
+  "transformers_version": "5.0.0.dev0",
+  "trk_assoc_iou_thresh": 0.5
+}

configuration.json

@@ -0,0 +1 @@
+{"framework": "pytorch", "task": "mask-generation", "allow_remote": true}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d06f0a5f84e435071fe6603e61d0b4cc7b40e0d39d487cfd4d67d8cc11cc14a
+size 3439938512

processor_config.json

@@ -0,0 +1,80 @@
+{
+  "image_processor": {
+    "crop_size": null,
+    "data_format": "channels_first",
+    "device": null,
+    "disable_grouping": null,
+    "do_center_crop": null,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_pad": null,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Sam3ImageProcessorFast",
+    "image_seq_length": null,
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "input_data_format": null,
+    "mask_size": {
+      "height": 288,
+      "width": 288
+    },
+    "pad_size": null,
+    "processor_class": "Sam3VideoProcessor",
+    "resample": 2,
+    "rescale_factor": 0.00392156862745098,
+    "return_tensors": null,
+    "size": {
+      "height": 1008,
+      "width": 1008
+    }
+  },
+  "processor_class": "Sam3VideoProcessor",
+  "target_size": 1008,
+  "video_processor": {
+    "crop_size": null,
+    "data_format": "channels_first",
+    "default_to_square": true,
+    "device": null,
+    "do_center_crop": null,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_pad": null,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": null,
+    "fps": null,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "input_data_format": null,
+    "num_frames": null,
+    "pad_size": null,
+    "processor_class": "Sam3VideoProcessor",
+    "resample": 2,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "return_tensors": null,
+    "size": {
+      "height": 1008,
+      "width": 1008
+    },
+    "video_metadata": null,
+    "video_processor_type": "Sam2VideoVideoProcessor"
+  }
+}

sam3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9999e2341ceef5e136daa386eecb55cb414446a00ac2b55eb2dfd2f7c3cf8c9e
+size 3450062241

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "max_length": 32,
+  "model_max_length": 32,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "Sam3VideoProcessor",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>"
+}