Update README.md

README.md

@@ -37,206 +37,21 @@ We're excited to unveil **Qwen2-VL**, the latest iteration of our Qwen-VL model,
     <img src="http://qianwen-res.oss-accelerate-overseas.aliyuncs.com/Qwen2-VL/mrope.png" width="80%"/>
 <p>
 
-We have three models with 2, 7 and 72 billion parameters. This repo contains the pretrained 2B Qwen2-VL model. For more information, visit our [Blog](https://qwenlm.github.io/blog/qwen2-vl/) and [GitHub](https://github.com/QwenLM/Qwen2-VL).
+We have three models with 2, 7 and 72 billion parameters. 
 
-## Requirements
-
-The code of Qwen2-VL has been in the latest Hugging face transformers and we advise you to build from source with command `pip install git+https://github.com/huggingface/transformers`, or you might encounter the following error:
-
-```
-KeyError: 'qwen2_vl'
-```
-
-## Quickstart
-
-Here we show a code snippet to show you how to use the chat model with `transformers`:
-
-### Single Media inference
-
-The model can accept both images and videos as input. Here's an example code for inference.
-
-```python
-from PIL import Image
-import requests
-import torch
-from torchvision import io
-from typing import Dict
-from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
-
-# Load the model in half-precision on the available device(s)
-model = Qwen2VLForConditionalGeneration.from_pretrained("Qwen/QQwen2-VL-2B-Base", device_map="auto")
-processor = AutoProcessor.from_pretrained("Qwen/QQwen2-VL-2B-Base")
-
-# Image
-url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
-image = Image.open(requests.get(url, stream=True).raw)
-
-conversation = [
-    {
-        "type":"image",
-    },
-    {
-        "type":"text",
-        "text":"In this image,"
-    }
-]
-
-
-# Preprocess the inputs
-text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
-# Excepted output: '<|vision_start|><|image_pad|><|vision_end|>In this image,'
-
-inputs = processor(text=[text_prompt], images=[image], padding=True, return_tensors="pt")
-inputs = inputs.to('cuda')
-
-# Inference: Generation of the output
-output_ids = model.generate(**inputs, max_new_tokens=128)
-generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, output_ids)]
-output_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
-print(output_text)
-
-
-
-# Video
-def fetch_video(ele: Dict, nframe_factor=2):
-    if isinstance(ele['video'], str):
-        def round_by_factor(number: int, factor: int) -> int:
-            return round(number / factor) * factor
-
-        video = ele["video"]
-        if video.startswith("file://"):
-            video = video[7:]
-
-        video, _, info = io.read_video(
-            video,
-            start_pts=ele.get("video_start", 0.0),
-            end_pts=ele.get("video_end", None),
-            pts_unit="sec",
-            output_format="TCHW",
-        )
-        assert not ("fps" in ele and "nframes" in ele), "Only accept either `fps` or `nframes`"
-        if "nframes" in ele:
-            nframes = round_by_factor(ele["nframes"], nframe_factor)
-        else:
-            fps = ele.get("fps", 1.0)
-            nframes = round_by_factor(video.size(0) / info["video_fps"] * fps, nframe_factor)
-        idx = torch.linspace(0, video.size(0) - 1, nframes, dtype=torch.int64)
-        return video[idx]
+This repo contains the **pretrained** 2B Qwen2-VL model.
 
-video_info = {"type": "video", "video": "/path/to/video.mp4", "fps": 1.0}
-video = fetch_video(video_info)
-conversation = [
-    {"type": "video"},
-    {"type": "text", "text": "What happened in the video? Answer:"},
-]
 
-# Preprocess the inputs
-text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
-# Excepted output: '<|vision_start|><|video_pad|><|vision_end|>What happened in the video? Answer:'
+For more information, visit our [Blog](https://qwenlm.github.io/blog/qwen2-vl/) and [GitHub](https://github.com/QwenLM/Qwen2-VL).
 
-inputs = processor(text=[text_prompt], videos=[video], padding=True, return_tensors="pt")
-inputs = inputs.to('cuda')
-
-# Inference: Generation of the output
-output_ids = model.generate(**inputs, max_new_tokens=128)
-generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, output_ids)]
-output_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
-print(output_text)
-```
-
-### Batch Mixed Media Inference
-
-The model can batch inputs composed of mixed samples of various types such as images, videos, and text. Here is an example.
-
-```python
-image1 = Image.open("/path/to/image1.jpg")
-image2 = Image.open("/path/to/image2.jpg")
-image3 = Image.open("/path/to/image3.jpg")
-image4 = Image.open("/path/to/image4.jpg")
-image5 = Image.open("/path/to/image5.jpg")
-video = fetch_video({
-    "type": "video",
-    "video": "/path/to/video.mp4",
-    "fps": 1.0
-})
-
-# Conversation for the first image
-conversation1 = [
-    {"type": "image"},
-    {"type": "text", "text": "In this image."}
-]
-
-# Conversation with two images
-conversation2 = [
-    {"type": "image"},
-    {"type": "image"},
-    {"type": "text", "text": "What is written in the pictures?"}
-]
-
-# Conversation with pure text
-conversation3 = "who are you?"
-
-
-# Conversation with mixed midia
-conversation4 = [
-    {"type": "image"},
-    {"type": "image"},
-    {"type": "video"},
-    {"type": "text", "text": "What are the common elements in these medias?"},
-]
-
-conversations = [conversation1, conversation2, conversation3, conversation4]
-# Preparation for batch inference
-texts = [processor.apply_chat_template(msg, add_generation_prompt=True) for msg in conversations]
-inputs = processor(
-    text=texts,
-    images=[image1, image2, image3, image4, image5],
-    videos=[video],
-    padding=True,
-    return_tensors="pt",
-)
-inputs = inputs.to('cuda')
-
-# Batch Inference
-output_ids = model.generate(**inputs, max_new_tokens=128)
-generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, output_ids)]
-output_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
-print(output_text)
-```
-
-#### Image Resolution for performance boost
+## Requirements
 
-The model supports a wide range of resolution inputs. By default, it uses the native resolution for input, but higher resolutions can enhance performance at the cost of more computation. Users can set the minimum and maximum number of pixels to achieve an optimal configuration for their needs, such as a token count range of 256-1280, to balance speed and memory usage.
+The code of Qwen2-VL has been in the latest Hugging Face `transformers` and we advise you to install the latest version with command `pip install -U transformers`, or you might encounter the following error:
 
-```python
-min_pixels = 256 * 28 * 28
-max_pixels = 1280 * 28 * 28
-processor = AutoProcessor.from_pretrained(
-    "Qwen/Qwen2-VL-2B-Base", min_pixels=min_pixels, max_pixels=max_pixels
-)
 ```
-
-#### Flash-Attention 2 to speed up generation
-
-First, make sure to install the latest version of Flash Attention 2:
-
-```bash
-pip install -U flash-attn --no-build-isolation
+KeyError: 'qwen2_vl'
 ```
 
-Also, you should have a hardware that is compatible with Flash-Attention 2. Read more about it in the official documentation of the [flash attention repository](https://github.com/Dao-AILab/flash-attention). FlashAttention-2 can only be used when a model is loaded in `torch.float16` or `torch.bfloat16`.
-
-To load and run a model using Flash Attention-2, simply add `attn_implementation="flash_attention_2"` when loading the model as follows:
-
-```python
-from transformers import Qwen2VLForConditionalGeneration
-
-model = Qwen2VLForConditionalGeneration.from_pretrained(
-    "Qwen/Qwen2-VL-2B-Instruct", 
-    torch_dtype=torch.bfloat16, 
-    attn_implementation="flash_attention_2",
-)
-```
 
 ## Limitations
 
@@ -257,8 +72,9 @@ If you find our work helpful, feel free to give us a cite.
 
 ```
 @article{Qwen2-VL,
-  title={Qwen2-VL},
-  author={Qwen team},
+  title={Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution}, 
+  author={Peng Wang and Shuai Bai and Sinan Tan and Shijie Wang and Zhihao Fan and Jinze Bai and Keqin Chen and Xuejing Liu and Jialin Wang and Wenbin Ge and Yang Fan and Kai Dang and Mengfei Du and Xuancheng Ren and Rui Men and Dayiheng Liu and Chang Zhou and Jingren Zhou and Junyang Lin},
+  journal={arXiv preprint arXiv:2409.12191},
   year={2024}
 }