# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


# embeddings_queue

created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 1
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for\nUni\ufb01ed Vision-Language Understanding and Generation\nJunnan Li Dongxu Li Caiming Xiong Steven Hoi\nSalesforce Research\nhttps://github.com/salesforce/BLIP\nAbstract\nVision-Language Pre-training (VLP) has ad-\nvanced the performance for many vision-language\ntasks. However, most existing pre-trained mod-\nels only excel in either understanding-based tasks\nor generation-based tasks. Furthermore, perfor-\nmance improvement has been largely achieved\nby scaling up the dataset with noisy image-text\npairs collected from the web, which is a subop-\ntimal source of supervision. In this paper, we\npropose BLIP, a new VLP framework which trans-\nfers \ufb02exibly to both vision-language understand-\ning and generation tasks. BLIP effectively uti-\nlizes the noisy web data by bootstrapping the\ncaptions, where a captioner generates synthetic\ncaptions and a \ufb01lter removes the noisy ones. We\nachieve state-of-the-art results on a wide range of\nvision-language tasks, such as image-text retrieval\n(+2.7% in average recall@1), image captioning\n(+2.8% in CIDEr), and VQA (+1.6% in VQA\nscore). BLIP also demonstrates strong general-\nization ability when directly transferred to video-\nlanguage tasks in a zero-shot manner. Code, mod-\nels, and datasets are released.\n1. Introduction\nVision-language pre-training has recently received tremen-\ndous success on various multimodal downstream tasks.\nHowever, existing methods have two major limitations:\n(1) Model perspective: most methods either adopt an\nencoder-based model (Radford et al., 2021; Li et al., 2021a),\nor an encoder-decoder (Cho et al., 2021; Wang et al., 2021)\nmodel. However, encoder-based models are less straightfor-\nward to directly transfer to text generation tasks (e.g. image\ncaptioning), whereas encoder-decoder models have not been\nsuccessfully adopted for image-text retrieval tasks.\n(2) Data perspective: most state-of-the-art methods (e.g.,\nCLIP (Radford et al., 2021), ALBEF (Li et al., 2021a),"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 2
encoding: FLOAT32
metadata:
{"chroma:document": "SimVLM (Wang et al., 2021)) pre-train on image-text pairs\nCap\n\u201cchocolate cake \nwith cream frosting \nand chocolate \nsprinkles on top\u201d\n\u201cblue sky bakery in \nsunset park \u201d \nFilt\nFilt\nFigure 1. We use a Captioner (Cap) to generate synthetic captions\nfor web images, and a Filter (Filt) to remove noisy captions.\ncollected from the web. Despite the performance gain ob-\ntained by scaling up the dataset, our paper shows that the\nnoisy web text is suboptimal for vision-language learning.\nTo this end, we propose BLIP: Bootstrapping Language-\nImage Pre-training for uni\ufb01ed vision-language understand-\ning and generation. BLIP is a new VLP framework which\nenables a wider range of downstream tasks than existing\nmethods. It introduces two contributions from the model\nand data perspective, respectively:\n(a) Multimodal mixture of Encoder-Decoder (MED): a new\nmodel architecture for effective multi-task pre-training and\n\ufb02exible transfer learning. An MED can operate either as\na unimodal encoder, or an image-grounded text encoder,\nor an image-grounded text decoder. The model is jointly\npre-trained with three vision-language objectives: image-\ntext contrastive learning, image-text matching, and image-\nconditioned language modeling.\n(b) Captioning and Filtering (CapFilt): a new dataset boos-\ntrapping method for learning from noisy image-text pairs.\nWe \ufb01netune a pre-trained MED into two modules: a cap-\ntioner to produce synthetic captions given web images, and\na \ufb01lter to remove noisy captions from both the original web\ntexts and the synthetic texts.\nWe perform extensive experiments and analysis, and make\nthe following key observations.\n\u2022 We show that the captioner and the \ufb01lter work together to\nachieve substantial performance improvement on various\ndownstream tasks by bootstrapping the captions. We also\n\ufb01nd that more diverse captions yield larger gains.\n\u2022 BLIP achieves state-of-the-art performance on a wide\nrange of vision-language tasks, including image-text re-\narXiv:2201.12086v2  [cs.CV]  15 Feb 2022"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 3
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nSelf Attention\nFeed Forward\nN\u00d7\nCross Attention\nFeed Forward\nBi Self-Att\nCross Attention\nFeed Forward\nBi Self-Att\nCross Attention\nFeed Forward\nCausal Self-Att\nITM\nITC\nLM\nN\u00d7\n\u201c[CLS] +           \u201d  \n\u201ca little girl holding a kitten next to a blue fence\u201d\n\u201c[Encode] +           \u201d  \n\u201c[Decode] +           \u201d  \nText\nEncoder\nImage\nEncoder\nImage-grounded \nText encoder\nImage-grounded \nText decoder\nFigure 2. Pre-training model architecture and objectives of BLIP (same parameters have the same color). We propose multimodal mixture\nof encoder-decoder, a uni\ufb01ed vision-language model which can operate in one of the three functionalities: (1) Unimodal encoder is\ntrained with an image-text contrastive (ITC) loss to align the vision and language representations. (2) Image-grounded text encoder uses\nadditional cross-attention layers to model vision-language interactions, and is trained with a image-text matching (ITM) loss to distinguish\nbetween positive and negative image-text pairs. (3) Image-grounded text decoder replaces the bi-directional self-attention layers with\ncausal self-attention layers, and shares the same cross-attention layers and feed forward networks as the encoder. The decoder is trained\nwith a language modeling (LM) loss to generate captions given images.\ntrieval, image captioning, visual question answering, vi-\nsual reasoning, and visual dialog. We also achieve state-of-\nthe-art zero-shot performance when directly transferring\nour models to two video-language tasks: text-to-video\nretrieval and videoQA.\n2. Related Work\n2.1. Vision-language Pre-training\nVision-language pre-training (VLP) aims to improve per-\nformance of downstream vision and language tasks by pre-\ntraining the model on large-scale image-text pairs. Due to\nthe prohibitive expense of acquiring human-annotated texts,\nmost methods (Chen et al., 2020; Li et al., 2020; 2021a;\nWang et al., 2021; Radford et al., 2021) use image and"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 4
encoding: FLOAT32
metadata:
{"chroma:document": "alt-text pairs crawled from the web (Sharma et al., 2018;\nChangpinyo et al., 2021; Jia et al., 2021), Despite the use of\nsimple rule-based \ufb01lters, noise is still prevalent in the web\ntexts. However, the negative impact of the noise has been\nlargely overlooked, shadowed by the performance gain ob-\ntained from scaling up the dataset. Our paper shows that the\nnoisy web texts are suboptimal for vision-language learning,\nand proposes CapFilt that utilizes web datasets in a more\neffective way.\nThere have been many attempts to unify various vision\nand language tasks into a single framework (Zhou et al.,\n2020; Cho et al., 2021; Wang et al., 2021). The biggest\nchallenge is to design model architectures that can perform\nboth understanding-based tasks (e.g. image-text retrieval)\nand generation-based tasks (e.g. image captioning). Neither\nencoder-based models (Li et al., 2021a;b; Radford et al.,\n2021) nor encoder-decoder models (Cho et al., 2021; Wang\net al., 2021) can excel at both types of tasks, whereas a single\nuni\ufb01ed encoder-decoder (Zhou et al., 2020) also limits the\nmodel\u2019s capability. Our proposed multimodal mixture of\nencoder-decoder model offers more \ufb02exibility and better\nperformance on a wide range of downstream tasks, in the\nmeantime keeping the pre-training simple and ef\ufb01cient.\n2.2. Knowledge Distillation\nKnowledge distillation (KD) (Hinton et al., 2015) aims to\nimprove the performance of a student model by distilling\nknowledge from a teacher model. Self-distillation is a spe-\ncial case of KD where the teacher and student have equal\nsizes. It has been shown to be effective for image classi-\n\ufb01cation (Xie et al., 2020), and recently for VLP (Li et al.,\n2021a). Different from mostly existing KD methods which\nsimply enforce the student to have the same class predic-\ntions as the teacher, our proposed CapFilt can be interpreted\nas a more effective way to perform KD in the context of\nVLP, where the captioner distills its knowledge through\nsemantically-rich synthetic captions, and the \ufb01lter distills"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 5
encoding: FLOAT32
metadata:
{"chroma:document": "its knowledge by removing noisy captions.\n2.3. Data Augmentation\nWhile data augmentation (DA) has been widely adopted in\ncomputer vision (Shorten & Khoshgoftaar, 2019), DA for\nlanguage tasks is less straightforward. Recently, generative\nlanguage models have been used to synthesize examples\nfor various NLP tasks (Kumar et al., 2020; Anaby-Tavor"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 6
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\net al., 2020; Puri et al., 2020; Yang et al., 2020). Differ-\nent from these methods which focus on the low-resource\nlanguage-only tasks, our method demonstrates the advan-\ntage of synthetic captions in large-scale vision-language\npre-training.\n3. Method\nWe propose BLIP, a uni\ufb01ed VLP framework to learn from\nnoisy image-text pairs. This section \ufb01rst introduces our new\nmodel architecture MED and its pre-training objectives, and\nthen delineates CapFilt for dataset bootstrapping.\n3.1. Model Architecture\nWe employ a visual transformer (Dosovitskiy et al., 2021)\nas our image encoder, which divides an input image into\npatches and encodes them as a sequence of embeddings,\nwith an additional [CLS] token to represent the global im-\nage feature. Compared to using pre-trained object detectors\nfor visual feature extraction (Chen et al., 2020), using a ViT\nis more computation-friendly and has been adopted by the\nmore recent methods (Li et al., 2021a; Kim et al., 2021).\nIn order to pre-train a uni\ufb01ed model with both understanding\nand generation capabilities, we propose multimodal mixture\nof encoder-decoder (MED), a multi-task model which can\noperate in one of the three functionalities:\n(1) Unimodal encoder, which separately encodes image\nand text. The text encoder is the same as BERT (Devlin et al.,\n2019), where a [CLS] token is appended to the beginning\nof the text input to summarize the sentence.\n(2) Image-grounded text encoder, which injects visual\ninformation by inserting one additional cross-attention (CA)\nlayer between the self-attention (SA) layer and the feed\nforward network (FFN) for each transformer block of the\ntext encoder. A task-speci\ufb01c [Encode] token is appended\nto the text, and the output embedding of [Encode] is used\nas the multimodal representation of the image-text pair.\n(3) Image-grounded text decoder, which replaces the bi-\ndirectional self-attention layers in the image-grounded text"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 7
encoding: FLOAT32
metadata:
{"chroma:document": "encoder with causal self-attention layers. A [Decode]\ntoken is used to signal the beginning of a sequence, and an\nend-of-sequence token is used to signal its end.\n3.2. Pre-training Objectives\nWe jointly optimize three objectives during pre-training,\nwith two understanding-based objectives and one generation-\nbased objective. Each image-text pair only requires one for-\nward pass through the computational-heavier visual trans-\nformer, and three forward passes through the text trans-\nformer, where different functionalities are activated to com-\npute the three losses as delineated below.\nImage-Text Contrastive Loss (ITC) activates the unimodal\nencoder. It aims to align the feature space of the visual trans-\nformer and the text transformer by encouraging positive\nimage-text pairs to have similar representations in contrast\nto the negative pairs. It has been shown to be an effective\nobjective for improving vision and language understand-\ning (Radford et al., 2021; Li et al., 2021a). We follow the\nITC loss by Li et al. (2021a), where a momentum encoder\nis introduced to produce features, and soft labels are created\nfrom the momentum encoder as training targets to account\nfor the potential positives in the negative pairs.\nImage-Text Matching Loss (ITM) activates the image-\ngrounded text encoder. It aims to learn image-text mul-\ntimodal representation that captures the \ufb01ne-grained align-\nment between vision and language. ITM is a binary clas-\nsi\ufb01cation task, where the model uses an ITM head (a lin-\near layer) to predict whether an image-text pair is positive\n(matched) or negative (unmatched) given their multimodal\nfeature. In order to \ufb01nd more informative negatives, we\nadopt the hard negative mining strategy by Li et al. (2021a),\nwhere negatives pairs with higher contrastive similarity in a\nbatch are more likely to be selected to compute the loss.\nLanguage Modeling Loss (LM) activates the image-\ngrounded text decoder, which aims to generate textual de-\nscriptions given an image. It optimizes a cross entropy loss"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 8
encoding: FLOAT32
metadata:
{"chroma:document": "which trains the model to maximize the likelihood of the\ntext in an autoregressive manner. We apply a label smooth-\ning of 0.1 when computing the loss. Compared to the MLM\nloss that has been widely-used for VLP, LM enables the\nmodel with the generalization capability to convert visual\ninformation into coherent captions.\nIn order to perform ef\ufb01cient pre-training while leveraging\nmulti-task learning, the text encoder and text decoder share\nall parameters except for the SA layers. The reason is that\nthe differences between the encoding and decoding tasks are\nbest captured by the SA layers. In particular, the encoder\nemploys bi-directional self-attention to build representations\nfor the current input tokens, while the decoder employs\ncausal self-attention to predict next tokens. On the other\nhand, the embedding layers, CA layers and FFN function\nsimilarly between encoding and decoding tasks, therefore\nsharing these layers can improve training ef\ufb01ciency while\nbene\ufb01ting from multi-task learning,\n3.3. CapFilt\nDue to the prohibitive annotation cost, there exist a lim-\nited number of high-quality human-annotated image-text\npairs {(Ih, Th)} (e.g., COCO (Lin et al., 2014)). Recent\nwork (Li et al., 2021a; Wang et al., 2021) utilizes a much\nlarger number of image and alt-text pairs {(Iw, Tw)} that\nare automatically collected from the web. However, the\nalt-texts often do not accurately describe the visual content\nof the images, making them a noisy signal that is suboptimal\nfor learning vision-language alignment."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 9
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nMultimodal Mixture of \nEncoder-Decoder\n\ud835\udc37 =\n\ud835\udc3c!, \ud835\udc47!\n+\n\ud835\udc3c\", \ud835\udc47\"\nPre-train\nFilter \n(Image-grounded \nText Encoder)\nCaptioner \n(Image-grounded \nText Decoder)\nITC&ITM finetune\nLM finetune\n\ud835\udc3c\", \ud835\udc47\"\n\ud835\udc3c!, \ud835\udc47#\n{\ud835\udc3c!}\nCaptioning\n\ud835\udc3c!, \ud835\udc47!\n\ud835\udc3c!, \ud835\udc47!\n+\n\ud835\udc3c!, \ud835\udc47#\nFiltering\nDataset Bootstrapping\nModel Pretraining\n\ud835\udc37 =\n\ud835\udc3c!, \ud835\udc47!\n+\n\ud835\udc3c!, \ud835\udc47#\n+\n\ud835\udc3c\", \ud835\udc47\"\nDownstream Tasks\nTo model\nTo data\n\ud835\udc3c!: web images\n\ud835\udc3c\": human-annotated\nimages\n\ud835\udc47!: web texts\n\ud835\udc47!: filtered web texts\n\ud835\udc47#: synthetic texts\n\ud835\udc47#: filtered synthetic\ntexts\n\ud835\udc47\": human-annotated\ntexts\nFigure 3. Learning framework of BLIP. We introduce a captioner to produce synthetic captions for web images, and a \ufb01lter to remove\nnoisy image-text pairs. The captioner and \ufb01lter are initialized from the same pre-trained model and \ufb01netuned individually on a small-scale\nhuman-annotated dataset. The bootstrapped dataset is used to pre-train a new model.\nWe propose Captioning and Filtering (CapFilt), a new\nmethod to improve the quality of the text corpus. Figure 3\ngives an illustration of CapFilt. It introduces two modules:\na captioner to generate captions given web images, and a\n\ufb01lter to remove noisy image-text pairs. Both the captioner\nand the \ufb01lter are initialized from the same pre-trained MED\nmodel, and \ufb01netuned individually on the COCO dataset.\nThe \ufb01netuning is a lightweight procedure.\nSpeci\ufb01cally, the captioner is an image-grounded text de-\ncoder. It is \ufb01netuned with the LM objective to decode texts\ngiven images. Given the web images Iw, the captioner gen-\nerates synthetic captions Ts with one caption per image.\nThe \ufb01lter is an image-grounded text encoder. It is \ufb01netuned\nwith the ITC and ITM objectives to learn whether a text\nmatches an image. The \ufb01lter removes noisy texts in both\nthe original web texts Tw and the synthetic texts Ts, where\na text is considered to be noisy if the ITM head predicts it\nas unmatched to the image. Finally, we combine the \ufb01ltered\nimage-text pairs with the human-annotated pairs to form a"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 10
encoding: FLOAT32
metadata:
{"chroma:document": "new dataset, which we use to pre-train a new model.\n4. Experiments and Discussions\nIn this section, we \ufb01rst introduce pre-training details. Then\nwe provide a detailed experimental analysis on our method.\n4.1. Pre-training Details\nOur models are implemented in PyTorch (Paszke et al.,\n2019) and pre-trained on two 16-GPU nodes.\nThe im-\nage transformer is initialized from ViT pre-trained on Ima-\ngeNet (Touvron et al., 2020; Dosovitskiy et al., 2021), and\nthe text transformer is initialized from BERTbase (Devlin\net al., 2019). We explore two variants of ViTs: ViT-B/16\nand ViT-L/16. Unless otherwise speci\ufb01ed, all results re-\nported in this paper as \u201cBLIP\u201d uses ViT-B. We pre-train the\nmodel for 20 epochs using a batch size of 2880 (ViT-B) /\n2400 (ViT-L). We use AdamW (Loshchilov & Hutter, 2017)\noptimizer with a weight decay of 0.05. The learning rate\nis warmed-up to 3e-4 (ViT-B) / 2e-4 (ViT-L) and decayed\nlinearly with a rate of 0.85. We take random image crops of\nresolution 224 \u00d7 224 during pre-training, and increase the\nimage resolution to 384 \u00d7 384 during \ufb01netuning. We use\nthe same pre-training dataset as Li et al. (2021a) with 14M\nimages in total, including two human-annotated datasets\n(COCO and Visual Genome (Krishna et al., 2017)), and\nthree web datasets (Conceptual Captions (Changpinyo et al.,\n2021), Conceptual 12M (Changpinyo et al., 2021), SBU cap-\ntions (Ordonez et al., 2011)). We also experimented with an\nadditional web dataset, LAION (Schuhmann et al., 2021),\nwhich contains 115M images with more noisy texts1. More\ndetails about the datasets can be found in the appendix.\n4.2. Effect of CapFilt\nIn Table 1, we compare models pre-trained on different\ndatasets to demonstrate the ef\ufb01cacy of CapFilt on down-\nstream tasks, including image-text retrieval and image cap-\ntioning with \ufb01netuned and zero-shot settings.\nWhen only the captioner or the \ufb01lter is applied to the dataset\nwith 14M images, performance improvement can be ob-\nserved. When applied together, their effects compliment"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 11
encoding: FLOAT32
metadata:
{"chroma:document": "each other, leading to substantial improvements compared\nto using the original noisy web texts.\nCapFilt can further boost performance with a larger dataset\nand a larger vision backbone, which veri\ufb01es its scalability\nin both the data size and the model size. Furthermore, by\nusing a large captioner and \ufb01lter with ViT-L, performance\nof the base model can also be improved.\n1We only download images whose shorter edge is larger than\n256 pixels from the original LAION400M. Due to the large size of\nLAION, we only use 1/5 of it each epoch during pre-training."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 12
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nPre-train\ndataset\nBootstrap\nVision\nbackbone\nRetrieval-FT (COCO)\nRetrieval-ZS (Flickr)\nCaption-FT (COCO)\nCaption-ZS (NoCaps)\nC\nF\nTR@1\nIR@1\nTR@1\nIR@1\nB@4\nCIDEr\nCIDEr\nSPICE\nCOCO+VG\n+CC+SBU\n(14M imgs)\n\u0017\n\u0017\nViT-B/16\n78.4\n60.7\n93.9\n82.1\n38.0\n127.8\n102.2\n13.9\n\u0017\n\u0013B\n79.1\n61.5\n94.1\n82.8\n38.1\n128.2\n102.7\n14.0\n\u0013B\n\u0017\n79.7\n62.0\n94.4\n83.6\n38.4\n128.9\n103.4\n14.2\n\u0013B\n\u0013B\n80.6\n63.1\n94.8\n84.9\n38.6\n129.7\n105.1\n14.4\nCOCO+VG\n+CC+SBU\n+LAION\n(129M imgs)\n\u0017\n\u0017\nViT-B/16\n79.6\n62.0\n94.3\n83.6\n38.8\n130.1\n105.4\n14.2\n\u0013B\n\u0013B\n81.9\n64.3\n96.0\n85.0\n39.4\n131.4\n106.3\n14.3\n\u0013L\n\u0013L\n81.2\n64.1\n96.0\n85.5\n39.7\n133.3\n109.6\n14.7\n\u0017\n\u0017\nViT-L/16\n80.6\n64.1\n95.1\n85.5\n40.3\n135.5\n112.5\n14.7\n\u0013L\n\u0013L\n82.4\n65.1\n96.7\n86.7\n40.4\n136.7\n113.2\n14.8\nTable 1. Evaluation of the effect of the captioner (C) and \ufb01lter (F) for dataset bootstrapping. Downstream tasks include image-text retrieval\nand image captioning with \ufb01netuning (FT) and zero-shot (ZS) settings. TR / IR@1: recall@1 for text retrieval / image retrieval. \u0013B/L:\ncaptioner or \ufb01lter uses ViT-B / ViT-L as vision backbone.\n\ud835\udc47!: \u201cfrom bridge \nnear my house\u201d\n\ud835\udc47\": \u201ca flock of birds \nflying over a lake at \nsunset\u201d\n\ud835\udc47!: \u201cin front of a house \ndoor in Reichenfels, \nAustria\u201d \n\ud835\udc47\": \u201ca potted plant sitting \non top of a pile of rocks\u201d\n\ud835\udc47!: \u201cthe current castle was \nbuilt in 1180, replacing a 9th \ncentury wooden castle\u201d\n\ud835\udc47\": \u201ca large building with a lot \nof windows on it\u201d \nFigure 4. Examples of the web text Tw and the synthetic text Ts. Green texts are accepted by the \ufb01lter, whereas red texts are rejected.\nGeneration\nmethod\nNoise\nratio\nRetrieval-FT (COCO)\nRetrieval-ZS (Flickr)\nCaption-FT (COCO)\nCaption-ZS (NoCaps)\nTR@1\nIR@1\nTR@1\nIR@1\nB@4\nCIDEr\nCIDEr\nSPICE\nNone\nN.A.\n78.4\n60.7\n93.9\n82.1\n38.0\n127.8\n102.2\n13.9\nBeam\n19%\n79.6\n61.9\n94.1\n83.1\n38.4\n128.9\n103.5\n14.2\nNucleus\n25%\n80.6\n63.1\n94.8\n84.9\n38.6\n129.7\n105.1\n14.4\nTable 2. Comparison between beam search and nucleus sampling for synthetic caption generation. Models are pre-trained on 14M images."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 13
encoding: FLOAT32
metadata:
{"chroma:document": "Layers shared\n#parameters\nRetrieval-FT (COCO)\nRetrieval-ZS (Flickr)\nCaption-FT (COCO)\nCaption-ZS (NoCaps)\nTR@1\nIR@1\nTR@1\nIR@1\nB@4\nCIDEr\nCIDEr\nSPICE\nAll\n224M\n77.3\n59.5\n93.1\n81.0\n37.2\n125.9\n100.9\n13.1\nAll except CA\n252M\n77.5\n59.9\n93.1\n81.3\n37.4\n126.1\n101.2\n13.1\nAll except SA\n252M\n78.4\n60.7\n93.9\n82.1\n38.0\n127.8\n102.2\n13.9\nNone\n361M\n78.3\n60.5\n93.6\n81.9\n37.8\n127.4\n101.8\n13.9\nTable 3. Comparison between different parameter sharing strategies for the text encoder and decoder during pre-training.\nIn Figure 4, we show some example captions and their\ncorresponding images, which qualitatively demonstrate the\neffect of the captioner to generate new textual descriptions,\nand the \ufb01lter to remove noisy captions from both the original\nweb texts and the synthetic texts. More examples can be\nfound in the appendix.\n4.3. Diversity is Key for Synthetic Captions\nIn CapFilt, we employ nucleus sampling (Holtzman et al.,\n2020) to generate synthetic captions. Nucleus sampling is a\nstochastic decoding method, where each token is sampled\nfrom a set of tokens whose cumulative probability mass\nexceeds a threshold p (p = 0.9 in our experiments). In\nTable 2, we compare it with beam search, a deterministic\ndecoding method which aims to generate captions with the\nhighest probability. Nucleus sampling leads to evidently\nbetter performance, despite being more noisy as suggested\nby a higher noise ratio from the \ufb01lter. We hypothesis that the\nreason is that nucleus sampling generates more diverse and\nsurprising captions, which contain more new information\nthat the model could bene\ufb01t from. On the other hand, beam\nsearch tends to generate safe captions that are common in\nthe dataset, hence offering less extra knowledge.\n4.4. Parameter Sharing and Decoupling\nDuring pre-training, the text encoder and decoder share all\nparameters except for the self-attention layers. In Table 3,\nwe evaluate models pre-trained with different parameter\nsharing strategies, where pre-training is performed on the\n14M images with web texts. As the result shows, sharing all"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 14
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nCaptioner &\nFilter\nNoise\nratio\nRetrieval-FT (COCO)\nRetrieval-ZS (Flickr)\nCaption-FT (COCO)\nCaption-ZS (NoCaps)\nTR@1\nIR@1\nTR@1\nIR@1\nB@4\nCIDEr\nCIDEr\nSPICE\nShare parameters\n8%\n79.8\n62.2\n94.3\n83.7\n38.4\n129.0\n103.5\n14.2\nDecoupled\n25%\n80.6\n63.1\n94.8\n84.9\n38.6\n129.7\n105.1\n14.4\nTable 4. Effect of sharing parameters between the captioner and \ufb01lter. Models are pre-trained on 14M images.\nMethod\nPre-train\nCOCO (5K test set)\nFlickr30K (1K test set)\n# Images\nTR\nIR\nTR\nIR\nR@1\nR@5\nR@10\nR@1\nR@5\nR@10\nR@1\nR@5\nR@10\nR@1\nR@5\nR@10\nUNITER (Chen et al., 2020)\n4M\n65.7\n88.6\n93.8\n52.9\n79.9\n88.0\n87.3\n98.0\n99.2\n75.6\n94.1\n96.8\nVILLA (Gan et al., 2020)\n4M\n-\n-\n-\n-\n-\n-\n87.9\n97.5\n98.8\n76.3\n94.2\n96.8\nOSCAR (Li et al., 2020)\n4M\n70.0\n91.1\n95.5\n54.0\n80.8\n88.5\n-\n-\n-\n-\n-\n-\nUNIMO (Li et al., 2021b)\n5.7M\n-\n-\n-\n-\n-\n-\n89.4\n98.9\n99.8\n78.0\n94.2\n97.1\nALIGN (Jia et al., 2021)\n1.8B\n77.0\n93.5\n96.9\n59.9\n83.3\n89.8\n95.3\n99.8\n100.0\n84.9\n97.4\n98.6\nALBEF (Li et al., 2021a)\n14M\n77.6\n94.3\n97.2\n60.7\n84.3\n90.5\n95.9\n99.8\n100.0\n85.6\n97.5\n98.9\nBLIP\n14M\n80.6\n95.2\n97.6\n63.1\n85.3\n91.1\n96.6\n99.8\n100.0\n87.2\n97.5\n98.8\nBLIP\n129M\n81.9\n95.4\n97.8\n64.3\n85.7\n91.5\n97.3\n99.9\n100.0\n87.3\n97.6\n98.9\nBLIPCapFilt-L\n129M\n81.2\n95.7\n97.9\n64.1\n85.8\n91.6\n97.2\n99.9\n100.0\n87.5\n97.7\n98.9\nBLIPViT-L\n129M\n82.4\n95.4\n97.9\n65.1\n86.3\n91.8\n97.4\n99.8\n99.9\n87.6\n97.7\n99.0\nTable 5. Comparison with state-of-the-art image-text retrieval methods, \ufb01netuned on COCO and Flickr30K datasets. BLIPCapFilt-L pre-trains\na model with ViT-B backbone using a dataset bootstrapped by captioner and \ufb01lter with ViT-L.\nMethod\nPre-train\nFlickr30K (1K test set)\n# Images\nTR\nIR\nR@1 R@5 R@10 R@1 R@5 R@10\nCLIP\n400M\n88.0\n98.7\n99.4\n68.7\n90.6\n95.2\nALIGN\n1.8B\n88.6\n98.7\n99.7\n75.7\n93.8\n96.8\nALBEF\n14M\n94.1\n99.5\n99.7\n82.8\n96.3\n98.1\nBLIP\n14M\n94.8\n99.7\n100.0\n84.9\n96.7\n98.3\nBLIP\n129M\n96.0\n99.9\n100.0\n85.0\n96.8\n98.6\nBLIPCapFilt-L 129M\n96.0\n99.9\n100.0\n85.5\n96.8\n98.7\nBLIPViT-L\n129M\n96.7\n100.0\n100.0\n86.7\n97.3\n98.7"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 15
encoding: FLOAT32
metadata:
{"chroma:document": "97.3\n98.7\nTable 6. Zero-shot image-text retrieval results on Flickr30K.\nlayers except for SA leads to better performance compared\nto not sharing, while also reducing the model size thus\nimproveing training ef\ufb01ciency. If the SA layers are shared,\nthe model\u2019s performance would degrade due to the con\ufb02ict\nbetween the encoding task and the decoding task.\nDuring CapFilt, the captioner and the \ufb01lter are end-to-end\n\ufb01netuned individually on COCO. In Table 4, we study the\neffect if the captioner and \ufb01lter share parameters in the same\nway as pre-training. The performance on the downstream\ntasks decreases, which we mainly attribute to con\ufb01rmation\nbias. Due to parameter sharing, noisy captions produced by\nthe captioner are less likely to be \ufb01ltered out by the \ufb01lter, as\nindicated by the lower noise ratio (8% compared to 25%).\n5. Comparison with State-of-the-arts\nIn this section, we compare BLIP to existing VLP methods\non a wide range of vision-language downstream tasks2. Next\n2we omit SNLI-VE from the benchmark because its test data\nhas been reported to be noisy (Do et al., 2020)\nwe brie\ufb02y introduce each task and \ufb01netuning strategy. More\ndetails can be found in the appendix.\n5.1. Image-Text Retrieval\nWe evaluate BLIP for both image-to-text retrieval (TR) and\ntext-to-image retrieval (IR) on COCO and Flickr30K (Plum-\nmer et al., 2015) datasets. We \ufb01netune the pre-trained model\nusing ITC and ITM losses. To enable faster inference speed,\nwe follow Li et al. (2021a) and \ufb01rst select k candidates\nbased on the image-text feature similarity, and then rerank\nthe selected candidates based on their pairwise ITM scores.\nWe set k = 256 for COCO and k = 128 for Flickr30K.\nAs shown in Table 5, BLIP achieves substantial performance\nimprovement compared with existing methods. Using the\nsame 14M pre-training images, BLIP outperforms the pre-\nvious best model ALBEF by +2.7% in average recall@1\non COCO. We also perform zero-shot retrieval by directly\ntransferring the model \ufb01netuned on COCO to Flickr30K."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 16
encoding: FLOAT32
metadata:
{"chroma:document": "The result is shown in Table 6, where BLIP also outperforms\nexisting methods by a large margin.\n5.2. Image Captioning\nWe consider two datasets for image captioning:\nNo-\nCaps (Agrawal et al., 2019) and COCO, both evaluated\nusing the model \ufb01netuned on COCO with the LM loss. Sim-\nilar as Wang et al. (2021), we add a prompt \u201ca picture of\u201d\nat the beginning of each caption, which leads to slightly\nbetter results. As shown in Table 7, BLIP with 14M pre-\ntraining images substantially outperforms methods using\na similar amount of pre-training data. BLIP with 129M\nimages achieves competitive performance as LEMON with"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 17
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nMethod\nPre-train\n#Images\nNoCaps validation\nCOCO Caption\nin-domain\nnear-domain\nout-domain\noverall\nKarpathy test\nC\nS\nC\nS\nC\nS\nC\nS\nB@4\nC\nEnc-Dec (Changpinyo et al., 2021)\n15M\n92.6\n12.5\n88.3\n12.1\n94.5\n11.9\n90.2\n12.1\n-\n110.9\nVinVL\u2020 (Zhang et al., 2021)\n5.7M\n103.1\n14.2\n96.1\n13.8\n88.3\n12.1\n95.5\n13.5\n38.2\n129.3\nLEMONbase\u2020 (Hu et al., 2021)\n12M\n104.5\n14.6\n100.7\n14.0\n96.7\n12.4\n100.4\n13.8\n-\n-\nLEMONbase\u2020 (Hu et al., 2021)\n200M\n107.7\n14.7\n106.2\n14.3\n107.9\n13.1\n106.8\n14.1\n40.3\n133.3\nBLIP\n14M\n111.3\n15.1\n104.5\n14.4\n102.4\n13.7\n105.1\n14.4\n38.6\n129.7\nBLIP\n129M\n109.1\n14.8\n105.8\n14.4\n105.7\n13.7\n106.3\n14.3\n39.4\n131.4\nBLIPCapFilt-L\n129M\n111.8\n14.9\n108.6\n14.8\n111.5\n14.2\n109.6\n14.7\n39.7\n133.3\nLEMONlarge\u2020 (Hu et al., 2021)\n200M\n116.9\n15.8\n113.3\n15.1\n111.3\n14.0\n113.4\n15.0\n40.6\n135.7\nSimVLMhuge (Wang et al., 2021)\n1.8B\n113.7\n-\n110.9\n-\n115.2\n-\n112.2\n-\n40.6\n143.3\nBLIPViT-L\n129M\n114.9\n15.2\n112.1\n14.9\n115.3\n14.4\n113.2\n14.8\n40.4\n136.7\nTable 7. Comparison with state-of-the-art image captioning methods on NoCaps and COCO Caption. All methods optimize the cross-\nentropy loss during \ufb01netuning. C: CIDEr, S: SPICE, B@4: BLEU@4. BLIPCapFilt-L is pre-trained on a dataset bootstrapped by captioner\nand \ufb01lter with ViT-L. VinVL\u2020 and LEMON\u2020 require an object detector pre-trained on 2.5M images with human-annotated bounding\nboxes and high resolution (800\u00d71333) input images. SimVLMhuge uses 13\u00d7 more training data and a larger vision backbone than ViT-L.\nImage \nEncoder\nQuestion \nEncoder\nImage\n\u201c[Encode] + Q \u201d\nAnswer \nDecoder\n\u201c[Decode]\u201d  \nanswer\n(a) VQA\nCross \nAttention\nImage #1\nMerge Layer\nCross \nAttention\nImage \nEncoder\nImage \nEncoder\ntrue/false\nN\u00d7\n\u201c[Encode] + Text \u201d\nImage #2\n\u2026\n\u2026\n(b) NLVR!\nImage \nEncoder\nCaption\nEncoder\nImage\n\u201c[Encode] + C \u201d\nDialog \nEncoder\n\u201c[Encode] + QA + Dialog History\u201d  \ntrue/false\n(c) VisDial\nFigure 5. Model architecture for the downstream tasks. Q: ques-\ntion; C: caption; QA: question-answer pair."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 18
encoding: FLOAT32
metadata:
{"chroma:document": "200M images. Note that LEMON requires a computational-\nheavy pre-trained object detector and higher resolution\n(800\u00d71333) input images, leading to substantially slower\ninference time than the detector-free BLIP which uses lower\nresolution (384\u00d7384) input images.\n5.3. Visual Question Answering (VQA)\nVQA (Antol et al., 2015) requires the model to predict an an-\nswer given an image and a question. Instead of formulating\nVQA as a multi-answer classi\ufb01cation task (Chen et al., 2020;\nMethod\nPre-train\n#Images\nVQA\nNLVR2\ntest-dev\ntest-std\ndev\ntest-P\nLXMERT\n180K\n72.42\n72.54\n74.90\n74.50\nUNITER\n4M\n72.70\n72.91\n77.18\n77.85\nVL-T5/BART\n180K\n-\n71.3\n-\n73.6\nOSCAR\n4M\n73.16\n73.44\n78.07\n78.36\nSOHO\n219K\n73.25\n73.47\n76.37\n77.32\nVILLA\n4M\n73.59\n73.67\n78.39\n79.30\nUNIMO\n5.6M\n75.06\n75.27\n-\n-\nALBEF\n14M\n75.84\n76.04\n82.55\n83.14\nSimVLMbase\u2020\n1.8B\n77.87\n78.14\n81.72\n81.77\nBLIP\n14M\n77.54\n77.62\n82.67\n82.30\nBLIP\n129M\n78.24\n78.17\n82.48\n83.08\nBLIPCapFilt-L\n129M\n78.25\n78.32\n82.15\n82.24\nTable 8. Comparison with state-of-the-art methods on VQA and\nNLVR2. ALBEF performs an extra pre-training step for NLVR2.\nSimVLM\u2020 uses 13\u00d7 more training data and a larger vision back-\nbone (ResNet+ViT) than BLIP.\nLi et al., 2020), we follow Li et al. (2021a) and consider it as\nan answer generation task, which enables open-ended VQA.\nAs shown in Figure 5(a), during \ufb01netuning, we rearrange the\npre-trained model, where an image-question is \ufb01rst encoded\ninto multimodal embeddings and then given to an answer\ndecoder. The VQA model is \ufb01netuned with the LM loss\nusing ground-truth answers as targets.\nThe results are shown in Table 8.\nUsing 14M images,\nBLIP outperforms ALBEF by +1.64% on the test set. Us-\ning 129M images, BLIP achieves better performance than\nSimVLM which uses 13\u00d7 more pre-training data and a\nlarger vision backbone with an additional convolution stage.\n5.4. Natural Language Visual Reasoning (NLVR2)\nNLVR2 (Suhr et al., 2019) asks the model to predict whether\na sentence describes a pair of images. In order to enable rea-"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 19
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nMethod\nMRR\u2191\nR@1\u2191\nR@5\u2191\nR@10\u2191\nMR\u2193\nVD-BERT\n67.44\n54.02\n83.96\n92.33\n3.53\nVD-ViLBERT\u2020\n69.10\n55.88\n85.50\n93.29\n3.25\nBLIP\n69.41\n56.44\n85.90\n93.30\n3.20\nTable 9. Comparison with state-of-the-art methods on VisDial v1.0\nvalidation set. VD-ViLBERT\u2020 (Murahari et al., 2020) pre-trains\nViLBERT (Lu et al., 2019) with additional VQA data.\nsoning over two images, we make a simple modi\ufb01cation to\nour pre-trained model which leads to a more computational-\nef\ufb01cient architecture than previous approaches (Li et al.,\n2021a; Wang et al., 2021). As shown in Figure 5(b), for\neach transformer block in the image-grounded text encoder,\nthere exist two cross-attention layers to process the two in-\nput images, and their outputs are merged and fed to the FFN.\nThe two CA layers are intialized from the same pre-trained\nweights. The merge layer performs simple average pooling\nin the \ufb01rst 6 layers of the encoder, and performs concate-\nnation followed by a linear projection in layer 6-12. An\nMLP classi\ufb01er is applied on the output embedding of the\n[Encode] token. As shown in Table 8, BLIP outperforms\nall existing methods except for ALBEF which performs an\nextra step of customized pre-training. Interestingly, perfor-\nmance on NLVR2 does not bene\ufb01t much from additional\nweb images, possibly due to the domain gap between web\ndata and downstream data.\n5.5. Visual Dialog (VisDial)\nVisDial (Das et al., 2017) extends VQA in a natural con-\nversational setting, where the model needs to predict an\nanswer not only based on the image-question pair, but also\nconsidering the dialog history and the image\u2019s caption. We\nfollow the discriminative setting where the model ranks a\npool of answer candidates (Gan et al., 2019; Wang et al.,\n2020; Murahari et al., 2020). As shown in Figure 5(c), we\nconcatenate image and caption embeddings, and pass them\nto the dialog encoder through cross-attention. The dialog\nencoder is trained with the ITM loss to discriminate whether"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 20
encoding: FLOAT32
metadata:
{"chroma:document": "the answer is true or false for a question, given the entire dia-\nlog history and the image-caption embeddings. As shown in\nTable 9, our method achieves state-of-the-art performance\non VisDial v1.0 validation set.\n5.6. Zero-shot Transfer to Video-Language Tasks\nOur image-language model has strong generalization ability\nto video-language tasks. In Table 10 and Table 11, we per-\nform zero-shot transfer to text-to-video retrieval and video\nquestion answering, where we directly evaluate the models\ntrained on COCO-retrieval and VQA, respectively. To pro-\ncess video input, we uniformly sample n frames per video\n(n = 8 for retrieval and n = 16 for QA), and concatenate\nthe frame features into a single sequence. Note that this\nsimple approach ignores all temporal information.\nMethod\nR1\u2191\nR5\u2191\nR10\u2191\nMdR\u2193\nzero-shot\nActBERT (Zhu & Yang, 2020)\n8.6\n23.4\n33.1\n36\nSupportSet (Patrick et al., 2021)\n8.7\n23.0\n31.1\n31\nMIL-NCE (Miech et al., 2020)\n9.9\n24.0\n32.4\n29.5\nVideoCLIP (Xu et al., 2021)\n10.4\n22.2\n30.0\n-\nFiT (Bain et al., 2021)\n18.7\n39.5\n51.6\n10\nBLIP\n43.3\n65.6\n74.7\n2\n\ufb01netuning\nClipBERT (Lei et al., 2021)\n22.0\n46.8\n59.9\n6\nVideoCLIP (Xu et al., 2021)\n30.9\n55.4\n66.8\n-\nTable 10. Comparisons with state-of-the-art methods for text-to-\nvideo retrieval on the 1k test split of the MSRVTT dataset.\nMethod\nMSRVTT-QA\nMSVD-QA\nzero-shot\nVQA-T (Yang et al., 2021)\n2.9\n7.5\nBLIP\n19.2\n35.2\n\ufb01netuning\nHME (Fan et al., 2019)\n33.0\n33.7\nHCRN (Le et al., 2020)\n35.6\n36.1\nVQA-T (Yang et al., 2021)\n41.5\n46.3\nTable 11. Comparisons with state-of-the-art methods for video\nquestion answering. We report top-1 test accuracy on two datasets.\nDespite the domain difference and lack of temporal mod-\neling, our models achieve state-of-the-art performance on\nboth video-language tasks. For text-to-video retrieval, zero-\nshot BLIP even outperforms models \ufb01netuned on the target\nvideo dataset by +12.4% in recall@1. Further performance\nimprovement can be achieved if the BLIP model is used to\ninitialize a video-language model with temporal modeling"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 21
encoding: FLOAT32
metadata:
{"chroma:document": "(e.g. replace our ViT with a TimeSformer (Bertasius et al.,\n2021)) and \ufb01netuned on video data.\n6. Additional Ablation Study\nIn this section, we provide additional ablation experiments\non CapFilt.\nImprovement with CapFilt is not due to longer training.\nSince the bootstrapped dataset contains more texts than the\noriginal dataset, training for the same number of epochs\ntakes longer with the bootstrapped dataset. To verify that\nthe effectiveness of CapFilt is not due to longer training,\nwe replicate the web text in the original dataset so that it\nhas the same number of training samples per epoch as the\nbootstrapped dataset. As shown in Table 12, longer training\nusing the noisy web texts does not improve performance.\nA new model should be trained on the bootstrapped\ndataset. The bootstrapped dataset is used to pre-train a\nnew model. We investigate the effect of continue training"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 22
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nCapFilt\n#Texts\nRetrieval-FT (COCO)\nRetrieval-ZS (Flickr)\nCaption-FT (COCO)\nCaption-ZS (NoCaps)\nTR@1\nIR@1\nTR@1\nIR@1\nB@4\nCIDEr\nCIDEr\nSPICE\nNo\n15.3M\n78.4\n60.7\n93.9\n82.1\n38.0\n127.8\n102.2\n13.9\nNo\n24.7M\n78.3\n60.5\n93.7\n82.2\n37.9\n127.7\n102.1\n14.0\nYes\n24.7M\n80.6\n63.1\n94.8\n84.9\n38.6\n129.7\n105.1\n14.4\nTable 12. The original web texts are replicated to have the same number of samples per epoch as the bootstrapped dataset. Results verify\nthat the improvement from CapFilt is not due to longer training time.\nContinue\nRetrieval-FT (COCO)\nRetrieval-ZS (Flickr)\nCaption-FT (COCO)\nCaption-ZS (NoCaps)\nTR@1\nIR@1\nTR@1\nIR@1\nB@4\nCIDEr\nCIDEr\nSPICE\nYes\n80.6\n63.0\n94.5\n84.6\n38.5\n129.9\n104.5\n14.2\nNo\n80.6\n63.1\n94.8\n84.9\n38.6\n129.7\n105.1\n14.4\nTable 13. Continue training the pre-trained model offers less gain compared to training a new model with the bootstrapped dataset.\nfrom the previous pre-trained model, using the bootstrapped\ndataset. Table 13 hows that continue training does not help.\nThis observation agrees with the common practice in knowl-\nedge distillation, where the student model cannot be initial-\nized from the teacher.\n7. Conclusion\nWe propose BLIP, a new VLP framework with state-\nof-the-art performance on a wide range of downstream\nvision-language tasks, including understanding-based and\ngeneration-based tasks. BLIP pre-trains a multimodal mix-\nture of encoder-decoder model using a dataset bootstrapped\nfrom large-scale noisy image-text pairs by injecting di-\nverse synthetic captions and removing noisy captions. Our\nbootstrapped dataset are released to facilitate future vision-\nlanguage research.\nThere are a few potential directions that can further enhance\nthe performance of BLIP: (1) Multiple rounds of dataset\nbootstrapping; (2) Generate multiple synthetic captions per\nimage to further enlarge the pre-training corpus; (3) Model\nensemble by training multiple different captioners and \ufb01lters"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 23
encoding: FLOAT32
metadata:
{"chroma:document": "and combining their forces in CapFilt. We hope that our pa-\nper motivates future work to focus on making improvements\nin both the model aspect and the data aspect, the bread and\nbutter of vision-language research.\nReferences\nAgrawal, H., Anderson, P., Desai, K., Wang, Y., Chen, X.,\nJain, R., Johnson, M., Batra, D., Parikh, D., and Lee, S.\nnocaps: novel object captioning at scale. In ICCV, pp.\n8947\u20138956, 2019.\nAnaby-Tavor, A., Carmeli, B., Goldbraich, E., Kantor, A.,\nKour, G., Shlomov, S., Tepper, N., and Zwerdling, N. Do\nnot have enough data? deep learning to the rescue! In\nAAAI, pp. 7383\u20137390, 2020.\nAntol, S., Agrawal, A., Lu, J., Mitchell, M., Batra, D.,\nZitnick, C. L., and Parikh, D. VQA: visual question\nanswering. In ICCV, pp. 2425\u20132433, 2015.\nBain, M., Nagrani, A., Varol, G., and Zisserman, A. Frozen\nin time: A joint video and image encoder for end-to-end\nretrieval. In ICCV, 2021.\nBertasius, G., Wang, H., and Torresani, L. Is space-time\nattention all you need for video understanding? In ICML,\n2021.\nChangpinyo, S., Sharma, P., Ding, N., and Soricut, R. Con-\nceptual 12M: Pushing web-scale image-text pre-training\nto recognize long-tail visual concepts. In CVPR, 2021.\nChen, Y., Li, L., Yu, L., Kholy, A. E., Ahmed, F., Gan, Z.,\nCheng, Y., and Liu, J. UNITER: universal image-text\nrepresentation learning. In ECCV, volume 12375, pp.\n104\u2013120, 2020.\nCho, J., Lei, J., Tan, H., and Bansal, M. Unifying vision-\nand-language tasks via text generation. arXiv preprint\narXiv:2102.02779, 2021.\nDas, A., Kottur, S., Gupta, K., Singh, A., Yadav, D., Moura,\nJ. M. F., Parikh, D., and Batra, D. Visual dialog. In CVPR,\npp. 1080\u20131089, 2017.\nDevlin, J., Chang, M., Lee, K., and Toutanova, K. BERT:\npre-training of deep bidirectional transformers for lan-\nguage understanding. In Burstein, J., Doran, C., and\nSolorio, T. (eds.), NAACL, pp. 4171\u20134186, 2019.\nDo, V., Camburu, O.-M., Akata, Z., and Lukasiewicz, T. e-\nsnli-ve: Corrected visual-textual entailment with natural\nlanguage explanations. arXiv preprint arXiv:2004.03744,\n2020."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 24
encoding: FLOAT32
metadata:
{"chroma:document": "2020.\nDosovitskiy, A., Beyer, L., Kolesnikov, A., Weissenborn,\nD., Zhai, X., Unterthiner, T., Dehghani, M., Minderer,\nM., Heigold, G., Gelly, S., Uszkoreit, J., and Houlsby, N.\nAn image is worth 16x16 words: Transformers for image\nrecognition at scale. In ICLR, 2021."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 25
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nFan, C., Zhang, X., Zhang, S., Wang, W., Zhang, C., and\nHuang, H. Heterogeneous memory enhanced multimodal\nattention model for video question answering. In CVPR,\npp. 1999\u20132007, 2019.\nGan, Z., Cheng, Y., Kholy, A. E., Li, L., Liu, J., and Gao, J.\nMulti-step reasoning via recurrent dual attention for vi-\nsual dialog. In Korhonen, A., Traum, D. R., and M`arquez,\nL. (eds.), ACL, pp. 6463\u20136474, 2019.\nGan, Z., Chen, Y., Li, L., Zhu, C., Cheng, Y., and Liu, J.\nLarge-scale adversarial training for vision-and-language\nrepresentation learning. In Larochelle, H., Ranzato, M.,\nHadsell, R., Balcan, M., and Lin, H. (eds.), NeurIPS,\n2020.\nGoyal, Y., Khot, T., Summers-Stay, D., Batra, D., and\nParikh, D. Making the V in VQA matter: Elevating the\nrole of image understanding in visual question answering.\nIn CVPR, pp. 6325\u20136334, 2017.\nHinton, G., Vinyals, O., and Dean, J.\nDistilling\nthe knowledge in a neural network.\narXiv preprint\narXiv:1503.02531, 2015.\nHoltzman, A., Buys, J., Du, L., Forbes, M., and Choi, Y.\nThe curious case of neural text degeneration. In ICLR,\n2020.\nHu, X., Gan, Z., Wang, J., Yang, Z., Liu, Z., Lu, Y., and\nWang, L. Scaling up vision-language pre-training for\nimage captioning, 2021.\nJia, C., Yang, Y., Xia, Y., Chen, Y.-T., Parekh, Z., Pham,\nH., Le, Q. V., Sung, Y., Li, Z., and Duerig, T. Scaling up\nvisual and vision-language representation learning with\nnoisy text supervision. arXiv preprint arXiv:2102.05918,\n2021.\nKarpathy, A. and Li, F. Deep visual-semantic alignments for\ngenerating image descriptions. In CVPR, pp. 3128\u20133137,\n2015.\nKim, J., Jun, J., and Zhang, B. Bilinear attention networks.\nIn Bengio, S., Wallach, H. M., Larochelle, H., Grauman,\nK., Cesa-Bianchi, N., and Garnett, R. (eds.), NIPS, pp.\n1571\u20131581, 2018.\nKim, W., Son, B., and Kim, I. Vilt: Vision-and-language\ntransformer without convolution or region supervision.\narXiv preprint arXiv:2102.03334, 2021."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 26
encoding: FLOAT32
metadata:
{"chroma:document": "Krishna, R., Zhu, Y., Groth, O., Johnson, J., Hata, K.,\nKravitz, J., Chen, S., Kalantidis, Y., Li, L., Shamma,\nD. A., Bernstein, M. S., and Fei-Fei, L. Visual genome:\nConnecting language and vision using crowdsourced\ndense image annotations. IJCV, 123(1):32\u201373, 2017.\nKumar, V., Choudhary, A., and Cho, E. Data augmentation\nusing pre-trained transformer models. arXiv preprint\narXiv:2003.02245, 2020.\nLe, T. M., Le, V., Venkatesh, S., and Tran, T. Hierarchical\nconditional relation networks for video question answer-\ning. In CVPR, pp. 9972\u20139981, 2020.\nLei, J., Li, L., Zhou, L., Gan, Z., Berg, T. L., Bansal, M.,\nand Liu, J. Less is more: Clipbert for video-and-language\nlearning via sparse sampling. In CVPR, pp. 7331\u20137341,\n2021.\nLi, J., Selvaraju, R. R., Gotmare, A. D., Joty, S., Xiong,\nC., and Hoi, S. Align before fuse: Vision and language\nrepresentation learning with momentum distillation. In\nNeurIPS, 2021a.\nLi, W., Gao, C., Niu, G., Xiao, X., Liu, H., Liu, J., Wu,\nH., and Wang, H. UNIMO: towards uni\ufb01ed-modal un-\nderstanding and generation via cross-modal contrastive\nlearning. In Zong, C., Xia, F., Li, W., and Navigli, R.\n(eds.), ACL, pp. 2592\u20132607, 2021b.\nLi, X., Yin, X., Li, C., Zhang, P., Hu, X., Zhang, L., Wang,\nL., Hu, H., Dong, L., Wei, F., Choi, Y., and Gao, J. Oscar:\nObject-semantics aligned pre-training for vision-language\ntasks. In ECCV, pp. 121\u2013137, 2020.\nLin, T., Maire, M., Belongie, S. J., Hays, J., Perona, P.,\nRamanan, D., Doll\u00b4ar, P., and Zitnick, C. L. Microsoft\nCOCO: common objects in context. In Fleet, D. J., Pajdla,\nT., Schiele, B., and Tuytelaars, T. (eds.), ECCV, volume\n8693, pp. 740\u2013755, 2014.\nLoshchilov, I. and Hutter, F. Decoupled weight decay regu-\nlarization. arXiv preprint arXiv:1711.05101, 2017.\nLu, J., Batra, D., Parikh, D., and Lee, S. Vilbert: Pretraining\ntask-agnostic visiolinguistic representations for vision-\nand-language tasks. In Wallach, H. M., Larochelle, H.,\nBeygelzimer, A., d\u2019Alch\u00b4e-Buc, F., Fox, E. B., and Garnett,\nR. (eds.), NeurIPS, pp. 13\u201323, 2019."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 27
encoding: FLOAT32
metadata:
{"chroma:document": "Miech, A., Alayrac, J.-B., Smaira, L., Laptev, I., Sivic, J.,\nand Zisserman, A. End-to-end learning of visual repre-\nsentations from uncurated instructional videos. In CVPR,\npp. 9879\u20139889, 2020.\nMurahari, V., Batra, D., Parikh, D., and Das, A. Large-scale\npretraining for visual dialog: A simple state-of-the-art\nbaseline. In Vedaldi, A., Bischof, H., Brox, T., and Frahm,\nJ. (eds.), ECCV, pp. 336\u2013352, 2020.\nOrdonez, V., Kulkarni, G., and Berg, T. L. Im2text: Describ-\ning images using 1 million captioned photographs. In\nShawe-Taylor, J., Zemel, R. S., Bartlett, P. L., Pereira, F.\nC. N., and Weinberger, K. Q. (eds.), NIPS, pp. 1143\u20131151,\n2011."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 28
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nPaszke, A., Gross, S., Massa, F., Lerer, A., Bradbury, J.,\nChanan, G., Killeen, T., Lin, Z., Gimelshein, N., Antiga,\nL., et al. Pytorch: An imperative style, high-performance\ndeep learning library. NeurIPS, 32:8026\u20138037, 2019.\nPatrick, M., Huang, P.-Y., Asano, Y., Metze, F., Hauptmann,\nA. G., Henriques, J. F., and Vedaldi, A. Support-set\nbottlenecks for video-text representation learning. In\nICLR, 2021.\nPlummer, B. A., Wang, L., Cervantes, C. M., Caicedo, J. C.,\nHockenmaier, J., and Lazebnik, S. Flickr30k entities:\nCollecting region-to-phrase correspondences for richer\nimage-to-sentence models.\nIn ICCV, pp. 2641\u20132649,\n2015.\nPuri, R., Spring, R., Shoeybi, M., Patwary, M., and Catan-\nzaro, B. Training question answering models from syn-\nthetic data. In Webber, B., Cohn, T., He, Y., and Liu, Y.\n(eds.), EMNLP, pp. 5811\u20135826, 2020.\nRadford, A., Kim, J. W., Hallacy, C., Ramesh, A., Goh, G.,\nAgarwal, S., Sastry, G., Askell, A., Mishkin, P., Clark, J.,\net al. Learning transferable visual models from natural\nlanguage supervision. arXiv preprint arXiv:2103.00020,\n2021.\nSchuhmann, C., Vencu, R., Beaumont, R., Kaczmarczyk,\nR., Mullis, C., Katta, A., Coombes, T., Jitsev, J., and\nKomatsuzaki, A. Laion-400m: Open dataset of clip-\n\ufb01ltered 400 million image-text pairs.\narXiv preprint\narXiv:2111.02114, 2021.\nSharma, P., Ding, N., Goodman, S., and Soricut, R. Con-\nceptual captions: A cleaned, hypernymed, image alt-text\ndataset for automatic image captioning. In Gurevych, I.\nand Miyao, Y. (eds.), ACL, pp. 2556\u20132565, 2018.\nShorten, C. and Khoshgoftaar, T. M. A survey on image\ndata augmentation for deep learning. J. Big Data, 6:60,\n2019.\nSuhr, A., Zhou, S., Zhang, A., Zhang, I., Bai, H., and\nArtzi, Y. A corpus for reasoning about natural language\ngrounded in photographs. In Korhonen, A., Traum, D. R.,\nand M`arquez, L. (eds.), ACL, pp. 6418\u20136428, 2019.\nTouvron, H., Cord, M., Douze, M., Massa, F., Sablayrolles,"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 29
encoding: FLOAT32
metadata:
{"chroma:document": "A., and J\u00b4egou, H. Training data-ef\ufb01cient image trans-\nformers & distillation through attention. arXiv preprint\narXiv:2012.12877, 2020.\nWang, Y., Joty, S. R., Lyu, M. R., King, I., Xiong, C., and\nHoi, S. C. H. VD-BERT: A uni\ufb01ed vision and dialog\ntransformer with BERT. In Webber, B., Cohn, T., He, Y.,\nand Liu, Y. (eds.), EMNLP, pp. 3325\u20133338, 2020.\nWang, Z., Yu, J., Yu, A. W., Dai, Z., Tsvetkov, Y., and Cao,\nY. Simvlm: Simple visual language model pretraining\nwith weak supervision. arXiv preprint arXiv:2108.10904,\n2021.\nXie, Q., Luong, M., Hovy, E. H., and Le, Q. V. Self-training\nwith noisy student improves imagenet classi\ufb01cation. In\nCVPR, pp. 10684\u201310695, 2020.\nXu, H., Ghosh, G., Huang, P.-Y., Okhonko, D., Aghajanyan,\nA., Metze, F., Zettlemoyer, L., and Feichtenhofer, C.\nVideoclip: Contrastive pre-training for zero-shot video-\ntext understanding. In EMNLP, pp. 6787\u20136800, 2021.\nYang, A., Miech, A., Sivic, J., Laptev, I., and Schmid, C.\nJust ask: Learning to answer questions from millions of\nnarrated videos. In ICCV, pp. 1686\u20131697, 2021.\nYang, Y., Malaviya, C., Fernandez, J., Swayamdipta, S.,\nBras, R. L., Wang, J., Bhagavatula, C., Choi, Y., and\nDowney, D. G-daug: Generative data augmentation for\ncommonsense reasoning. In Cohn, T., He, Y., and Liu, Y.\n(eds.), EMNLP Findings, pp. 1008\u20131025, 2020.\nZhang, P., Li, X., Hu, X., Yang, J., Zhang, L., Wang, L.,\nChoi, Y., and Gao, J. Vinvl: Making visual representa-\ntions matter in vision-language models. arXiv preprint\narXiv:2101.00529, 2021.\nZhou, L., Palangi, H., Zhang, L., Hu, H., Corso, J. J., and\nGao, J. Uni\ufb01ed vision-language pre-training for image\ncaptioning and VQA. In AAAI, pp. 13041\u201313049, 2020.\nZhu, L. and Yang, Y. Actbert: Learning global-local video-\ntext representations. In CVPR, pp. 8746\u20138755, 2020."}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 30
encoding: FLOAT32
metadata:
{"chroma:document": "BLIP: Bootstrapping Language-Image Pre-training for Uni\ufb01ed Vision-Language Understanding and Generation\nA. Downstream Task Details\nTable 14 shows the hyperparameters that we use for \ufb01ne-\ntuning on the downstream vision-language tasks. All tasks\nuses AdamW optimizer with a weight decay of 0.05 and a\ncosine learning rate schedule. We use an image resolution\nof 384 \u00d7 384, except for VQA where we follow Wang et al.\n(2021) and use 480 \u00d7 480 images. Next we delineate the\ndataset details.\nImage-Text Retrieval. We use the Karpathy split (Karpa-\nthy & Li, 2015) for both COCO and Flickr30K. COCO\ncontains\n113/5k/5k\nimages\nfor\ntrain/validation/test,\nand\nFlickr30K\ncontains\n29k/1k/1k\nimages\nfor\ntrain/validation/test.\nImage Captioning. We \ufb01netune on COCO\u2019s Karpathy train\nsplit, and evaluate on COCO\u2019s Karpathy test split and No-\nCaps validation split. During inference, we use beam search\nwith a beam size of 3, and set the maximum generation\nlength as 20.\nVQA. We experiment with the VQA2.0 dataset (Goyal\net al., 2017), which contains 83k/41k/81k images for train-\ning/validation/test. Following Li et al. (2021a), we use\nboth training and validation splits for training, and include\nadditional training samples from Visual Genome. During\ninference on VQA, we use the decoder to rank the 3,128\ncandidate answers (Li et al., 2021a; Kim et al., 2018).\nNLVR2. We conduct experiment on the of\ufb01cial split (Suhr\net al., 2019).\nVisDial. We \ufb01netune on the training split of VisDial v1.0\nand evaluate on its validation set.\nTask\ninit LR (ViT-L)\nbatch size\n#epoch\nRetrieval\n1e\u22125 (5e\u22126)\n256\n6\nCaptioning\n1e\u22125 (2e\u22126)\n256\n5\nVQA\n2e\u22125\n256\n10\nNLVR2\n3e\u22125\n256\n15\nVisDial\n2e\u22125\n240\n20\nTable 14. Finetuning hyperparameters for downstream tasks.\nB. Additional Examples of Synthetic Captions\nIn Figure 6, we show additional examples of images and\ntexts where the web captions are \ufb01ltered out, and the syn-\nthetic captions are kept as clean training samples.\nC. Pre-training Dataset Details\nTable 15 shows the statistics of the pre-training datasets.\nCOCO\nVG\nSBU\nCC3M\nCC12M"}


created_at: 2023-12-28 05:20:31
topic: persistent://default/default/d839b303-06cc-49b2-8160-1c48f5ecf4d2
id: 31
encoding: FLOAT32
metadata:
{"chroma:document": "CC12M\nLAION\n# image\n113K\n100K\n860K\n3M\n10M\n115M\n# text\n567K\n769K\n860K\n3M\n10M\n115M\nTable 15. Statistics of the pre-training datasets.\n\ud835\udc47!: \u201ca week spent at our \nrented beach house in \nSandbridge\u201d\n\ud835\udc47\": \u201can outdoor walkway \non a grass covered hill\u201d\n\ud835\udc47!: \u201cthat's what a sign \nsays over the door\u201d\n\ud835\udc47\": \u201cthe car is driving \npast a small old \nbuilding\u201d\n\ud835\udc47!: \u201chand held through the \nglass in my front bedroom \nwindow\u201d\n\ud835\udc47\": \u201ca moon against the night \nsky with a black background\u201d\n\ud835\udc47!: \u201cstunning sky over \nwalney island, lake \ndistrict, july 2009\u201d\n\ud835\udc47\": \u201can outdoor walkway \non a grass covered hill\u201d\n\ud835\udc47!: \u201cliving in my \nlittle white house\u201d\n\ud835\udc47\": \u201ca tiny white \nflower with a bee \nin it\u201d\n\ud835\udc47!: \u201cthe pink rock \nfrom below\u201d\n\ud835\udc47\": \u201csome colorful\ntrees that are on a hill \nin the mountains\u201d\nFigure 6. Examples of the web text Tw and the synthetic text Ts. Green texts are accepted by the \ufb01lter, whereas red texts are rejected."}


# embedding_metadata

key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for
Uniﬁed Vision-Language Understanding and Generation
Junnan Li Dongxu Li Caiming Xiong Steven Hoi
Salesforce Research
https://github.com/salesforce/BLIP
Abstract
Vision-Language Pre-training (VLP) has ad-
vanced the performance for many vision-language
tasks. However, most existing pre-trained mod-
els only excel in either understanding-based tasks
or generation-based tasks. Furthermore, perfor-
mance improvement has been largely achieved
by scaling up the dataset with noisy image-text
pairs collected from the web, which is a subop-
timal source of supervision. In this paper, we
propose BLIP, a new VLP framework which trans-
fers ﬂexibly to both vision-language understand-
ing and generation tasks. BLIP effectively uti-
lizes the noisy web data by bootstrapping the
captions, where a captioner generates synthetic
captions and a ﬁlter removes the noisy ones. We
achieve state-of-the-art results on a wide range of
vision-language tasks, such as image-text retrieval
(+2.7% in average recall@1), image captioning
(+2.8% in CIDEr), and VQA (+1.6% in VQA
score). BLIP also demonstrates strong general-
ization ability when directly transferred to video-
language tasks in a zero-shot manner. Code, mod-
els, and datasets are released.
1. Introduction
Vision-language pre-training has recently received tremen-
dous success on various multimodal downstream tasks.
However, existing methods have two major limitations:
(1) Model perspective: most methods either adopt an
encoder-based model (Radford et al., 2021; Li et al., 2021a),
or an encoder-decoder (Cho et al., 2021; Wang et al., 2021)
model. However, encoder-based models are less straightfor-
ward to directly transfer to text generation tasks (e.g. image
captioning), whereas encoder-decoder models have not been
successfully adopted for image-text retrieval tasks.
(2) Data perspective: most state-of-the-art methods (e.g.,
CLIP (Radford et al., 2021), ALBEF (Li et al., 2021a),


key: chroma:document
string_value:
SimVLM (Wang et al., 2021)) pre-train on image-text pairs
Cap
“chocolate cake 
with cream frosting 
and chocolate 
sprinkles on top”
“blue sky bakery in 
sunset park ” 
Filt
Filt
Figure 1. We use a Captioner (Cap) to generate synthetic captions
for web images, and a Filter (Filt) to remove noisy captions.
collected from the web. Despite the performance gain ob-
tained by scaling up the dataset, our paper shows that the
noisy web text is suboptimal for vision-language learning.
To this end, we propose BLIP: Bootstrapping Language-
Image Pre-training for uniﬁed vision-language understand-
ing and generation. BLIP is a new VLP framework which
enables a wider range of downstream tasks than existing
methods. It introduces two contributions from the model
and data perspective, respectively:
(a) Multimodal mixture of Encoder-Decoder (MED): a new
model architecture for effective multi-task pre-training and
ﬂexible transfer learning. An MED can operate either as
a unimodal encoder, or an image-grounded text encoder,
or an image-grounded text decoder. The model is jointly
pre-trained with three vision-language objectives: image-
text contrastive learning, image-text matching, and image-
conditioned language modeling.
(b) Captioning and Filtering (CapFilt): a new dataset boos-
trapping method for learning from noisy image-text pairs.
We ﬁnetune a pre-trained MED into two modules: a cap-
tioner to produce synthetic captions given web images, and
a ﬁlter to remove noisy captions from both the original web
texts and the synthetic texts.
We perform extensive experiments and analysis, and make
the following key observations.
• We show that the captioner and the ﬁlter work together to
achieve substantial performance improvement on various
downstream tasks by bootstrapping the captions. We also
ﬁnd that more diverse captions yield larger gains.
• BLIP achieves state-of-the-art performance on a wide
range of vision-language tasks, including image-text re-
arXiv:2201.12086v2  [cs.CV]  15 Feb 2022


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
Self Attention
Feed Forward
N×
Cross Attention
Feed Forward
Bi Self-Att
Cross Attention
Feed Forward
Bi Self-Att
Cross Attention
Feed Forward
Causal Self-Att
ITM
ITC
LM
N×
“[CLS] +           ”  
“a little girl holding a kitten next to a blue fence”
“[Encode] +           ”  
“[Decode] +           ”  
Text
Encoder
Image
Encoder
Image-grounded 
Text encoder
Image-grounded 
Text decoder
Figure 2. Pre-training model architecture and objectives of BLIP (same parameters have the same color). We propose multimodal mixture
of encoder-decoder, a uniﬁed vision-language model which can operate in one of the three functionalities: (1) Unimodal encoder is
trained with an image-text contrastive (ITC) loss to align the vision and language representations. (2) Image-grounded text encoder uses
additional cross-attention layers to model vision-language interactions, and is trained with a image-text matching (ITM) loss to distinguish
between positive and negative image-text pairs. (3) Image-grounded text decoder replaces the bi-directional self-attention layers with
causal self-attention layers, and shares the same cross-attention layers and feed forward networks as the encoder. The decoder is trained
with a language modeling (LM) loss to generate captions given images.
trieval, image captioning, visual question answering, vi-
sual reasoning, and visual dialog. We also achieve state-of-
the-art zero-shot performance when directly transferring
our models to two video-language tasks: text-to-video
retrieval and videoQA.
2. Related Work
2.1. Vision-language Pre-training
Vision-language pre-training (VLP) aims to improve per-
formance of downstream vision and language tasks by pre-
training the model on large-scale image-text pairs. Due to
the prohibitive expense of acquiring human-annotated texts,
most methods (Chen et al., 2020; Li et al., 2020; 2021a;
Wang et al., 2021; Radford et al., 2021) use image and


key: chroma:document
string_value:
alt-text pairs crawled from the web (Sharma et al., 2018;
Changpinyo et al., 2021; Jia et al., 2021), Despite the use of
simple rule-based ﬁlters, noise is still prevalent in the web
texts. However, the negative impact of the noise has been
largely overlooked, shadowed by the performance gain ob-
tained from scaling up the dataset. Our paper shows that the
noisy web texts are suboptimal for vision-language learning,
and proposes CapFilt that utilizes web datasets in a more
effective way.
There have been many attempts to unify various vision
and language tasks into a single framework (Zhou et al.,
2020; Cho et al., 2021; Wang et al., 2021). The biggest
challenge is to design model architectures that can perform
both understanding-based tasks (e.g. image-text retrieval)
and generation-based tasks (e.g. image captioning). Neither
encoder-based models (Li et al., 2021a;b; Radford et al.,
2021) nor encoder-decoder models (Cho et al., 2021; Wang
et al., 2021) can excel at both types of tasks, whereas a single
uniﬁed encoder-decoder (Zhou et al., 2020) also limits the
model’s capability. Our proposed multimodal mixture of
encoder-decoder model offers more ﬂexibility and better
performance on a wide range of downstream tasks, in the
meantime keeping the pre-training simple and efﬁcient.
2.2. Knowledge Distillation
Knowledge distillation (KD) (Hinton et al., 2015) aims to
improve the performance of a student model by distilling
knowledge from a teacher model. Self-distillation is a spe-
cial case of KD where the teacher and student have equal
sizes. It has been shown to be effective for image classi-
ﬁcation (Xie et al., 2020), and recently for VLP (Li et al.,
2021a). Different from mostly existing KD methods which
simply enforce the student to have the same class predic-
tions as the teacher, our proposed CapFilt can be interpreted
as a more effective way to perform KD in the context of
VLP, where the captioner distills its knowledge through
semantically-rich synthetic captions, and the ﬁlter distills


key: chroma:document
string_value:
its knowledge by removing noisy captions.
2.3. Data Augmentation
While data augmentation (DA) has been widely adopted in
computer vision (Shorten & Khoshgoftaar, 2019), DA for
language tasks is less straightforward. Recently, generative
language models have been used to synthesize examples
for various NLP tasks (Kumar et al., 2020; Anaby-Tavor


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
et al., 2020; Puri et al., 2020; Yang et al., 2020). Differ-
ent from these methods which focus on the low-resource
language-only tasks, our method demonstrates the advan-
tage of synthetic captions in large-scale vision-language
pre-training.
3. Method
We propose BLIP, a uniﬁed VLP framework to learn from
noisy image-text pairs. This section ﬁrst introduces our new
model architecture MED and its pre-training objectives, and
then delineates CapFilt for dataset bootstrapping.
3.1. Model Architecture
We employ a visual transformer (Dosovitskiy et al., 2021)
as our image encoder, which divides an input image into
patches and encodes them as a sequence of embeddings,
with an additional [CLS] token to represent the global im-
age feature. Compared to using pre-trained object detectors
for visual feature extraction (Chen et al., 2020), using a ViT
is more computation-friendly and has been adopted by the
more recent methods (Li et al., 2021a; Kim et al., 2021).
In order to pre-train a uniﬁed model with both understanding
and generation capabilities, we propose multimodal mixture
of encoder-decoder (MED), a multi-task model which can
operate in one of the three functionalities:
(1) Unimodal encoder, which separately encodes image
and text. The text encoder is the same as BERT (Devlin et al.,
2019), where a [CLS] token is appended to the beginning
of the text input to summarize the sentence.
(2) Image-grounded text encoder, which injects visual
information by inserting one additional cross-attention (CA)
layer between the self-attention (SA) layer and the feed
forward network (FFN) for each transformer block of the
text encoder. A task-speciﬁc [Encode] token is appended
to the text, and the output embedding of [Encode] is used
as the multimodal representation of the image-text pair.
(3) Image-grounded text decoder, which replaces the bi-
directional self-attention layers in the image-grounded text


key: chroma:document
string_value:
encoder with causal self-attention layers. A [Decode]
token is used to signal the beginning of a sequence, and an
end-of-sequence token is used to signal its end.
3.2. Pre-training Objectives
We jointly optimize three objectives during pre-training,
with two understanding-based objectives and one generation-
based objective. Each image-text pair only requires one for-
ward pass through the computational-heavier visual trans-
former, and three forward passes through the text trans-
former, where different functionalities are activated to com-
pute the three losses as delineated below.
Image-Text Contrastive Loss (ITC) activates the unimodal
encoder. It aims to align the feature space of the visual trans-
former and the text transformer by encouraging positive
image-text pairs to have similar representations in contrast
to the negative pairs. It has been shown to be an effective
objective for improving vision and language understand-
ing (Radford et al., 2021; Li et al., 2021a). We follow the
ITC loss by Li et al. (2021a), where a momentum encoder
is introduced to produce features, and soft labels are created
from the momentum encoder as training targets to account
for the potential positives in the negative pairs.
Image-Text Matching Loss (ITM) activates the image-
grounded text encoder. It aims to learn image-text mul-
timodal representation that captures the ﬁne-grained align-
ment between vision and language. ITM is a binary clas-
siﬁcation task, where the model uses an ITM head (a lin-
ear layer) to predict whether an image-text pair is positive
(matched) or negative (unmatched) given their multimodal
feature. In order to ﬁnd more informative negatives, we
adopt the hard negative mining strategy by Li et al. (2021a),
where negatives pairs with higher contrastive similarity in a
batch are more likely to be selected to compute the loss.
Language Modeling Loss (LM) activates the image-
grounded text decoder, which aims to generate textual de-
scriptions given an image. It optimizes a cross entropy loss


key: chroma:document
string_value:
which trains the model to maximize the likelihood of the
text in an autoregressive manner. We apply a label smooth-
ing of 0.1 when computing the loss. Compared to the MLM
loss that has been widely-used for VLP, LM enables the
model with the generalization capability to convert visual
information into coherent captions.
In order to perform efﬁcient pre-training while leveraging
multi-task learning, the text encoder and text decoder share
all parameters except for the SA layers. The reason is that
the differences between the encoding and decoding tasks are
best captured by the SA layers. In particular, the encoder
employs bi-directional self-attention to build representations
for the current input tokens, while the decoder employs
causal self-attention to predict next tokens. On the other
hand, the embedding layers, CA layers and FFN function
similarly between encoding and decoding tasks, therefore
sharing these layers can improve training efﬁciency while
beneﬁting from multi-task learning,
3.3. CapFilt
Due to the prohibitive annotation cost, there exist a lim-
ited number of high-quality human-annotated image-text
pairs {(Ih, Th)} (e.g., COCO (Lin et al., 2014)). Recent
work (Li et al., 2021a; Wang et al., 2021) utilizes a much
larger number of image and alt-text pairs {(Iw, Tw)} that
are automatically collected from the web. However, the
alt-texts often do not accurately describe the visual content
of the images, making them a noisy signal that is suboptimal
for learning vision-language alignment.


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
Multimodal Mixture of 
Encoder-Decoder
𝐷 =
𝐼!, 𝑇!
+
𝐼", 𝑇"
Pre-train
Filter 
(Image-grounded 
Text Encoder)
Captioner 
(Image-grounded 
Text Decoder)
ITC&ITM finetune
LM finetune
𝐼", 𝑇"
𝐼!, 𝑇#
{𝐼!}
Captioning
𝐼!, 𝑇!
𝐼!, 𝑇!
+
𝐼!, 𝑇#
Filtering
Dataset Bootstrapping
Model Pretraining
𝐷 =
𝐼!, 𝑇!
+
𝐼!, 𝑇#
+
𝐼", 𝑇"
Downstream Tasks
To model
To data
𝐼!: web images
𝐼": human-annotated
images
𝑇!: web texts
𝑇!: filtered web texts
𝑇#: synthetic texts
𝑇#: filtered synthetic
texts
𝑇": human-annotated
texts
Figure 3. Learning framework of BLIP. We introduce a captioner to produce synthetic captions for web images, and a ﬁlter to remove
noisy image-text pairs. The captioner and ﬁlter are initialized from the same pre-trained model and ﬁnetuned individually on a small-scale
human-annotated dataset. The bootstrapped dataset is used to pre-train a new model.
We propose Captioning and Filtering (CapFilt), a new
method to improve the quality of the text corpus. Figure 3
gives an illustration of CapFilt. It introduces two modules:
a captioner to generate captions given web images, and a
ﬁlter to remove noisy image-text pairs. Both the captioner
and the ﬁlter are initialized from the same pre-trained MED
model, and ﬁnetuned individually on the COCO dataset.
The ﬁnetuning is a lightweight procedure.
Speciﬁcally, the captioner is an image-grounded text de-
coder. It is ﬁnetuned with the LM objective to decode texts
given images. Given the web images Iw, the captioner gen-
erates synthetic captions Ts with one caption per image.
The ﬁlter is an image-grounded text encoder. It is ﬁnetuned
with the ITC and ITM objectives to learn whether a text
matches an image. The ﬁlter removes noisy texts in both
the original web texts Tw and the synthetic texts Ts, where
a text is considered to be noisy if the ITM head predicts it
as unmatched to the image. Finally, we combine the ﬁltered
image-text pairs with the human-annotated pairs to form a


key: chroma:document
string_value:
new dataset, which we use to pre-train a new model.
4. Experiments and Discussions
In this section, we ﬁrst introduce pre-training details. Then
we provide a detailed experimental analysis on our method.
4.1. Pre-training Details
Our models are implemented in PyTorch (Paszke et al.,
2019) and pre-trained on two 16-GPU nodes.
The im-
age transformer is initialized from ViT pre-trained on Ima-
geNet (Touvron et al., 2020; Dosovitskiy et al., 2021), and
the text transformer is initialized from BERTbase (Devlin
et al., 2019). We explore two variants of ViTs: ViT-B/16
and ViT-L/16. Unless otherwise speciﬁed, all results re-
ported in this paper as “BLIP” uses ViT-B. We pre-train the
model for 20 epochs using a batch size of 2880 (ViT-B) /
2400 (ViT-L). We use AdamW (Loshchilov & Hutter, 2017)
optimizer with a weight decay of 0.05. The learning rate
is warmed-up to 3e-4 (ViT-B) / 2e-4 (ViT-L) and decayed
linearly with a rate of 0.85. We take random image crops of
resolution 224 × 224 during pre-training, and increase the
image resolution to 384 × 384 during ﬁnetuning. We use
the same pre-training dataset as Li et al. (2021a) with 14M
images in total, including two human-annotated datasets
(COCO and Visual Genome (Krishna et al., 2017)), and
three web datasets (Conceptual Captions (Changpinyo et al.,
2021), Conceptual 12M (Changpinyo et al., 2021), SBU cap-
tions (Ordonez et al., 2011)). We also experimented with an
additional web dataset, LAION (Schuhmann et al., 2021),
which contains 115M images with more noisy texts1. More
details about the datasets can be found in the appendix.
4.2. Effect of CapFilt
In Table 1, we compare models pre-trained on different
datasets to demonstrate the efﬁcacy of CapFilt on down-
stream tasks, including image-text retrieval and image cap-
tioning with ﬁnetuned and zero-shot settings.
When only the captioner or the ﬁlter is applied to the dataset
with 14M images, performance improvement can be ob-
served. When applied together, their effects compliment


key: chroma:document
string_value:
each other, leading to substantial improvements compared
to using the original noisy web texts.
CapFilt can further boost performance with a larger dataset
and a larger vision backbone, which veriﬁes its scalability
in both the data size and the model size. Furthermore, by
using a large captioner and ﬁlter with ViT-L, performance
of the base model can also be improved.
1We only download images whose shorter edge is larger than
256 pixels from the original LAION400M. Due to the large size of
LAION, we only use 1/5 of it each epoch during pre-training.


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
Pre-train
dataset
Bootstrap
Vision
backbone
Retrieval-FT (COCO)
Retrieval-ZS (Flickr)
Caption-FT (COCO)
Caption-ZS (NoCaps)
C
F
TR@1
IR@1
TR@1
IR@1
B@4
CIDEr
CIDEr
SPICE
COCO+VG
+CC+SBU
(14M imgs)


ViT-B/16
78.4
60.7
93.9
82.1
38.0
127.8
102.2
13.9

B
79.1
61.5
94.1
82.8
38.1
128.2
102.7
14.0
B

79.7
62.0
94.4
83.6
38.4
128.9
103.4
14.2
B
B
80.6
63.1
94.8
84.9
38.6
129.7
105.1
14.4
COCO+VG
+CC+SBU
+LAION
(129M imgs)


ViT-B/16
79.6
62.0
94.3
83.6
38.8
130.1
105.4
14.2
B
B
81.9
64.3
96.0
85.0
39.4
131.4
106.3
14.3
L
L
81.2
64.1
96.0
85.5
39.7
133.3
109.6
14.7


ViT-L/16
80.6
64.1
95.1
85.5
40.3
135.5
112.5
14.7
L
L
82.4
65.1
96.7
86.7
40.4
136.7
113.2
14.8
Table 1. Evaluation of the effect of the captioner (C) and ﬁlter (F) for dataset bootstrapping. Downstream tasks include image-text retrieval
and image captioning with ﬁnetuning (FT) and zero-shot (ZS) settings. TR / IR@1: recall@1 for text retrieval / image retrieval. B/L:
captioner or ﬁlter uses ViT-B / ViT-L as vision backbone.
𝑇!: “from bridge 
near my house”
𝑇": “a flock of birds 
flying over a lake at 
sunset”
𝑇!: “in front of a house 
door in Reichenfels, 
Austria” 
𝑇": “a potted plant sitting 
on top of a pile of rocks”
𝑇!: “the current castle was 
built in 1180, replacing a 9th 
century wooden castle”
𝑇": “a large building with a lot 
of windows on it” 
Figure 4. Examples of the web text Tw and the synthetic text Ts. Green texts are accepted by the ﬁlter, whereas red texts are rejected.
Generation
method
Noise
ratio
Retrieval-FT (COCO)
Retrieval-ZS (Flickr)
Caption-FT (COCO)
Caption-ZS (NoCaps)
TR@1
IR@1
TR@1
IR@1
B@4
CIDEr
CIDEr
SPICE
None
N.A.
78.4
60.7
93.9
82.1
38.0
127.8
102.2
13.9
Beam
19%
79.6
61.9
94.1
83.1
38.4
128.9
103.5
14.2
Nucleus
25%
80.6
63.1
94.8
84.9
38.6
129.7
105.1
14.4
Table 2. Comparison between beam search and nucleus sampling for synthetic caption generation. Models are pre-trained on 14M images.


key: chroma:document
string_value:
Layers shared
#parameters
Retrieval-FT (COCO)
Retrieval-ZS (Flickr)
Caption-FT (COCO)
Caption-ZS (NoCaps)
TR@1
IR@1
TR@1
IR@1
B@4
CIDEr
CIDEr
SPICE
All
224M
77.3
59.5
93.1
81.0
37.2
125.9
100.9
13.1
All except CA
252M
77.5
59.9
93.1
81.3
37.4
126.1
101.2
13.1
All except SA
252M
78.4
60.7
93.9
82.1
38.0
127.8
102.2
13.9
None
361M
78.3
60.5
93.6
81.9
37.8
127.4
101.8
13.9
Table 3. Comparison between different parameter sharing strategies for the text encoder and decoder during pre-training.
In Figure 4, we show some example captions and their
corresponding images, which qualitatively demonstrate the
effect of the captioner to generate new textual descriptions,
and the ﬁlter to remove noisy captions from both the original
web texts and the synthetic texts. More examples can be
found in the appendix.
4.3. Diversity is Key for Synthetic Captions
In CapFilt, we employ nucleus sampling (Holtzman et al.,
2020) to generate synthetic captions. Nucleus sampling is a
stochastic decoding method, where each token is sampled
from a set of tokens whose cumulative probability mass
exceeds a threshold p (p = 0.9 in our experiments). In
Table 2, we compare it with beam search, a deterministic
decoding method which aims to generate captions with the
highest probability. Nucleus sampling leads to evidently
better performance, despite being more noisy as suggested
by a higher noise ratio from the ﬁlter. We hypothesis that the
reason is that nucleus sampling generates more diverse and
surprising captions, which contain more new information
that the model could beneﬁt from. On the other hand, beam
search tends to generate safe captions that are common in
the dataset, hence offering less extra knowledge.
4.4. Parameter Sharing and Decoupling
During pre-training, the text encoder and decoder share all
parameters except for the self-attention layers. In Table 3,
we evaluate models pre-trained with different parameter
sharing strategies, where pre-training is performed on the
14M images with web texts. As the result shows, sharing all


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
Captioner &
Filter
Noise
ratio
Retrieval-FT (COCO)
Retrieval-ZS (Flickr)
Caption-FT (COCO)
Caption-ZS (NoCaps)
TR@1
IR@1
TR@1
IR@1
B@4
CIDEr
CIDEr
SPICE
Share parameters
8%
79.8
62.2
94.3
83.7
38.4
129.0
103.5
14.2
Decoupled
25%
80.6
63.1
94.8
84.9
38.6
129.7
105.1
14.4
Table 4. Effect of sharing parameters between the captioner and ﬁlter. Models are pre-trained on 14M images.
Method
Pre-train
COCO (5K test set)
Flickr30K (1K test set)
# Images
TR
IR
TR
IR
R@1
R@5
R@10
R@1
R@5
R@10
R@1
R@5
R@10
R@1
R@5
R@10
UNITER (Chen et al., 2020)
4M
65.7
88.6
93.8
52.9
79.9
88.0
87.3
98.0
99.2
75.6
94.1
96.8
VILLA (Gan et al., 2020)
4M
-
-
-
-
-
-
87.9
97.5
98.8
76.3
94.2
96.8
OSCAR (Li et al., 2020)
4M
70.0
91.1
95.5
54.0
80.8
88.5
-
-
-
-
-
-
UNIMO (Li et al., 2021b)
5.7M
-
-
-
-
-
-
89.4
98.9
99.8
78.0
94.2
97.1
ALIGN (Jia et al., 2021)
1.8B
77.0
93.5
96.9
59.9
83.3
89.8
95.3
99.8
100.0
84.9
97.4
98.6
ALBEF (Li et al., 2021a)
14M
77.6
94.3
97.2
60.7
84.3
90.5
95.9
99.8
100.0
85.6
97.5
98.9
BLIP
14M
80.6
95.2
97.6
63.1
85.3
91.1
96.6
99.8
100.0
87.2
97.5
98.8
BLIP
129M
81.9
95.4
97.8
64.3
85.7
91.5
97.3
99.9
100.0
87.3
97.6
98.9
BLIPCapFilt-L
129M
81.2
95.7
97.9
64.1
85.8
91.6
97.2
99.9
100.0
87.5
97.7
98.9
BLIPViT-L
129M
82.4
95.4
97.9
65.1
86.3
91.8
97.4
99.8
99.9
87.6
97.7
99.0
Table 5. Comparison with state-of-the-art image-text retrieval methods, ﬁnetuned on COCO and Flickr30K datasets. BLIPCapFilt-L pre-trains
a model with ViT-B backbone using a dataset bootstrapped by captioner and ﬁlter with ViT-L.
Method
Pre-train
Flickr30K (1K test set)
# Images
TR
IR
R@1 R@5 R@10 R@1 R@5 R@10
CLIP
400M
88.0
98.7
99.4
68.7
90.6
95.2
ALIGN
1.8B
88.6
98.7
99.7
75.7
93.8
96.8
ALBEF
14M
94.1
99.5
99.7
82.8
96.3
98.1
BLIP
14M
94.8
99.7
100.0
84.9
96.7
98.3
BLIP
129M
96.0
99.9
100.0
85.0
96.8
98.6
BLIPCapFilt-L 129M
96.0
99.9
100.0
85.5
96.8
98.7
BLIPViT-L
129M
96.7
100.0
100.0
86.7
97.3
98.7


key: chroma:document
string_value:
97.3
98.7
Table 6. Zero-shot image-text retrieval results on Flickr30K.
layers except for SA leads to better performance compared
to not sharing, while also reducing the model size thus
improveing training efﬁciency. If the SA layers are shared,
the model’s performance would degrade due to the conﬂict
between the encoding task and the decoding task.
During CapFilt, the captioner and the ﬁlter are end-to-end
ﬁnetuned individually on COCO. In Table 4, we study the
effect if the captioner and ﬁlter share parameters in the same
way as pre-training. The performance on the downstream
tasks decreases, which we mainly attribute to conﬁrmation
bias. Due to parameter sharing, noisy captions produced by
the captioner are less likely to be ﬁltered out by the ﬁlter, as
indicated by the lower noise ratio (8% compared to 25%).
5. Comparison with State-of-the-arts
In this section, we compare BLIP to existing VLP methods
on a wide range of vision-language downstream tasks2. Next
2we omit SNLI-VE from the benchmark because its test data
has been reported to be noisy (Do et al., 2020)
we brieﬂy introduce each task and ﬁnetuning strategy. More
details can be found in the appendix.
5.1. Image-Text Retrieval
We evaluate BLIP for both image-to-text retrieval (TR) and
text-to-image retrieval (IR) on COCO and Flickr30K (Plum-
mer et al., 2015) datasets. We ﬁnetune the pre-trained model
using ITC and ITM losses. To enable faster inference speed,
we follow Li et al. (2021a) and ﬁrst select k candidates
based on the image-text feature similarity, and then rerank
the selected candidates based on their pairwise ITM scores.
We set k = 256 for COCO and k = 128 for Flickr30K.
As shown in Table 5, BLIP achieves substantial performance
improvement compared with existing methods. Using the
same 14M pre-training images, BLIP outperforms the pre-
vious best model ALBEF by +2.7% in average recall@1
on COCO. We also perform zero-shot retrieval by directly
transferring the model ﬁnetuned on COCO to Flickr30K.


key: chroma:document
string_value:
The result is shown in Table 6, where BLIP also outperforms
existing methods by a large margin.
5.2. Image Captioning
We consider two datasets for image captioning:
No-
Caps (Agrawal et al., 2019) and COCO, both evaluated
using the model ﬁnetuned on COCO with the LM loss. Sim-
ilar as Wang et al. (2021), we add a prompt “a picture of”
at the beginning of each caption, which leads to slightly
better results. As shown in Table 7, BLIP with 14M pre-
training images substantially outperforms methods using
a similar amount of pre-training data. BLIP with 129M
images achieves competitive performance as LEMON with


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
Method
Pre-train
#Images
NoCaps validation
COCO Caption
in-domain
near-domain
out-domain
overall
Karpathy test
C
S
C
S
C
S
C
S
B@4
C
Enc-Dec (Changpinyo et al., 2021)
15M
92.6
12.5
88.3
12.1
94.5
11.9
90.2
12.1
-
110.9
VinVL† (Zhang et al., 2021)
5.7M
103.1
14.2
96.1
13.8
88.3
12.1
95.5
13.5
38.2
129.3
LEMONbase† (Hu et al., 2021)
12M
104.5
14.6
100.7
14.0
96.7
12.4
100.4
13.8
-
-
LEMONbase† (Hu et al., 2021)
200M
107.7
14.7
106.2
14.3
107.9
13.1
106.8
14.1
40.3
133.3
BLIP
14M
111.3
15.1
104.5
14.4
102.4
13.7
105.1
14.4
38.6
129.7
BLIP
129M
109.1
14.8
105.8
14.4
105.7
13.7
106.3
14.3
39.4
131.4
BLIPCapFilt-L
129M
111.8
14.9
108.6
14.8
111.5
14.2
109.6
14.7
39.7
133.3
LEMONlarge† (Hu et al., 2021)
200M
116.9
15.8
113.3
15.1
111.3
14.0
113.4
15.0
40.6
135.7
SimVLMhuge (Wang et al., 2021)
1.8B
113.7
-
110.9
-
115.2
-
112.2
-
40.6
143.3
BLIPViT-L
129M
114.9
15.2
112.1
14.9
115.3
14.4
113.2
14.8
40.4
136.7
Table 7. Comparison with state-of-the-art image captioning methods on NoCaps and COCO Caption. All methods optimize the cross-
entropy loss during ﬁnetuning. C: CIDEr, S: SPICE, B@4: BLEU@4. BLIPCapFilt-L is pre-trained on a dataset bootstrapped by captioner
and ﬁlter with ViT-L. VinVL† and LEMON† require an object detector pre-trained on 2.5M images with human-annotated bounding
boxes and high resolution (800×1333) input images. SimVLMhuge uses 13× more training data and a larger vision backbone than ViT-L.
Image 
Encoder
Question 
Encoder
Image
“[Encode] + Q ”
Answer 
Decoder
“[Decode]”  
answer
(a) VQA
Cross 
Attention
Image #1
Merge Layer
Cross 
Attention
Image 
Encoder
Image 
Encoder
true/false
N×
“[Encode] + Text ”
Image #2
…
…
(b) NLVR!
Image 
Encoder
Caption
Encoder
Image
“[Encode] + C ”
Dialog 
Encoder
“[Encode] + QA + Dialog History”  
true/false
(c) VisDial
Figure 5. Model architecture for the downstream tasks. Q: ques-
tion; C: caption; QA: question-answer pair.


key: chroma:document
string_value:
200M images. Note that LEMON requires a computational-
heavy pre-trained object detector and higher resolution
(800×1333) input images, leading to substantially slower
inference time than the detector-free BLIP which uses lower
resolution (384×384) input images.
5.3. Visual Question Answering (VQA)
VQA (Antol et al., 2015) requires the model to predict an an-
swer given an image and a question. Instead of formulating
VQA as a multi-answer classiﬁcation task (Chen et al., 2020;
Method
Pre-train
#Images
VQA
NLVR2
test-dev
test-std
dev
test-P
LXMERT
180K
72.42
72.54
74.90
74.50
UNITER
4M
72.70
72.91
77.18
77.85
VL-T5/BART
180K
-
71.3
-
73.6
OSCAR
4M
73.16
73.44
78.07
78.36
SOHO
219K
73.25
73.47
76.37
77.32
VILLA
4M
73.59
73.67
78.39
79.30
UNIMO
5.6M
75.06
75.27
-
-
ALBEF
14M
75.84
76.04
82.55
83.14
SimVLMbase†
1.8B
77.87
78.14
81.72
81.77
BLIP
14M
77.54
77.62
82.67
82.30
BLIP
129M
78.24
78.17
82.48
83.08
BLIPCapFilt-L
129M
78.25
78.32
82.15
82.24
Table 8. Comparison with state-of-the-art methods on VQA and
NLVR2. ALBEF performs an extra pre-training step for NLVR2.
SimVLM† uses 13× more training data and a larger vision back-
bone (ResNet+ViT) than BLIP.
Li et al., 2020), we follow Li et al. (2021a) and consider it as
an answer generation task, which enables open-ended VQA.
As shown in Figure 5(a), during ﬁnetuning, we rearrange the
pre-trained model, where an image-question is ﬁrst encoded
into multimodal embeddings and then given to an answer
decoder. The VQA model is ﬁnetuned with the LM loss
using ground-truth answers as targets.
The results are shown in Table 8.
Using 14M images,
BLIP outperforms ALBEF by +1.64% on the test set. Us-
ing 129M images, BLIP achieves better performance than
SimVLM which uses 13× more pre-training data and a
larger vision backbone with an additional convolution stage.
5.4. Natural Language Visual Reasoning (NLVR2)
NLVR2 (Suhr et al., 2019) asks the model to predict whether
a sentence describes a pair of images. In order to enable rea-


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
Method
MRR↑
R@1↑
R@5↑
R@10↑
MR↓
VD-BERT
67.44
54.02
83.96
92.33
3.53
VD-ViLBERT†
69.10
55.88
85.50
93.29
3.25
BLIP
69.41
56.44
85.90
93.30
3.20
Table 9. Comparison with state-of-the-art methods on VisDial v1.0
validation set. VD-ViLBERT† (Murahari et al., 2020) pre-trains
ViLBERT (Lu et al., 2019) with additional VQA data.
soning over two images, we make a simple modiﬁcation to
our pre-trained model which leads to a more computational-
efﬁcient architecture than previous approaches (Li et al.,
2021a; Wang et al., 2021). As shown in Figure 5(b), for
each transformer block in the image-grounded text encoder,
there exist two cross-attention layers to process the two in-
put images, and their outputs are merged and fed to the FFN.
The two CA layers are intialized from the same pre-trained
weights. The merge layer performs simple average pooling
in the ﬁrst 6 layers of the encoder, and performs concate-
nation followed by a linear projection in layer 6-12. An
MLP classiﬁer is applied on the output embedding of the
[Encode] token. As shown in Table 8, BLIP outperforms
all existing methods except for ALBEF which performs an
extra step of customized pre-training. Interestingly, perfor-
mance on NLVR2 does not beneﬁt much from additional
web images, possibly due to the domain gap between web
data and downstream data.
5.5. Visual Dialog (VisDial)
VisDial (Das et al., 2017) extends VQA in a natural con-
versational setting, where the model needs to predict an
answer not only based on the image-question pair, but also
considering the dialog history and the image’s caption. We
follow the discriminative setting where the model ranks a
pool of answer candidates (Gan et al., 2019; Wang et al.,
2020; Murahari et al., 2020). As shown in Figure 5(c), we
concatenate image and caption embeddings, and pass them
to the dialog encoder through cross-attention. The dialog
encoder is trained with the ITM loss to discriminate whether


key: chroma:document
string_value:
the answer is true or false for a question, given the entire dia-
log history and the image-caption embeddings. As shown in
Table 9, our method achieves state-of-the-art performance
on VisDial v1.0 validation set.
5.6. Zero-shot Transfer to Video-Language Tasks
Our image-language model has strong generalization ability
to video-language tasks. In Table 10 and Table 11, we per-
form zero-shot transfer to text-to-video retrieval and video
question answering, where we directly evaluate the models
trained on COCO-retrieval and VQA, respectively. To pro-
cess video input, we uniformly sample n frames per video
(n = 8 for retrieval and n = 16 for QA), and concatenate
the frame features into a single sequence. Note that this
simple approach ignores all temporal information.
Method
R1↑
R5↑
R10↑
MdR↓
zero-shot
ActBERT (Zhu & Yang, 2020)
8.6
23.4
33.1
36
SupportSet (Patrick et al., 2021)
8.7
23.0
31.1
31
MIL-NCE (Miech et al., 2020)
9.9
24.0
32.4
29.5
VideoCLIP (Xu et al., 2021)
10.4
22.2
30.0
-
FiT (Bain et al., 2021)
18.7
39.5
51.6
10
BLIP
43.3
65.6
74.7
2
ﬁnetuning
ClipBERT (Lei et al., 2021)
22.0
46.8
59.9
6
VideoCLIP (Xu et al., 2021)
30.9
55.4
66.8
-
Table 10. Comparisons with state-of-the-art methods for text-to-
video retrieval on the 1k test split of the MSRVTT dataset.
Method
MSRVTT-QA
MSVD-QA
zero-shot
VQA-T (Yang et al., 2021)
2.9
7.5
BLIP
19.2
35.2
ﬁnetuning
HME (Fan et al., 2019)
33.0
33.7
HCRN (Le et al., 2020)
35.6
36.1
VQA-T (Yang et al., 2021)
41.5
46.3
Table 11. Comparisons with state-of-the-art methods for video
question answering. We report top-1 test accuracy on two datasets.
Despite the domain difference and lack of temporal mod-
eling, our models achieve state-of-the-art performance on
both video-language tasks. For text-to-video retrieval, zero-
shot BLIP even outperforms models ﬁnetuned on the target
video dataset by +12.4% in recall@1. Further performance
improvement can be achieved if the BLIP model is used to
initialize a video-language model with temporal modeling


key: chroma:document
string_value:
(e.g. replace our ViT with a TimeSformer (Bertasius et al.,
2021)) and ﬁnetuned on video data.
6. Additional Ablation Study
In this section, we provide additional ablation experiments
on CapFilt.
Improvement with CapFilt is not due to longer training.
Since the bootstrapped dataset contains more texts than the
original dataset, training for the same number of epochs
takes longer with the bootstrapped dataset. To verify that
the effectiveness of CapFilt is not due to longer training,
we replicate the web text in the original dataset so that it
has the same number of training samples per epoch as the
bootstrapped dataset. As shown in Table 12, longer training
using the noisy web texts does not improve performance.
A new model should be trained on the bootstrapped
dataset. The bootstrapped dataset is used to pre-train a
new model. We investigate the effect of continue training


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
CapFilt
#Texts
Retrieval-FT (COCO)
Retrieval-ZS (Flickr)
Caption-FT (COCO)
Caption-ZS (NoCaps)
TR@1
IR@1
TR@1
IR@1
B@4
CIDEr
CIDEr
SPICE
No
15.3M
78.4
60.7
93.9
82.1
38.0
127.8
102.2
13.9
No
24.7M
78.3
60.5
93.7
82.2
37.9
127.7
102.1
14.0
Yes
24.7M
80.6
63.1
94.8
84.9
38.6
129.7
105.1
14.4
Table 12. The original web texts are replicated to have the same number of samples per epoch as the bootstrapped dataset. Results verify
that the improvement from CapFilt is not due to longer training time.
Continue
Retrieval-FT (COCO)
Retrieval-ZS (Flickr)
Caption-FT (COCO)
Caption-ZS (NoCaps)
TR@1
IR@1
TR@1
IR@1
B@4
CIDEr
CIDEr
SPICE
Yes
80.6
63.0
94.5
84.6
38.5
129.9
104.5
14.2
No
80.6
63.1
94.8
84.9
38.6
129.7
105.1
14.4
Table 13. Continue training the pre-trained model offers less gain compared to training a new model with the bootstrapped dataset.
from the previous pre-trained model, using the bootstrapped
dataset. Table 13 hows that continue training does not help.
This observation agrees with the common practice in knowl-
edge distillation, where the student model cannot be initial-
ized from the teacher.
7. Conclusion
We propose BLIP, a new VLP framework with state-
of-the-art performance on a wide range of downstream
vision-language tasks, including understanding-based and
generation-based tasks. BLIP pre-trains a multimodal mix-
ture of encoder-decoder model using a dataset bootstrapped
from large-scale noisy image-text pairs by injecting di-
verse synthetic captions and removing noisy captions. Our
bootstrapped dataset are released to facilitate future vision-
language research.
There are a few potential directions that can further enhance
the performance of BLIP: (1) Multiple rounds of dataset
bootstrapping; (2) Generate multiple synthetic captions per
image to further enlarge the pre-training corpus; (3) Model
ensemble by training multiple different captioners and ﬁlters


key: chroma:document
string_value:
and combining their forces in CapFilt. We hope that our pa-
per motivates future work to focus on making improvements
in both the model aspect and the data aspect, the bread and
butter of vision-language research.
References
Agrawal, H., Anderson, P., Desai, K., Wang, Y., Chen, X.,
Jain, R., Johnson, M., Batra, D., Parikh, D., and Lee, S.
nocaps: novel object captioning at scale. In ICCV, pp.
8947–8956, 2019.
Anaby-Tavor, A., Carmeli, B., Goldbraich, E., Kantor, A.,
Kour, G., Shlomov, S., Tepper, N., and Zwerdling, N. Do
not have enough data? deep learning to the rescue! In
AAAI, pp. 7383–7390, 2020.
Antol, S., Agrawal, A., Lu, J., Mitchell, M., Batra, D.,
Zitnick, C. L., and Parikh, D. VQA: visual question
answering. In ICCV, pp. 2425–2433, 2015.
Bain, M., Nagrani, A., Varol, G., and Zisserman, A. Frozen
in time: A joint video and image encoder for end-to-end
retrieval. In ICCV, 2021.
Bertasius, G., Wang, H., and Torresani, L. Is space-time
attention all you need for video understanding? In ICML,
2021.
Changpinyo, S., Sharma, P., Ding, N., and Soricut, R. Con-
ceptual 12M: Pushing web-scale image-text pre-training
to recognize long-tail visual concepts. In CVPR, 2021.
Chen, Y., Li, L., Yu, L., Kholy, A. E., Ahmed, F., Gan, Z.,
Cheng, Y., and Liu, J. UNITER: universal image-text
representation learning. In ECCV, volume 12375, pp.
104–120, 2020.
Cho, J., Lei, J., Tan, H., and Bansal, M. Unifying vision-
and-language tasks via text generation. arXiv preprint
arXiv:2102.02779, 2021.
Das, A., Kottur, S., Gupta, K., Singh, A., Yadav, D., Moura,
J. M. F., Parikh, D., and Batra, D. Visual dialog. In CVPR,
pp. 1080–1089, 2017.
Devlin, J., Chang, M., Lee, K., and Toutanova, K. BERT:
pre-training of deep bidirectional transformers for lan-
guage understanding. In Burstein, J., Doran, C., and
Solorio, T. (eds.), NAACL, pp. 4171–4186, 2019.
Do, V., Camburu, O.-M., Akata, Z., and Lukasiewicz, T. e-
snli-ve: Corrected visual-textual entailment with natural
language explanations. arXiv preprint arXiv:2004.03744,
2020.


key: chroma:document
string_value:
2020.
Dosovitskiy, A., Beyer, L., Kolesnikov, A., Weissenborn,
D., Zhai, X., Unterthiner, T., Dehghani, M., Minderer,
M., Heigold, G., Gelly, S., Uszkoreit, J., and Houlsby, N.
An image is worth 16x16 words: Transformers for image
recognition at scale. In ICLR, 2021.


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
Fan, C., Zhang, X., Zhang, S., Wang, W., Zhang, C., and
Huang, H. Heterogeneous memory enhanced multimodal
attention model for video question answering. In CVPR,
pp. 1999–2007, 2019.
Gan, Z., Cheng, Y., Kholy, A. E., Li, L., Liu, J., and Gao, J.
Multi-step reasoning via recurrent dual attention for vi-
sual dialog. In Korhonen, A., Traum, D. R., and M`arquez,
L. (eds.), ACL, pp. 6463–6474, 2019.
Gan, Z., Chen, Y., Li, L., Zhu, C., Cheng, Y., and Liu, J.
Large-scale adversarial training for vision-and-language
representation learning. In Larochelle, H., Ranzato, M.,
Hadsell, R., Balcan, M., and Lin, H. (eds.), NeurIPS,
2020.
Goyal, Y., Khot, T., Summers-Stay, D., Batra, D., and
Parikh, D. Making the V in VQA matter: Elevating the
role of image understanding in visual question answering.
In CVPR, pp. 6325–6334, 2017.
Hinton, G., Vinyals, O., and Dean, J.
Distilling
the knowledge in a neural network.
arXiv preprint
arXiv:1503.02531, 2015.
Holtzman, A., Buys, J., Du, L., Forbes, M., and Choi, Y.
The curious case of neural text degeneration. In ICLR,
2020.
Hu, X., Gan, Z., Wang, J., Yang, Z., Liu, Z., Lu, Y., and
Wang, L. Scaling up vision-language pre-training for
image captioning, 2021.
Jia, C., Yang, Y., Xia, Y., Chen, Y.-T., Parekh, Z., Pham,
H., Le, Q. V., Sung, Y., Li, Z., and Duerig, T. Scaling up
visual and vision-language representation learning with
noisy text supervision. arXiv preprint arXiv:2102.05918,
2021.
Karpathy, A. and Li, F. Deep visual-semantic alignments for
generating image descriptions. In CVPR, pp. 3128–3137,
2015.
Kim, J., Jun, J., and Zhang, B. Bilinear attention networks.
In Bengio, S., Wallach, H. M., Larochelle, H., Grauman,
K., Cesa-Bianchi, N., and Garnett, R. (eds.), NIPS, pp.
1571–1581, 2018.
Kim, W., Son, B., and Kim, I. Vilt: Vision-and-language
transformer without convolution or region supervision.
arXiv preprint arXiv:2102.03334, 2021.


key: chroma:document
string_value:
Krishna, R., Zhu, Y., Groth, O., Johnson, J., Hata, K.,
Kravitz, J., Chen, S., Kalantidis, Y., Li, L., Shamma,
D. A., Bernstein, M. S., and Fei-Fei, L. Visual genome:
Connecting language and vision using crowdsourced
dense image annotations. IJCV, 123(1):32–73, 2017.
Kumar, V., Choudhary, A., and Cho, E. Data augmentation
using pre-trained transformer models. arXiv preprint
arXiv:2003.02245, 2020.
Le, T. M., Le, V., Venkatesh, S., and Tran, T. Hierarchical
conditional relation networks for video question answer-
ing. In CVPR, pp. 9972–9981, 2020.
Lei, J., Li, L., Zhou, L., Gan, Z., Berg, T. L., Bansal, M.,
and Liu, J. Less is more: Clipbert for video-and-language
learning via sparse sampling. In CVPR, pp. 7331–7341,
2021.
Li, J., Selvaraju, R. R., Gotmare, A. D., Joty, S., Xiong,
C., and Hoi, S. Align before fuse: Vision and language
representation learning with momentum distillation. In
NeurIPS, 2021a.
Li, W., Gao, C., Niu, G., Xiao, X., Liu, H., Liu, J., Wu,
H., and Wang, H. UNIMO: towards uniﬁed-modal un-
derstanding and generation via cross-modal contrastive
learning. In Zong, C., Xia, F., Li, W., and Navigli, R.
(eds.), ACL, pp. 2592–2607, 2021b.
Li, X., Yin, X., Li, C., Zhang, P., Hu, X., Zhang, L., Wang,
L., Hu, H., Dong, L., Wei, F., Choi, Y., and Gao, J. Oscar:
Object-semantics aligned pre-training for vision-language
tasks. In ECCV, pp. 121–137, 2020.
Lin, T., Maire, M., Belongie, S. J., Hays, J., Perona, P.,
Ramanan, D., Doll´ar, P., and Zitnick, C. L. Microsoft
COCO: common objects in context. In Fleet, D. J., Pajdla,
T., Schiele, B., and Tuytelaars, T. (eds.), ECCV, volume
8693, pp. 740–755, 2014.
Loshchilov, I. and Hutter, F. Decoupled weight decay regu-
larization. arXiv preprint arXiv:1711.05101, 2017.
Lu, J., Batra, D., Parikh, D., and Lee, S. Vilbert: Pretraining
task-agnostic visiolinguistic representations for vision-
and-language tasks. In Wallach, H. M., Larochelle, H.,
Beygelzimer, A., d’Alch´e-Buc, F., Fox, E. B., and Garnett,
R. (eds.), NeurIPS, pp. 13–23, 2019.


key: chroma:document
string_value:
Miech, A., Alayrac, J.-B., Smaira, L., Laptev, I., Sivic, J.,
and Zisserman, A. End-to-end learning of visual repre-
sentations from uncurated instructional videos. In CVPR,
pp. 9879–9889, 2020.
Murahari, V., Batra, D., Parikh, D., and Das, A. Large-scale
pretraining for visual dialog: A simple state-of-the-art
baseline. In Vedaldi, A., Bischof, H., Brox, T., and Frahm,
J. (eds.), ECCV, pp. 336–352, 2020.
Ordonez, V., Kulkarni, G., and Berg, T. L. Im2text: Describ-
ing images using 1 million captioned photographs. In
Shawe-Taylor, J., Zemel, R. S., Bartlett, P. L., Pereira, F.
C. N., and Weinberger, K. Q. (eds.), NIPS, pp. 1143–1151,
2011.


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
Paszke, A., Gross, S., Massa, F., Lerer, A., Bradbury, J.,
Chanan, G., Killeen, T., Lin, Z., Gimelshein, N., Antiga,
L., et al. Pytorch: An imperative style, high-performance
deep learning library. NeurIPS, 32:8026–8037, 2019.
Patrick, M., Huang, P.-Y., Asano, Y., Metze, F., Hauptmann,
A. G., Henriques, J. F., and Vedaldi, A. Support-set
bottlenecks for video-text representation learning. In
ICLR, 2021.
Plummer, B. A., Wang, L., Cervantes, C. M., Caicedo, J. C.,
Hockenmaier, J., and Lazebnik, S. Flickr30k entities:
Collecting region-to-phrase correspondences for richer
image-to-sentence models.
In ICCV, pp. 2641–2649,
2015.
Puri, R., Spring, R., Shoeybi, M., Patwary, M., and Catan-
zaro, B. Training question answering models from syn-
thetic data. In Webber, B., Cohn, T., He, Y., and Liu, Y.
(eds.), EMNLP, pp. 5811–5826, 2020.
Radford, A., Kim, J. W., Hallacy, C., Ramesh, A., Goh, G.,
Agarwal, S., Sastry, G., Askell, A., Mishkin, P., Clark, J.,
et al. Learning transferable visual models from natural
language supervision. arXiv preprint arXiv:2103.00020,
2021.
Schuhmann, C., Vencu, R., Beaumont, R., Kaczmarczyk,
R., Mullis, C., Katta, A., Coombes, T., Jitsev, J., and
Komatsuzaki, A. Laion-400m: Open dataset of clip-
ﬁltered 400 million image-text pairs.
arXiv preprint
arXiv:2111.02114, 2021.
Sharma, P., Ding, N., Goodman, S., and Soricut, R. Con-
ceptual captions: A cleaned, hypernymed, image alt-text
dataset for automatic image captioning. In Gurevych, I.
and Miyao, Y. (eds.), ACL, pp. 2556–2565, 2018.
Shorten, C. and Khoshgoftaar, T. M. A survey on image
data augmentation for deep learning. J. Big Data, 6:60,
2019.
Suhr, A., Zhou, S., Zhang, A., Zhang, I., Bai, H., and
Artzi, Y. A corpus for reasoning about natural language
grounded in photographs. In Korhonen, A., Traum, D. R.,
and M`arquez, L. (eds.), ACL, pp. 6418–6428, 2019.
Touvron, H., Cord, M., Douze, M., Massa, F., Sablayrolles,


key: chroma:document
string_value:
A., and J´egou, H. Training data-efﬁcient image trans-
formers & distillation through attention. arXiv preprint
arXiv:2012.12877, 2020.
Wang, Y., Joty, S. R., Lyu, M. R., King, I., Xiong, C., and
Hoi, S. C. H. VD-BERT: A uniﬁed vision and dialog
transformer with BERT. In Webber, B., Cohn, T., He, Y.,
and Liu, Y. (eds.), EMNLP, pp. 3325–3338, 2020.
Wang, Z., Yu, J., Yu, A. W., Dai, Z., Tsvetkov, Y., and Cao,
Y. Simvlm: Simple visual language model pretraining
with weak supervision. arXiv preprint arXiv:2108.10904,
2021.
Xie, Q., Luong, M., Hovy, E. H., and Le, Q. V. Self-training
with noisy student improves imagenet classiﬁcation. In
CVPR, pp. 10684–10695, 2020.
Xu, H., Ghosh, G., Huang, P.-Y., Okhonko, D., Aghajanyan,
A., Metze, F., Zettlemoyer, L., and Feichtenhofer, C.
Videoclip: Contrastive pre-training for zero-shot video-
text understanding. In EMNLP, pp. 6787–6800, 2021.
Yang, A., Miech, A., Sivic, J., Laptev, I., and Schmid, C.
Just ask: Learning to answer questions from millions of
narrated videos. In ICCV, pp. 1686–1697, 2021.
Yang, Y., Malaviya, C., Fernandez, J., Swayamdipta, S.,
Bras, R. L., Wang, J., Bhagavatula, C., Choi, Y., and
Downey, D. G-daug: Generative data augmentation for
commonsense reasoning. In Cohn, T., He, Y., and Liu, Y.
(eds.), EMNLP Findings, pp. 1008–1025, 2020.
Zhang, P., Li, X., Hu, X., Yang, J., Zhang, L., Wang, L.,
Choi, Y., and Gao, J. Vinvl: Making visual representa-
tions matter in vision-language models. arXiv preprint
arXiv:2101.00529, 2021.
Zhou, L., Palangi, H., Zhang, L., Hu, H., Corso, J. J., and
Gao, J. Uniﬁed vision-language pre-training for image
captioning and VQA. In AAAI, pp. 13041–13049, 2020.
Zhu, L. and Yang, Y. Actbert: Learning global-local video-
text representations. In CVPR, pp. 8746–8755, 2020.


key: chroma:document
string_value:
BLIP: Bootstrapping Language-Image Pre-training for Uniﬁed Vision-Language Understanding and Generation
A. Downstream Task Details
Table 14 shows the hyperparameters that we use for ﬁne-
tuning on the downstream vision-language tasks. All tasks
uses AdamW optimizer with a weight decay of 0.05 and a
cosine learning rate schedule. We use an image resolution
of 384 × 384, except for VQA where we follow Wang et al.
(2021) and use 480 × 480 images. Next we delineate the
dataset details.
Image-Text Retrieval. We use the Karpathy split (Karpa-
thy & Li, 2015) for both COCO and Flickr30K. COCO
contains
113/5k/5k
images
for
train/validation/test,
and
Flickr30K
contains
29k/1k/1k
images
for
train/validation/test.
Image Captioning. We ﬁnetune on COCO’s Karpathy train
split, and evaluate on COCO’s Karpathy test split and No-
Caps validation split. During inference, we use beam search
with a beam size of 3, and set the maximum generation
length as 20.
VQA. We experiment with the VQA2.0 dataset (Goyal
et al., 2017), which contains 83k/41k/81k images for train-
ing/validation/test. Following Li et al. (2021a), we use
both training and validation splits for training, and include
additional training samples from Visual Genome. During
inference on VQA, we use the decoder to rank the 3,128
candidate answers (Li et al., 2021a; Kim et al., 2018).
NLVR2. We conduct experiment on the ofﬁcial split (Suhr
et al., 2019).
VisDial. We ﬁnetune on the training split of VisDial v1.0
and evaluate on its validation set.
Task
init LR (ViT-L)
batch size
#epoch
Retrieval
1e−5 (5e−6)
256
6
Captioning
1e−5 (2e−6)
256
5
VQA
2e−5
256
10
NLVR2
3e−5
256
15
VisDial
2e−5
240
20
Table 14. Finetuning hyperparameters for downstream tasks.
B. Additional Examples of Synthetic Captions
In Figure 6, we show additional examples of images and
texts where the web captions are ﬁltered out, and the syn-
thetic captions are kept as clean training samples.
C. Pre-training Dataset Details
Table 15 shows the statistics of the pre-training datasets.
COCO
VG
SBU
CC3M
CC12M


key: chroma:document
string_value:
CC12M
LAION
# image
113K
100K
860K
3M
10M
115M
# text
567K
769K
860K
3M
10M
115M
Table 15. Statistics of the pre-training datasets.
𝑇!: “a week spent at our 
rented beach house in 
Sandbridge”
𝑇": “an outdoor walkway 
on a grass covered hill”
𝑇!: “that's what a sign 
says over the door”
𝑇": “the car is driving 
past a small old 
building”
𝑇!: “hand held through the 
glass in my front bedroom 
window”
𝑇": “a moon against the night 
sky with a black background”
𝑇!: “stunning sky over 
walney island, lake 
district, july 2009”
𝑇": “an outdoor walkway 
on a grass covered hill”
𝑇!: “living in my 
little white house”
𝑇": “a tiny white 
flower with a bee 
in it”
𝑇!: “the pink rock 
from below”
𝑇": “some colorful
trees that are on a hill 
in the mountains”
Figure 6. Examples of the web text Tw and the synthetic text Ts. Green texts are accepted by the ﬁlter, whereas red texts are rejected.


