Zero-Shot Image Classification
OpenCLIP
Safetensors
clip
rwightman HF staff commited on
Commit
2861d43
1 Parent(s): 96c82fc
README.md ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - clip
4
+ library_name: open_clip
5
+ pipeline_tag: zero-shot-image-classification
6
+ license: apache-2.0
7
+ datasets:
8
+ - mlfoundations/datacomp_1b
9
+ ---
10
+ # Model card for ViT-H-14-CLIPA-datacomp1B
11
+
12
+ A CLIPA-v2 model...
13
+
14
+ ## Model Details
15
+ - **Model Type:** Contrastive Image-Text, Zero-Shot Image Classification.
16
+ - **Original:** https://github.com/UCSC-VLAA/CLIPA
17
+ - **Dataset:** mlfoundations/datacomp_1b
18
+ - **Papers:**
19
+ - CLIPA-v2: Scaling CLIP Training with 81.1% Zero-shot ImageNet Accuracy within a $10,000 Budget; An Extra $4,000 Unlocks 81.8% Accuracy: https://arxiv.org/abs/2306.15658
20
+ - An Inverse Scaling Law for CLIP Training: https://arxiv.org/abs/2305.07017
21
+
22
+ ## Model Usage
23
+ ### With OpenCLIP
24
+ ```
25
+ import torch
26
+ import torch.nn.functional as F
27
+ from urllib.request import urlopen
28
+ from PIL import Image
29
+ from open_clip import create_model_from_pretrained, get_tokenizer
30
+
31
+ model, preprocess = create_model_from_pretrained('hf-hub:ViT-H-14-CLIPA')
32
+ tokenizer = get_tokenizer('hf-hub:ViT-H-14-CLIPA')
33
+
34
+ image = Image.open(urlopen(
35
+ 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png'
36
+ ))
37
+ image = preprocess(image).unsqueeze(0)
38
+
39
+ text = tokenizer(["a diagram", "a dog", "a cat", "a beignet"], context_length=model.context_length)
40
+
41
+ with torch.no_grad(), torch.cuda.amp.autocast():
42
+ image_features = model.encode_image(image)
43
+ text_features = model.encode_text(text)
44
+ image_features = F.normalize(image_features, dim=-1)
45
+ text_features = F.normalize(text_features, dim=-1)
46
+
47
+ text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
48
+
49
+ print("Label probs:", text_probs) # prints: [[0., 0., 0., 1.0]]
50
+ ```
51
+
52
+ ## Citation
53
+ ```bibtex
54
+ @article{li2023clipav2,
55
+ title={CLIPA-v2: Scaling CLIP Training with 81.1% Zero-shot ImageNet Accuracy within a $10,000 Budget; An Extra $4,000 Unlocks 81.8% Accuracy},
56
+ author={Xianhang Li and Zeyu Wang and Cihang Xie},
57
+ journal={arXiv preprint arXiv:2306.15658},
58
+ year={2023},
59
+ }
60
+ ```
61
+ ```bibtex
62
+ @inproceedings{li2023clipa,
63
+ title={An Inverse Scaling Law for CLIP Training},
64
+ author={Xianhang Li and Zeyu Wang and Cihang Xie},
65
+ booktitle={NeurIPS},
66
+ year={2023},
67
+ }
68
+ ```
added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "[CLS]": 101,
3
+ "[MASK]": 103,
4
+ "[PAD]": 0,
5
+ "[SEP]": 102,
6
+ "[UNK]": 100
7
+ }
open_clip_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_cfg": {
3
+ "embed_dim": 1024,
4
+ "vision_cfg": {
5
+ "image_size": 224,
6
+ "layers": 32,
7
+ "width": 1280,
8
+ "head_width": 80,
9
+ "patch_size": 14,
10
+ "no_ln_pre": true,
11
+ "pool_type": "avg",
12
+ "final_ln_after_pool": true
13
+ },
14
+ "text_cfg": {
15
+ "context_length": 32,
16
+ "vocab_size": 32000,
17
+ "hf_tokenizer_name": "bert-base-uncased",
18
+ "tokenizer_kwargs": {
19
+ "strip_sep_token": true
20
+ },
21
+ "width": 1024,
22
+ "heads": 16,
23
+ "layers": 24,
24
+ "pool_type": "last",
25
+ "no_causal_mask": true
26
+ }
27
+ },
28
+ "preprocess_cfg": {
29
+ "mean": [
30
+ 0.485,
31
+ 0.456,
32
+ 0.406
33
+ ],
34
+ "std": [
35
+ 0.229,
36
+ 0.224,
37
+ 0.225
38
+ ],
39
+ "interpolation": "bilinear",
40
+ "resize_mode": "squash"
41
+ }
42
+ }
open_clip_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05fdb3e9071caf865704ba794d4d6f770f0bffb07d82a9a1acff8f381c4d4b27
3
+ size 3873019860
open_clip_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0af7f7d05e15ff450843e049554b27620d7c036dd3f9a268600ffd2c6bf6c6df
3
+ size 3873205914
special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "[CLS]",
3
+ "mask_token": "[MASK]",
4
+ "pad_token": "[PAD]",
5
+ "sep_token": "[SEP]",
6
+ "unk_token": "[UNK]"
7
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[PAD]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "100": {
12
+ "content": "[UNK]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "101": {
20
+ "content": "[CLS]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "102": {
28
+ "content": "[SEP]",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "103": {
36
+ "content": "[MASK]",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "[CLS]",
47
+ "do_lower_case": true,
48
+ "mask_token": "[MASK]",
49
+ "model_max_length": 512,
50
+ "pad_token": "[PAD]",
51
+ "sep_token": "[SEP]",
52
+ "strip_accents": null,
53
+ "tokenize_chinese_chars": true,
54
+ "tokenizer_class": "BertTokenizer",
55
+ "unk_token": "[UNK]"
56
+ }
vocab.txt ADDED
The diff for this file is too large to render. See raw diff