Hugging Face's logo

Mozilla
/
distilvit

Image-to-Text
Transformers
ONNX
Safetensors
vision-encoder-decoder
image-captioning
Eval Results
Inference Endpoints
Model card Files Community
1
tarekziade commited on
Commit
418a5c8
•
1 Parent(s): 9060760

fined tuned on alt-text-validation

Browse files
Files changed (20) hide show
  1. README.md +6 -2
  2. config.json +1 -1
  3. generation_config.json +1 -4
  4. merges.txt +0 -0
  5. model.safetensors +1 -1
  6. onnx/decoder_model.onnx +1 -1
  7. onnx/decoder_model_merged.onnx +1 -1
  8. onnx/decoder_model_merged_quantized.onnx +2 -2
  9. onnx/decoder_model_quantized.onnx +2 -2
  10. onnx/decoder_with_past_model.onnx +1 -1
  11. onnx/decoder_with_past_model_quantized.onnx +2 -2
  12. onnx/encoder_model.onnx +1 -1
  13. onnx/encoder_model_quantized.onnx +1 -1
  14. preprocessor_config.json +0 -0
  15. quantize_config.json +79 -79
  16. special_tokens_map.json +4 -28
  17. tokenizer.json +1 -1
  18. tokenizer_config.json +0 -0
  19. training_args.bin +1 -1
  20. vocab.json +0 -0
README.md CHANGED
@@ -68,8 +68,12 @@ You can get that checkpoint using the 3083a3cef6e3c8dd90df3f088074bbe836b0f403 c
68
 
69
  It was then further fine-tuned on :
70
 
71
- - Flickr30k debiased: https://huggingface.co/datasets/Mozilla/flickr30k-transformed-captions
72
- - DocOrNot: https://huggingface.co/datasets/Mozilla/docornot
 
 
 
 
73
 
74
  You can find the code used to create the model here: https://github.com/mozilla/distilvit
75
 
 
68
 
69
  It was then further fine-tuned on :
70
 
71
+ - [Flickr30k debiased](https://huggingface.co/datasets/Mozilla/flickr30k-transformed-captions)
72
+ - [DocOrNot](https://huggingface.co/datasets/Mozilla/docornot)
73
+ - [Alt Text Validation](https://huggingface.co/datasets/Mozilla/alt-text-validation)
74
+
75
+ For the latter, the dataset was annotated by our team to correct the alt text generayed by the model,
76
+ using the [checkvite tool](https://github.com/mozila/checkvite).
77
 
78
  You can find the code used to create the model here: https://github.com/mozilla/distilvit
79
 
config.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "_name_or_path": "mozilla/distilvit",
3
  "architectures": [
4
  "VisionEncoderDecoderModel"
5
  ],
 
1
  {
2
+ "_name_or_path": "/Volumes/SSD/vit-base-patch16-224-distilgpt2",
3
  "architectures": [
4
  "VisionEncoderDecoderModel"
5
  ],
generation_config.json CHANGED
@@ -1,6 +1,5 @@
1
  {
2
  "bos_token_id": 50256,
3
- "do_sample": false,
4
  "early_stopping": true,
5
  "eos_token_id": 50256,
6
  "max_length": 50,
@@ -8,8 +7,6 @@
8
  "num_beams": 2,
9
  "pad_token_id": 50256,
10
  "repetition_penalty": 1.2,
11
- "transformers_version": "4.33.2",
12
  "seed": 12,
13
- "use_cache": true
14
  }
15
-
 
1
  {
2
  "bos_token_id": 50256,
 
3
  "early_stopping": true,
4
  "eos_token_id": 50256,
5
  "max_length": 50,
 
7
  "num_beams": 2,
8
  "pad_token_id": 50256,
9
  "repetition_penalty": 1.2,
 
10
  "seed": 12,
11
+ "transformers_version": "4.33.2"
12
  }
 
merges.txt CHANGED
File without changes
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:eb10c7e6caaf59d39c22b873daa806f504ea670c4757dbd67a8b5e3be97be6b3
3
  size 729979160
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:74574a184acd40eb36ffb93d03ea3780bbe8f8f5d019aabbdf7554304e704a05
3
  size 729979160
onnx/decoder_model.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7351a7eaeeec455e70e68007a1dcf7307e245473f53da6b13312016ad5d992d3
3
  size 385864797
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3704718aee4c999920574a3fb45b959ee59a9eb687bc5ff39c1ec6c3b3d32fc
3
  size 385864797
onnx/decoder_model_merged.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f951eec84e5b043b03ac6a94556e7b0f16846282bc577adec061c848119aadac
3
  size 387342586
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dfc0cac17407a66735b4b4dd5ce62900967a015a618f24c6036b628e1c600328
3
  size 387342586
onnx/decoder_model_merged_quantized.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1ff8ee2b032955fd71b217c61751a92a6386e7b0e31228ef806a89a923bca34d
3
- size 99759578
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:083cc1aa2c10ee51436601f996091d4633ff2ad6ab933f1197e23c14893fc89e
3
+ size 99759579
onnx/decoder_model_quantized.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:737759dec1b90e1e29eb61e271951db84e3e677d6f8332db64bd4a15fe931711
3
- size 98065762
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:230dd40ee4ba5bd9ec09ad783b45d98ca8acd7b37ff1f73ddbfa6a63c63163ba
3
+ size 98065763
onnx/decoder_with_past_model.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b99196af93603df1ef7eeb9676177d99c0a4a11b01e266ae5cb9327abe968c54
3
  size 385864377
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aae44e11c82e59d84ac7c9f654942dfbcbb2c0d2a03bc8ec49876e81dc3393a1
3
  size 385864377
onnx/decoder_with_past_model_quantized.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:12e5aad5d80e36cd4f9b2d45297732b01fd9e6503e6393aa393b4c13533c53e7
3
- size 98063169
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1823eff0a8dc274eabb3df67551bc7330d4e22a22c23676ff978eb33c643896
3
+ size 98063170
onnx/encoder_model.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:aacd44f8cc26c1a0539f5db1fcb98cf3baad2c5e62583b913ed501daf1419e96
3
  size 343440632
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca2910ada0c94d6f7d72c80e7e4f40916c0bfbd1cff763ad86d96e02fd527e90
3
  size 343440632
onnx/encoder_model_quantized.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:13062fd1801c38f1039ed439e792b9e08dc8eb0b006d4dd64146575e641c4286
3
  size 87038173
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9930375f2182d3cb829d78750939633587891b18edb0a66eeb569317ccc0776
3
  size 87038173
preprocessor_config.json CHANGED
File without changes
quantize_config.json CHANGED
@@ -2,122 +2,122 @@
2
  "per_channel": false,
3
  "reduce_range": false,
4
  "per_model_config": {
5
- "decoder_with_past_model": {
6
  "op_types": [
7
- "Sqrt",
8
- "Split",
9
- "Unsqueeze",
10
- "Tanh",
11
- "Gemm",
12
  "MatMul",
13
- "Cast",
14
- "Div",
15
- "Pow",
16
  "Add",
17
- "Constant",
18
- "Reshape",
19
  "ConstantOfShape",
20
- "Range",
21
- "Slice",
 
 
 
 
22
  "Where",
23
- "Concat",
 
 
 
 
 
24
  "Shape",
25
- "Squeeze",
26
  "ReduceMean",
27
- "Softmax",
28
- "Gather",
29
- "Sub",
30
- "Mul",
31
- "Transpose"
32
  ],
33
- "weight_type": "QInt8"
34
  },
35
  "decoder_model": {
36
  "op_types": [
37
- "Sqrt",
38
- "Split",
39
- "Unsqueeze",
40
- "Tanh",
41
- "Gemm",
42
  "MatMul",
43
- "Cast",
44
- "Div",
45
- "Pow",
46
  "Add",
47
- "Constant",
48
- "Reshape",
49
  "ConstantOfShape",
50
- "Range",
51
- "Slice",
 
 
 
52
  "Where",
53
- "Concat",
54
- "Shape",
 
 
 
 
55
  "Squeeze",
 
 
 
 
 
56
  "ReduceMean",
57
- "Softmax",
58
- "Gather",
59
- "Sub",
60
- "Mul",
61
- "Transpose"
62
  ],
63
  "weight_type": "QInt8"
64
  },
65
- "encoder_model": {
66
  "op_types": [
67
- "Sqrt",
68
- "Unsqueeze",
69
- "Equal",
70
  "MatMul",
71
- "Div",
72
- "Pow",
73
  "Add",
74
- "Erf",
75
- "Constant",
76
- "Reshape",
77
  "ConstantOfShape",
78
- "Slice",
 
 
 
 
79
  "Where",
80
- "Conv",
81
- "Concat",
 
 
 
 
 
 
 
 
 
82
  "Shape",
83
  "ReduceMean",
84
- "Softmax",
85
- "Gather",
86
- "Expand",
87
- "Mul",
88
- "Sub",
89
- "Transpose"
90
  ],
91
- "weight_type": "QUInt8"
92
  },
93
  "decoder_model_merged": {
94
  "op_types": [
95
- "Sqrt",
96
- "Split",
97
- "If",
98
- "Unsqueeze",
99
- "Tanh",
100
- "Gemm",
101
  "MatMul",
102
- "Cast",
103
- "Div",
104
- "Pow",
105
  "Add",
106
- "Constant",
107
- "Reshape",
108
  "ConstantOfShape",
109
- "Range",
110
- "Slice",
 
 
 
 
111
  "Where",
112
- "Concat",
113
- "Shape",
 
 
 
114
  "Squeeze",
 
 
 
 
 
 
115
  "ReduceMean",
116
- "Softmax",
117
- "Gather",
118
- "Sub",
119
- "Mul",
120
- "Transpose"
121
  ],
122
  "weight_type": "QInt8"
123
  }
 
2
  "per_channel": false,
3
  "reduce_range": false,
4
  "per_model_config": {
5
+ "encoder_model": {
6
  "op_types": [
 
 
 
 
 
7
  "MatMul",
 
 
 
8
  "Add",
9
+ "Equal",
 
10
  "ConstantOfShape",
11
+ "Pow",
12
+ "Erf",
13
+ "Sub",
14
+ "Mul",
15
+ "Reshape",
16
+ "Div",
17
  "Where",
18
+ "Unsqueeze",
19
+ "Expand",
20
+ "Gather",
21
+ "Softmax",
22
+ "Transpose",
23
+ "Constant",
24
  "Shape",
 
25
  "ReduceMean",
26
+ "Concat",
27
+ "Sqrt",
28
+ "Slice",
29
+ "Conv"
 
30
  ],
31
+ "weight_type": "QUInt8"
32
  },
33
  "decoder_model": {
34
  "op_types": [
 
 
 
 
 
35
  "MatMul",
 
 
 
36
  "Add",
 
 
37
  "ConstantOfShape",
38
+ "Pow",
39
+ "Sub",
40
+ "Mul",
41
+ "Reshape",
42
+ "Div",
43
  "Where",
44
+ "Tanh",
45
+ "Unsqueeze",
46
+ "Gather",
47
+ "Softmax",
48
+ "Split",
49
+ "Slice",
50
  "Squeeze",
51
+ "Range",
52
+ "Transpose",
53
+ "Cast",
54
+ "Constant",
55
+ "Shape",
56
  "ReduceMean",
57
+ "Concat",
58
+ "Sqrt",
59
+ "Gemm"
 
 
60
  ],
61
  "weight_type": "QInt8"
62
  },
63
+ "decoder_with_past_model": {
64
  "op_types": [
 
 
 
65
  "MatMul",
 
 
66
  "Add",
 
 
 
67
  "ConstantOfShape",
68
+ "Pow",
69
+ "Sub",
70
+ "Mul",
71
+ "Reshape",
72
+ "Div",
73
  "Where",
74
+ "Tanh",
75
+ "Unsqueeze",
76
+ "Gather",
77
+ "Softmax",
78
+ "Split",
79
+ "Slice",
80
+ "Squeeze",
81
+ "Range",
82
+ "Transpose",
83
+ "Cast",
84
+ "Constant",
85
  "Shape",
86
  "ReduceMean",
87
+ "Concat",
88
+ "Sqrt",
89
+ "Gemm"
 
 
 
90
  ],
91
+ "weight_type": "QInt8"
92
  },
93
  "decoder_model_merged": {
94
  "op_types": [
 
 
 
 
 
 
95
  "MatMul",
 
 
 
96
  "Add",
 
 
97
  "ConstantOfShape",
98
+ "Pow",
99
+ "Sub",
100
+ "Mul",
101
+ "Gemm",
102
+ "Reshape",
103
+ "Div",
104
  "Where",
105
+ "Tanh",
106
+ "Unsqueeze",
107
+ "Gather",
108
+ "Softmax",
109
+ "Split",
110
  "Squeeze",
111
+ "Range",
112
+ "Transpose",
113
+ "Cast",
114
+ "Constant",
115
+ "If",
116
+ "Shape",
117
  "ReduceMean",
118
+ "Concat",
119
+ "Sqrt",
120
+ "Slice"
 
 
121
  ],
122
  "weight_type": "QInt8"
123
  }
special_tokens_map.json CHANGED
@@ -1,30 +1,6 @@
1
  {
2
- "bos_token": {
3
- "content": "<|endoftext|>",
4
- "lstrip": false,
5
- "normalized": true,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "<|endoftext|>",
11
- "lstrip": false,
12
- "normalized": true,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": {
17
- "content": "<|endoftext|>",
18
- "lstrip": false,
19
- "normalized": true,
20
- "rstrip": false,
21
- "single_word": false
22
- },
23
- "unk_token": {
24
- "content": "<|endoftext|>",
25
- "lstrip": false,
26
- "normalized": true,
27
- "rstrip": false,
28
- "single_word": false
29
- }
30
  }
 
1
  {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "unk_token": "<|endoftext|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  }
tokenizer.json CHANGED
@@ -9,7 +9,7 @@
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
12
- "normalized": true,
13
  "special": true
14
  }
15
  ],
 
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
12
+ "normalized": false,
13
  "special": true
14
  }
15
  ],
tokenizer_config.json CHANGED
File without changes
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5d6dbda62eac800cee527380ece0a33f7df91a801dc0529c3c35871cb1123276
3
  size 5176
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac92771c5176fd55689bb44c37e3467ce9f70d1a9d642ce3f6e6e4e449de33cd
3
  size 5176
vocab.json CHANGED
File without changes