TeamAlerito commited on
Commit
5319378
1 Parent(s): f6b8980

Upload README.md

Browse files
Files changed (1) hide show
  1. README.md +100 -0
README.md ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - image-classification
4
+ library_name: generic
5
+ ---
6
+
7
+ ## Example
8
+
9
+ The model is by no means a state-of-the-art model, but nevertheless
10
+ produces reasonable image captioning results. It was mainly fine-tuned
11
+ as a proof-of-concept for the 🤗 FlaxVisionEncoderDecoder Framework.
12
+
13
+ The model can be used as follows:
14
+
15
+ **In PyTorch**
16
+ ```python
17
+
18
+ import torch
19
+ import requests
20
+ from PIL import Image
21
+ from transformers import ViTFeatureExtractor, AutoTokenizer, VisionEncoderDecoderModel
22
+
23
+
24
+ loc = "ydshieh/vit-gpt2-coco-en"
25
+
26
+ feature_extractor = ViTFeatureExtractor.from_pretrained(loc)
27
+ tokenizer = AutoTokenizer.from_pretrained(loc)
28
+ model = VisionEncoderDecoderModel.from_pretrained(loc)
29
+ model.eval()
30
+
31
+
32
+ def predict(image):
33
+
34
+ pixel_values = feature_extractor(images=image, return_tensors="pt").pixel_values
35
+
36
+ with torch.no_grad():
37
+ output_ids = model.generate(pixel_values, max_length=16, num_beams=4, return_dict_in_generate=True).sequences
38
+
39
+ preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
40
+ preds = [pred.strip() for pred in preds]
41
+
42
+ return preds
43
+
44
+
45
+ # We will verify our results on an image of cute cats
46
+ url = "http://images.cocodataset.org/val2017/000000039769.jpg"
47
+ with Image.open(requests.get(url, stream=True).raw) as image:
48
+ preds = predict(image)
49
+
50
+ print(preds)
51
+ # should produce
52
+ # ['a cat laying on top of a couch next to another cat']
53
+
54
+ ```
55
+
56
+ **In Flax**
57
+ ```python
58
+
59
+ import jax
60
+ import requests
61
+ from PIL import Image
62
+ from transformers import ViTFeatureExtractor, AutoTokenizer, FlaxVisionEncoderDecoderModel
63
+
64
+
65
+ loc = "ydshieh/vit-gpt2-coco-en"
66
+
67
+ feature_extractor = ViTFeatureExtractor.from_pretrained(loc)
68
+ tokenizer = AutoTokenizer.from_pretrained(loc)
69
+ model = FlaxVisionEncoderDecoderModel.from_pretrained(loc)
70
+
71
+ gen_kwargs = {"max_length": 16, "num_beams": 4}
72
+
73
+
74
+ # This takes sometime when compiling the first time, but the subsequent inference will be much faster
75
+ @jax.jit
76
+ def generate(pixel_values):
77
+ output_ids = model.generate(pixel_values, **gen_kwargs).sequences
78
+ return output_ids
79
+
80
+
81
+ def predict(image):
82
+
83
+ pixel_values = feature_extractor(images=image, return_tensors="np").pixel_values
84
+ output_ids = generate(pixel_values)
85
+ preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
86
+ preds = [pred.strip() for pred in preds]
87
+
88
+ return preds
89
+
90
+
91
+ # We will verify our results on an image of cute cats
92
+ url = "http://images.cocodataset.org/val2017/000000039769.jpg"
93
+ with Image.open(requests.get(url, stream=True).raw) as image:
94
+ preds = predict(image)
95
+
96
+ print(preds)
97
+ # should produce
98
+ # ['a cat laying on top of a couch next to another cat']
99
+
100
+ ```