Duplicate from florentgbelidji/blip_captioning

Co-authored-by: Florent Gbelidji <florentgbelidji@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,31 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,97 @@
+---
+tags:
+- image-to-text
+- image-captioning
+- endpoints-template
+license: bsd-3-clause
+library_name: generic
+---
+
+# Fork of [salesforce/BLIP](https://github.com/salesforce/BLIP) for a `image-captioning` task on 🤗Inference endpoint.
+
+This repository implements a `custom` task for `image-captioning` for 🤗 Inference Endpoints. The code for the customized pipeline is in the [pipeline.py](https://huggingface.co/florentgbelidji/blip_captioning/blob/main/pipeline.py).
+To use deploy this model a an Inference Endpoint you have to select `Custom` as task to use the `pipeline.py` file. -> _double check if it is selected_
+### expected Request payload
+```json
+{
+  "image": "/9j/4AAQSkZJRgABAQEBLAEsAAD/2wBDAAMCAgICAgMC....", // base64 image as bytes
+}
+```
+below is an example on how to run a request using Python and `requests`.
+## Run Request 
+1. prepare an image. 
+```bash
+!wget https://huggingface.co/datasets/mishig/sample_images/resolve/main/palace.jpg
+```
+2.run request
+
+```python
+import json
+from typing import List
+import requests as r
+import base64
+
+ENDPOINT_URL = ""
+HF_TOKEN = ""
+
+def predict(path_to_image: str = None):
+    with open(path_to_image, "rb") as i:
+        image = i.read()
+    payload = {
+        "inputs": [image],
+        "parameters": {
+                   "do_sample": True,
+                   "top_p":0.9,
+                   "min_length":5,
+                   "max_length":20
+        }
+    }
+    response = r.post(
+        ENDPOINT_URL, headers={"Authorization": f"Bearer {HF_TOKEN}"}, json=payload
+    )
+    return response.json()
+prediction = predict(
+    path_to_image="palace.jpg"
+)
+
+```
+Example parameters depending on the decoding strategy:
+
+1. Beam search
+
+``` 
+        "parameters": {
+                   "num_beams":5,
+                   "max_length":20
+        }
+```
+
+2. Nucleus sampling
+
+``` 
+        "parameters": {
+                   "num_beams":1,
+                   "max_length":20,
+                   "do_sample": True,
+                   "top_k":50,
+                   "top_p":0.95
+        }
+```
+
+3. Contrastive search
+
+``` 
+        "parameters": {
+                   "penalty_alpha":0.6,
+                   "top_k":4
+                   "max_length":512
+        }
+```
+
+See [generate()](https://huggingface.co/docs/transformers/v4.25.1/en/main_classes/text_generation#transformers.GenerationMixin.generate) doc for additional detail
+
+
+expected output
+```python
+['buckingham palace with flower beds and red flowers']
+```

handler.py

@@ -0,0 +1,49 @@
+# +
+from typing import  Dict, List, Any
+from PIL import Image
+import torch
+import os
+from io import BytesIO
+from transformers import BlipForConditionalGeneration, BlipProcessor
+# -
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+class EndpointHandler():
+    def __init__(self, path=""):
+        # load the optimized model
+        
+        self.processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") 
+        self.model = BlipForConditionalGeneration.from_pretrained(
+            "Salesforce/blip-image-captioning-base"
+        ).to(device)
+        self.model.eval()
+        self.model = self.model.to(device)
+        
+
+
+    def __call__(self, data: Any) -> Dict[str, Any]:
+        """
+        Args:
+            data (:obj:):
+                includes the input data and the parameters for the inference.
+        Return:
+            A :obj:`dict`:. The object returned should be a dict of one list like {"captions": ["A hugging face at the office"]} containing :
+                - "caption": A string corresponding to the generated caption.
+        """
+        inputs = data.pop("inputs", data)
+        parameters = data.pop("parameters", {})
+ 
+        raw_images = [Image.open(BytesIO(_img)) for _img in inputs]
+                                     
+        processed_image = self.processor(images=raw_images, return_tensors="pt") 
+        processed_image["pixel_values"] = processed_image["pixel_values"].to(device)
+        processed_image = {**processed_image, **parameters}
+        
+        with torch.no_grad():
+            out = self.model.generate(
+                **processed_image
+            )
+        captions = self.processor.batch_decode(out, skip_special_tokens=True)
+        # postprocess the prediction
+        return {"captions": captions}