initial commit

.gitattributes

@@ -25,3 +25,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+encoder filter=lfs diff=lfs merge=lfs -text
+decoder filter=lfs diff=lfs merge=lfs -text

decoder/.gitattributes

@@ -0,0 +1 @@
+model.h5 filter=lfs diff=lfs merge=lfs -text

decoder/keras_metadata_decoder.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8ccb81427499e2bd01421f651a779215faeb2306ee3c998daadd78a6b6fcf28
+size 10036

decoder/model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe2d82bd1ce2018eb05d618e5e33e1de50862f99b67670344a9b0440161e9bb5
+size 24349932

decoder/saved_model.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d522774f3f411985799ed44b15176540ae1d0966e2e21e05e23450d9c8544679
+size 797127

decoder/variables/.gitattributes

@@ -0,0 +1,3 @@
+variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+/Users/mervenoyan/Desktop/caption/image-captioning/encoder/variables/variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+/Users/mervenoyan/Desktop/caption/image-captioning/decoder/variables/variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text

decoder/variables/variables.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e49b95332c394b3c1f844f4275cd315f4e0760f9fc0ea085b2bf85b1726e323a
+size 24321506

encoder/.gitattributes

@@ -0,0 +1 @@
+model.h5 filter=lfs diff=lfs merge=lfs -text

encoder/keras_metadata_encoder.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:972a2170cf22e5827c66ccedc5c2d10b5bee7b9a3808316a3377c14f1a463b99
+size 1732

encoder/model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5b38e2f804ca719f2fe5a2fc31b4906bfc8502b5c057edfce08a9baa55443e6
+size 2109280

encoder/saved_model.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f99dcc649b8ded921c0ccd0f3ef375ff2a3a498f710abe60655d1ccb40c9b074
+size 37214

encoder/variables/.gitattributes

@@ -0,0 +1,2 @@
+variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+/Users/mervenoyan/Desktop/caption/image-captioning/encoder/variables/variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text

encoder/variables/variables.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c6e09edb4de9bb605fcaafb659672dfbce0dc7b7f8e730b8c054e1a3c80ad5c
+size 2099011

pipeline.py

@@ -0,0 +1,70 @@
+import tensorflow as tf
+import numpy as np 
+from tensorflow import keras
+import os
+from typing import Dict, List, Any
+import pickle
+from PIL import Image
+class PreTrainedPipeline():
+    def __init__(self, path=""):
+        
+        # load the model
+        self.decoder = keras.models.load_model(os.path.join(path, "decoder"))
+        self.decoder = keras.models.load_model(os.path.join(path, "encoder"))
+        
+        image_model = tf.keras.applications.InceptionV3(include_top=False,
+                                                weights='imagenet')
+        new_input = image_model.input
+        hidden_layer = image_model.layers[-1].output
+
+        self.image_features_extract_model = tf.keras.Model(new_input, hidden_layer)
+        
+        with open('tokenizer.pickle', 'rb') as handle:
+            self.tokenizer = pickle.load(handle)
+
+
+
+    def load_image(image_path):
+        img = tf.io.read_file(image_path)
+        img = tf.io.decode_jpeg(img, channels=3)
+        img = tf.image.resize(img, (299, 299))
+        img = tf.keras.applications.inception_v3.preprocess_input(img)
+        return img, image_path
+
+    def __call__(self, inputs: "Image.Image") -> List[Dict[str, Any]]:
+        """
+        Args:
+            inputs (:obj:`PIL.Image`):
+                The raw image representation as PIL.
+                No transformation made whatsoever from the input. Make all necessary transformations here.
+        Return:
+            A :obj:`list`:. The list contains items that are dicts should be liked {"label": "XXX", "score": 0.82}
+                It is preferred if the returned list is in decreasing `score` order
+        """
+
+        hidden = tf.zeros((1, 512))
+
+        temp_input = tf.expand_dims(load_image(image)[0], 0)
+        img_tensor_val = self.image_features_extract_model(temp_input)
+        img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0],
+                                                    -1,
+                                                    img_tensor_val.shape[3]))
+
+        features = self.encoder(img_tensor_val)
+
+        dec_input = tf.expand_dims([self.tokenizer.word_index['<start>']], 0)
+        result = []
+
+        for i in range(max_length):
+            predictions, hidden, attention_weights = self.decoder(dec_input,
+                                                            features,
+                                                            hidden)
+
+            predicted_id = tf.random.categorical(predictions, 1)[0][0].numpy()
+            result.append(self.tokenizer.index_word[predicted_id])
+
+            if self.tokenizer.index_word[predicted_id] == '<end>':
+                return result
+
+            dec_input = tf.expand_dims([predicted_id], 0)
+        return result