init

.gitattributes

@@ -25,3 +25,36 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+coco_dataset/data/* filter=lfs diff=lfs merge=lfs -text
+coco_dataset/data/**/ filter=lfs diff=lfs merge=lfs -text
+coco_dataset/data/**/* filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/* filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/ filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/*/ filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/*" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/* filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/* filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/*" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/* filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/*" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/**" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/**/* filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/**/*" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/**/**" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/**/**/* filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/**/**/*" filter=lfs diff=lfs merge=lfs -text
+image_caption_dataset/coco_dataset/**/**/**/**/**/**/**" filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/* filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/*" filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/**" filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/**/* filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/**/*" filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/**/**" filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/**/**/* filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/**/**/*" filter=lfs diff=lfs merge=lfs -text
+coco_dataset/dummy_data/**/**/**" filter=lfs diff=lfs merge=lfs -text
+

README.md

@@ -0,0 +1,41 @@
+## Example
+
+The model is by no means a state-of-the-art model, but nevertheless 
+produces reasonable image captioning results. It was mainly fine-tuned 
+as a proof-of-concept for the 🤗 FlaxVisionEncoderDecoder Framework.
+
+The model can be used as follows:
+
+```python
+
+import requests
+from PIL import Image
+
+from transformers import ViTFeatureExtractor, AutoTokenizer, FlaxVisionEncoderDecoderModel
+
+loc = "ydshieh/flax-vit-gpt2-coco-en"
+
+feature_extractor = ViTFeatureExtractor.from_pretrained(loc)
+tokenizer = AutoTokenizer.from_pretrained(loc)
+model = FlaxVisionEncoderDecoderModel.from_pretrained(loc)
+
+# We will verify our results on an image of cute cats
+url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+with Image.open(requests.get(url, stream=True).raw) as img:
+    pixel_values = feature_extractor(images=img, return_tensors="np").pixel_values
+
+def generate_step(pixel_values):
+
+    output_ids = model.generate(pixel_values, max_length=16, num_beams=4).sequences
+    preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
+    preds = [pred.strip() for pred in preds]
+
+    return preds
+
+preds = generate_step(pixel_values)
+print(preds)
+
+# should produce
+# ['a cat laying on top of a couch next to another cat']
+
+```

coco_dataset/coco_dataset.py

@@ -0,0 +1,191 @@
+import csv
+import json
+import os
+
+import datasets
+import pandas as pd
+import numpy as np
+
+
+class ImageCaptionBuilderConfig(datasets.BuilderConfig):
+
+    def __init__(self, name, splits, **kwargs):
+
+        super().__init__(name, **kwargs)
+
+        self.splits = splits
+
+
+# TODO: Add BibTeX citation
+# Find for instance the citation on arxiv or on the dataset repo/website
+_CITATION = """\
+@InProceedings{None,
+    title = {COCO dataset},
+    author={...},
+    year={...}
+}
+"""
+
+# TODO: Add description of the dataset here
+# You can copy an official description
+_DESCRIPTION = """\
+
+"""
+
+# TODO: Add a link to an official homepage for the dataset here
+_HOMEPAGE = ""
+
+# TODO: Add the licence for the dataset here if you can find it
+_LICENSE = ""
+
+# TODO: Add link to the official dataset URLs here
+# The HuggingFace dataset library don't host the datasets but only point to the original files
+# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
+_URLs = {}
+
+
+# TODO: Name of the dataset usually match the script name with CamelCase instead of snake_case
+class ImageCaptionDataset(datasets.GeneratorBasedBuilder):
+    """TODO: Short description of my dataset."""
+
+    VERSION = datasets.Version("0.0.0")
+
+    BUILDER_CONFIG_CLASS = ImageCaptionBuilderConfig
+    BUILDER_CONFIGS = [
+        ImageCaptionBuilderConfig(name='2017', splits=['train', 'valid', 'test']),
+    ]
+    DEFAULT_CONFIG_NAME = "2017"
+
+    def _info(self):
+        # TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
+
+        feature_dict = {
+            "image_id": datasets.Value("int64"),
+            "caption_id": datasets.Value("int64"),
+            "caption": datasets.Value("string"),
+            "height": datasets.Value("int64"),
+            "width": datasets.Value("int64"),
+            "file_name": datasets.Value("string"),
+            "coco_url": datasets.Value("string"),
+            "image_path": datasets.Value("string"),
+        }
+
+        features = datasets.Features(feature_dict)
+
+        return datasets.DatasetInfo(
+            # This is the description that will appear on the datasets page.
+            description=_DESCRIPTION,
+            # This defines the different columns of the dataset and their types
+            features=features,  # Here we define them above because they are different between the two configurations
+            # If there's a common (input, target) tuple from the features,
+            # specify them here. They'll be used if as_supervised=True in
+            # builder.as_dataset.
+            supervised_keys=None,
+            # Homepage of the dataset for documentation
+            homepage=_HOMEPAGE,
+            # License for the dataset if available
+            license=_LICENSE,
+            # Citation for the dataset
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        """Returns SplitGenerators."""
+        # TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
+        # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name
+
+        data_dir = self.config.data_dir
+
+        splits = []
+        for split in self.config.splits:
+            if split == 'train':
+                dataset = datasets.SplitGenerator(
+                    name=datasets.Split.TRAIN,
+                    # These kwargs will be passed to _generate_examples
+                    gen_kwargs={
+                        "json_path": os.path.join(data_dir, f"captions_train{self.config.name}.json"),
+                        "image_dir": os.path.join(data_dir, f'train{self.config.name}'),
+                        "split": "train",
+                    }
+                )
+            elif split in ['val', 'valid', 'validation', 'dev']:
+                dataset = datasets.SplitGenerator(
+                    name=datasets.Split.VALIDATION,
+                    # These kwargs will be passed to _generate_examples
+                    gen_kwargs={
+                        "json_path": os.path.join(data_dir, f"captions_val{self.config.name}.json"),
+                        "image_dir": os.path.join(data_dir, f'val{self.config.name}'),
+                        "split": "valid",
+                    },
+                )
+            elif split == 'test':
+                dataset = datasets.SplitGenerator(
+                    name=datasets.Split.TEST,
+                    # These kwargs will be passed to _generate_examples
+                    gen_kwargs={
+                        "json_path": os.path.join(data_dir, f'image_info_test{self.config.name}.json'),
+                        "image_dir": os.path.join(data_dir, f'test{self.config.name}'),
+                        "split": "test",
+                    },
+                )
+            else:
+                continue
+
+            splits.append(dataset)
+
+        return splits
+
+    def _generate_examples(
+        # method parameters are unpacked from `gen_kwargs` as given in `_split_generators`
+        self, json_path, image_dir, split
+    ):
+        """ Yields examples as (key, example) tuples. """
+        # This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
+        # The `key` is here for legacy reason (tfds) and is not important in itself.
+
+        _features = ["image_id", "caption_id", "caption", "height", "width", "file_name", "coco_url", "image_path", "id"]
+        features = list(_features)
+
+        if split in "valid":
+            split = "val"
+
+        with open(json_path, 'r', encoding='UTF-8') as fp:
+            data = json.load(fp)
+
+        # list of dict
+        images = data["images"]
+        entries = images
+
+        # build a dict of image_id -> image info dict
+        d = {image["id"]: image for image in images}
+
+        # list of dict
+        if split in ["train", "val"]:
+            annotations = data["annotations"]
+
+            # build a dict of image_id ->
+            for annotation in annotations:
+                _id = annotation["id"]
+                image_info = d[annotation["image_id"]]
+                annotation.update(image_info)
+                annotation["id"] = _id
+
+            entries = annotations
+
+        for id_, entry in enumerate(entries):
+
+            entry = {k: v for k, v in entry.items() if k in features}
+
+            if split == "test":
+                entry["image_id"] = entry["id"]
+                entry["id"] = -1
+                entry["caption"] = -1
+
+            entry["caption_id"] = entry.pop("id")
+            entry["image_path"] = os.path.join(image_dir, entry["file_name"])
+
+            entry = {k: entry[k] for k in _features if k in entry}
+
+            print(entry)
+
+            yield str((entry["image_id"], entry["caption_id"])), entry

coco_dataset/dummy_data/annotations_trainval2017.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:803f8cc8c2bd46633be6866aa0230d25916e0c2de6cad85bf11573fbe352efc6
+size 7458

coco_dataset/dummy_data/captions_train2017.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3c77d344bd683c0e9b235892a2bc6db096bcd470feb80a5928b98696739a34d
+size 20225

coco_dataset/dummy_data/captions_val2017.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d081d092d0c7f7d052ce5759f366c0ddfe743418ea4d26ccad6a9d91ae0e4f51
+size 20148

coco_dataset/dummy_data/image_info_test-dev2017.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47b81764a5e1b843363a4ee847c12caa19f2f80fef0c43fb512110c575f27b61
+size 14535

coco_dataset/dummy_data/image_info_test2017.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eba76001d86881501288d8600812a43dcda2c3dd2f4bc46b84966ef45412ee80
+size 15439

coco_dataset/dummy_data/image_info_test2017.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e30ca36eba1cb0a631bf78201c92f508be9e105777d0c8b763edbc8b8517d5a2
+size 3498

coco_dataset/dummy_data/test2017.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abdc12551acb154ed65c38f493ce930ad22fffd74c1b1e1f811a46212eb28e91
+size 3023535

coco_dataset/dummy_data/train2017.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:370a953a7907709727f53c989d8282a1c0e47fb6fc0900cebf1c67ba362c6e73
+size 3590323

coco_dataset/dummy_data/val2017.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3625319b0adc788f918dea4a32a3c551323fd883888e8a03668cb47dd823b94
+size 2556555