Merge pull request #19 from pcuenca/main

encoding/vqgan-jax-encoding-yfcc100m-splitted.ipynb

@@ -0,0 +1,462 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d0b72877",
+   "metadata": {},
+   "source": [
+    "# vqgan-jax-encoding-yfcc100m"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "747733a4",
+   "metadata": {},
+   "source": [
+    "Same as `vqgan-jax-encoding-with-captions`, but for YFCC100M.\n",
+    "\n",
+    "This dataset was prepared by @borisdayma in Json lines format."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3b59489e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import io\n",
+    "\n",
+    "import requests\n",
+    "from PIL import Image\n",
+    "import numpy as np\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "import torch\n",
+    "import torchvision.transforms as T\n",
+    "import torchvision.transforms.functional as TF\n",
+    "from torchvision.transforms import InterpolationMode\n",
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "from torchvision.datasets.folder import default_loader\n",
+    "\n",
+    "import jax\n",
+    "from jax import pmap"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "511c3b9e",
+   "metadata": {},
+   "source": [
+    "## VQGAN-JAX model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bb408f6c",
+   "metadata": {},
+   "source": [
+    "`dalle_mini` is a local package that contains the VQGAN-JAX model and other utilities."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2ca50dc7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dalle_mini.vqgan_jax.modeling_flax_vqgan import VQModel"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b60da9a",
+   "metadata": {},
+   "source": [
+    "We'll use a VQGAN trained by using Taming Transformers and converted to a JAX model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "29ce8b15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = VQModel.from_pretrained(\"flax-community/vqgan_f16_16384\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c7c4c1e6",
+   "metadata": {},
+   "source": [
+    "## Dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd4c608e",
+   "metadata": {},
+   "source": [
+    "I splitted the files to do the process iteratively. Pandas struggles with memory and `datasets` has problems when filtering files, as described [in this issue](https://github.com/huggingface/datasets/issues/2644)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "6c058636",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "81b19eca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "yfcc100m = Path('/sddata/dalle-mini/YFCC100M_OpenAI_subset')\n",
+    "# Images are 'sharded' from the following directory\n",
+    "yfcc100m_images = yfcc100m/'data'/'images'\n",
+    "yfcc100m_metadata_splits = yfcc100m/'metadata_splitted'\n",
+    "yfcc100m_output = yfcc100m/'metadata_encoded'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "40873de9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_04'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_25'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_17'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_10'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_22'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_28'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_09'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_03'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_07'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_26'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_14'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_19'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_13'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_21'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_00'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_02'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_08'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_11'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_29'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_23'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_24'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_16'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_05'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_01'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_12'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_18'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_20'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_27'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_15'),\n",
+       " PosixPath('/sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_06')]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_splits = [x for x in yfcc100m_metadata_splits.iterdir() if x.is_file()]\n",
+    "all_splits"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f604e3c9",
+   "metadata": {},
+   "source": [
+    "### Cleanup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "dea06b92",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def image_exists(root: str, name: str, ext: str):\n",
+    "    image_path = (Path(root)/name[0:3]/name[3:6]/name).with_suffix(ext)\n",
+    "    return image_path.exists()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "1d34d7aa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class YFC100Dataset(Dataset):\n",
+    "    def __init__(self, image_list: pd.DataFrame, images_root: str, image_size: int, max_items=None):\n",
+    "        \"\"\"\n",
+    "        :param image_list: DataFrame with clean entries - all images must exist.\n",
+    "        :param images_root: Root directory containing the images\n",
+    "        :param image_size: Image size. Source images will be resized and center-cropped.\n",
+    "        :max_items: Limit dataset size for debugging\n",
+    "        \"\"\"\n",
+    "        self.image_list = image_list\n",
+    "        self.images_root = Path(images_root)\n",
+    "        if max_items is not None: self.image_list = self.image_list[:max_items]\n",
+    "        self.image_size = image_size\n",
+    "        \n",
+    "    def __len__(self):\n",
+    "        return len(self.image_list)\n",
+    "    \n",
+    "    def _get_raw_image(self, i):\n",
+    "        image_name = self.image_list.iloc[0].key\n",
+    "        image_path = (self.images_root/image_name[0:3]/image_name[3:6]/image_name).with_suffix('.jpg')\n",
+    "        return default_loader(image_path)\n",
+    "    \n",
+    "    def resize_image(self, image):\n",
+    "        s = min(image.size)\n",
+    "        r = self.image_size / s\n",
+    "        s = (round(r * image.size[1]), round(r * image.size[0]))\n",
+    "        image = TF.resize(image, s, interpolation=InterpolationMode.LANCZOS)\n",
+    "        image = TF.center_crop(image, output_size = 2 * [self.image_size])\n",
+    "        # FIXME: np.array is necessary in my installation, but it should be automatic\n",
+    "        image = torch.unsqueeze(T.ToTensor()(np.array(image)), 0)\n",
+    "        image = image.permute(0, 2, 3, 1).numpy()\n",
+    "        return image\n",
+    "        \n",
+    "    def __getitem__(self, i):\n",
+    "        image = self._get_raw_image(i)\n",
+    "        image = self.resize_image(image)\n",
+    "        # Just return the image, not the caption\n",
+    "        return image"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "62ad01c3",
+   "metadata": {},
+   "source": [
+    "## Encoding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "88f36d0b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def encode(model, batch):\n",
+    "    print(\"jitting encode function\")\n",
+    "    _, indices = model.encode(batch)\n",
+    "\n",
+    "#     # FIXME: The model does not run in my computer (no cudNN currently installed) - faking it\n",
+    "#     indices = np.random.randint(0, 16384, (batch.shape[0], 256))\n",
+    "    return indices"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d1f45dd8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#FIXME\n",
+    "# import random\n",
+    "# model = {}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "1f35f0cb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from flax.training.common_utils import shard\n",
+    "\n",
+    "def superbatch_generator(dataloader):\n",
+    "    iter_loader = iter(dataloader)\n",
+    "    for batch in iter_loader:\n",
+    "        batch = batch.squeeze(1)\n",
+    "        # Skip incomplete last batch\n",
+    "        if batch.shape[0] == dataloader.batch_size:\n",
+    "            yield shard(batch)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "2210705b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import jax\n",
+    "\n",
+    "def encode_captioned_dataset(dataset, output_jsonl, batch_size=32, num_workers=16):\n",
+    "    if os.path.isfile(output_jsonl):\n",
+    "        print(f\"Destination file {output_jsonl} already exists, please move away.\")\n",
+    "        return\n",
+    "    \n",
+    "    num_tpus = jax.device_count()\n",
+    "    dataloader = DataLoader(dataset, batch_size=num_tpus*batch_size, num_workers=num_workers)\n",
+    "    superbatches = superbatch_generator(dataloader)\n",
+    "    \n",
+    "    p_encoder = pmap(lambda batch: encode(model, batch))\n",
+    "\n",
+    "    # We save each superbatch to avoid reallocation of buffers as we process them.\n",
+    "    # We keep the file open to prevent excessive file seeks.\n",
+    "    with open(output_jsonl, \"w\") as file:\n",
+    "        iterations = len(dataset) // (batch_size * num_tpus)\n",
+    "        for n in tqdm(range(iterations)):\n",
+    "            superbatch = next(superbatches)\n",
+    "            encoded = p_encoder(superbatch.numpy())\n",
+    "            encoded = encoded.reshape(-1, encoded.shape[-1])\n",
+    "\n",
+    "            # Extract fields from the dataset internal `image_list` property, and save to disk\n",
+    "            # We need to read from the df because the Dataset only returns images\n",
+    "            start_index = n * batch_size * num_tpus\n",
+    "            end_index = (n+1) * batch_size * num_tpus\n",
+    "            keys = dataset.image_list[\"key\"][start_index:end_index].values\n",
+    "            captions = dataset.image_list[\"caption\"][start_index:end_index].values\n",
+    "#             encoded_as_string = list(map(lambda item: np.array2string(item, separator=',', max_line_width=50000, formatter={'int':lambda x: str(x)}), encoded))\n",
+    "            batch_df = pd.DataFrame.from_dict({\"key\": keys, \"caption\": captions, \"encoding\": encoded})\n",
+    "            batch_df.to_json(file, orient='records', lines=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "7704863d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing /sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_04\n",
+      "54024 selected from 500000 total entries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:absl:Starting the local TPU driver.\n",
+      "INFO:absl:Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://\n",
+      "INFO:absl:Unable to initialize backend 'tpu': Invalid argument: TpuPlatform is not available.\n",
+      "  0%|                                                                                        | 0/31 [00:00<?, ?it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "jitting encode function\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|███████████████████████████████████████████████████████████████████████████████| 31/31 [00:02<00:00, 10.61it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing /sddata/dalle-mini/YFCC100M_OpenAI_subset/metadata_splitted/metadata_split_25\n",
+      "99530 selected from 500000 total entries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  3%|██▌                                                                             | 1/31 [00:01<00:53,  1.79s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "jitting encode function\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|███████████████████████████████████████████████████████████████████████████████| 31/31 [00:03<00:00,  9.92it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for split in all_splits:\n",
+    "    print(f\"Processing {split}\")\n",
+    "    df = pd.read_json(split, orient=\"records\", lines=True)\n",
+    "    df['image_exists'] = df.apply(lambda row: image_exists(yfcc100m_images, row['key'], '.' + row['ext']), axis=1)\n",
+    "    print(f\"{len(df[df.image_exists])} selected from {len(df)} total entries\")\n",
+    "    \n",
+    "    df = df[df.image_exists]\n",
+    "    captions = df.apply(lambda row: ' '.join([row[\"title_clean\"], row[\"description_clean\"]]), axis=1)\n",
+    "    df[\"caption\"] = captions.values\n",
+    "    \n",
+    "    dataset = YFC100Dataset(\n",
+    "        image_list = df,\n",
+    "        images_root = yfcc100m_images,\n",
+    "        image_size = 256,\n",
+    "#         max_items = 2000,\n",
+    "    )\n",
+    "    \n",
+    "    encode_captioned_dataset(dataset, yfcc100m_output/split.name, batch_size=64, num_workers=16)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8953dd84",
+   "metadata": {},
+   "source": [
+    "----"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

encoding/vqgan-jax-encoding-yfcc100m.ipynb

@@ -0,0 +1,1609 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d0b72877",
+   "metadata": {},
+   "source": [
+    "# vqgan-jax-encoding-yfcc100m"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ba7b31e6",
+   "metadata": {},
+   "source": [
+    "Same as `vqgan-jax-encoding-with-captions`, but for YFCC100M.\n",
+    "\n",
+    "This dataset was prepared by @borisdayma in Json lines format."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3b59489e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import io\n",
+    "\n",
+    "import requests\n",
+    "from PIL import Image\n",
+    "import numpy as np\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "import torch\n",
+    "import torchvision.transforms as T\n",
+    "import torchvision.transforms.functional as TF\n",
+    "from torchvision.transforms import InterpolationMode\n",
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "from torchvision.datasets.folder import default_loader\n",
+    "\n",
+    "import jax\n",
+    "from jax import pmap"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "511c3b9e",
+   "metadata": {},
+   "source": [
+    "## VQGAN-JAX model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bb408f6c",
+   "metadata": {},
+   "source": [
+    "`dalle_mini` is a local package that contains the VQGAN-JAX model and other utilities."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2ca50dc7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dalle_mini.vqgan_jax.modeling_flax_vqgan import VQModel"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b60da9a",
+   "metadata": {},
+   "source": [
+    "We'll use a VQGAN trained by using Taming Transformers and converted to a JAX model."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ad05a1bd",
+   "metadata": {},
+   "source": [
+    "**Disabling** Does not work in my local system right now."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "29ce8b15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#model = VQModel.from_pretrained(\"flax-community/vqgan_f16_16384\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c7c4c1e6",
+   "metadata": {},
+   "source": [
+    "## Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 79,
+   "id": "33861477",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 80,
+   "id": "81b19eca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "yfcc100m = Path('/sddata/dalle-mini/YFCC100M_OpenAI_subset')\n",
+    "# Images are 'sharded' from the following directory\n",
+    "yfcc100m_images = yfcc100m/'data'/'images'\n",
+    "yfcc100m_metadata = yfcc100m/'metadata_YFCC100M.jsonl'\n",
+    "yfcc100m_output = yfcc100m/'metadata_encoded.jsonl'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1c58bb4a",
+   "metadata": {},
+   "source": [
+    "### Cleanup"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1a14ae3d",
+   "metadata": {},
+   "source": [
+    "We need to select entries with images that exist. Otherwise we can't build batches because `Dataloader` does not support `None` in batches. We use Huggingface Datasets, I understand they support threaded reading of jsonl files, and I was running out of memory when using pandas."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 81,
+   "id": "7811648c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import datasets\n",
+    "from datasets import Dataset, load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 82,
+   "id": "753659fe",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using custom data configuration default-57592e8ed16d752b\n",
+      "Reusing dataset json (/home/pedro/.cache/huggingface/datasets/json/default-57592e8ed16d752b/0.0.0/793d004298099bd3c4e61eb7878475bcf1dc212bf2e34437d85126758720d7f9)\n"
+     ]
+    }
+   ],
+   "source": [
+    "dataset = load_dataset(\"json\", data_files=[str(yfcc100m_metadata)])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 83,
+   "id": "9343df1b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['photoid', 'uid', 'unickname', 'datetaken', 'dateuploaded', 'capturedevice', 'title', 'description', 'usertags', 'machinetags', 'longitude', 'latitude', 'accuracy', 'pageurl', 'downloadurl', 'licensename', 'licenseurl', 'serverid', 'farmid', 'secret', 'secretoriginal', 'ext', 'marker', 'key', 'title_clean', 'description_clean'],\n",
+       "    num_rows: 14825233\n",
+       "})"
+      ]
+     },
+     "execution_count": 83,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset = dataset['train']\n",
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 84,
+   "id": "c4794c29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def image_exists(root: str, name: str, ext: str):\n",
+    "    image_path = (Path(root)/name[0:3]/name[3:6]/name).with_suffix(ext)\n",
+    "    return image_path.exists()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 90,
+   "id": "1b500078",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def select_existing_rows(examples):\n",
+    "    # Select lists we want to keep\n",
+    "    keys = examples['key']\n",
+    "    titles_clean = examples['title_clean']\n",
+    "    descriptions_clean = examples.get('description_clean', '')\n",
+    "    exts = examples['ext']\n",
+    "    \n",
+    "    result = {'key': [], 'title_clean': [], 'description_clean': [], 'ext': []}\n",
+    "    for i, image_name in enumerate(keys):\n",
+    "        print(i, image_name)\n",
+    "        if image_exists(root=str(yfcc100m_images), name=image_name, ext='.' + exts[i]):\n",
+    "            result[\"key\"].append(image_name)\n",
+    "            result[\"title_clean\"].append(titles_clean[i])\n",
+    "            result[\"description_clean\"].append(descriptions_clean[i])\n",
+    "            result[\"ext\"].append(exts[i])\n",
+    "    print(f'returning {len(result[\"key\"])}')\n",
+    "    return result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "id": "467378c1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b72e866c3f174e9e9aa2430e204f2baf",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Selecting rows with images that exist:   0%|          | 0/14826 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0 d29e7c6a3028418c64eb15e3cf577c2\n",
+      "1 d29f01b149167d683f9ddde464bb3db\n",
+      "2 d296e9e34bdae41edb6c679ff824ab2a\n",
+      "3 d29ce96395848478b1e8396e44899\n",
+      "4 d29abf32c4e12ff881f975b70e0cec0\n",
+      "5 d298a61f2f7be6c9e2c2af81755b489\n",
+      "6 d29b1b973ab1a95a37cd4cda37999fb\n",
+      "7 d290d566266ad568e94128d4135b41a\n",
+      "8 d29b1ac2a497b0d9a4a43c3a51d13fb\n",
+      "9 d29ebe6c96f53b2f5d7f5eed9b2b2898\n",
+      "10 d29ec1b3f75749a231ee1d9d206baf6e\n",
+      "11 d290bee419ce98d9a79ccf512a47a79\n",
+      "12 d29bc1eff62a477131516c40a54f2dce\n",
+      "13 d292a123bcf58e13128d2067593d81\n",
+      "14 d294424637d532d8cfbcf2ca99b85f\n",
+      "15 d29a51d8502f531115b108d59c811ab\n",
+      "16 d29a9f0fce210c7e050877a53697031\n",
+      "17 d290c750469f11795ed85fa62e4b52\n",
+      "18 d29e13badf42d839b421478be4452dbe\n",
+      "19 d29c1d635348aa35474a90f57aafb7\n",
+      "20 d291a7c7c71455d5b3cdd97ca5e4c\n",
+      "21 d295f95d7cb204dc812a476af5f4f8a\n",
+      "22 d2932ecd1053165aa3d7b9e68547e0b6\n",
+      "23 d29cd5a4b1d6a759b63df357ef2b\n",
+      "24 d294e885117ca7d9b328c5b9388f52\n",
+      "25 d2999b54832bb275a7e2eea47e98f11\n",
+      "26 d29f89d491812beb84e62223b4541d7\n",
+      "27 d2993599afe456ba786060129fc9cdfd\n",
+      "28 d290ceb78d0f7c8c49930cd96b12b27\n",
+      "29 d29db640e6943c341e3df6b4a815a17\n",
+      "30 d29d8ae6354fd9c1613ac3750feb298\n",
+      "31 d29b7d6de63ce541b71ceb78745fbfc\n",
+      "32 d290cc9739e6f554b8f27f6496af5a6\n",
+      "33 d29dcfe9b1c7381614d5bd8290d435\n",
+      "34 d297e414783424b8d8339d8c9b54ca72\n",
+      "35 d2918885fdc74e96cdec3fd49e409667\n",
+      "36 d29d30289c89a2e9fc6234283b0397\n",
+      "37 d291536c1f1b3be24034663d3e57c84b\n",
+      "38 d29cccb1954ac8268963b8614d4541\n",
+      "39 d2945c8676633807e51169a7d123f49\n",
+      "40 d29b46d79ae192685dfebbafe681bf5\n",
+      "41 d298c4194cae157a78d9c85f6965ced6\n",
+      "42 d29f4f4050493b5b11bf029731250\n",
+      "43 d29fff53248edfe2539d2e2edf9bc4e\n",
+      "44 d2913017cf52bd6b239c16119ee955d\n",
+      "45 d29b53425360629f945b442e1819182b\n",
+      "46 d29f737d2137b6a58c8a3db2673dfa\n",
+      "47 d295c31c46ee0107f3224a12bb18e0\n",
+      "48 d29735527519d9efd3477ede346d077\n",
+      "49 d29549dc1a2f31e65add5159aca6ca7e\n",
+      "50 d29a56bba12ada01da573a325d2bbd\n",
+      "51 d29eb4236f7a4564cb0368c98b9d15a\n",
+      "52 d29417751c506a499af2bf9bba1c91dc\n",
+      "53 d299335cdf1679c9a5fe6b655e22cfc6\n",
+      "54 d29ad98d3118d6e21a94156f427812d3\n",
+      "55 d29a7bd530fa6cec73f55f5fdec35\n",
+      "56 d2983c9adab124234cf170b157d986aa\n",
+      "57 d299aa496b76993e1fbfaca5fadefa82\n",
+      "58 d29444db0139d8bdbdd96723aaba0\n",
+      "59 d29c2fcbaeb4a6bd7b7eb13e467823\n",
+      "60 d29b9d5c68124a5e56a4594974c9e7ec\n",
+      "61 d29c941e4349d152939733a01debb9ce\n",
+      "62 d2971493e2bdea2a48c3e3f7b9f3b9f9\n",
+      "63 d29f28e4b5254594fb581803aeaf1d7\n",
+      "64 d292145142221a995b2b17ef267fd5\n",
+      "65 d299bbbad76835c2edf1f012bd899883\n",
+      "66 d29462a3fdd5c994b2a09958f4413f39\n",
+      "67 d294a4f2273696fbcbc52c9ae3fa4cc7\n",
+      "68 d29b29cbf37f4b3278d34a9e2274cdc\n",
+      "69 d294f51aaae56a1eca32fac5551330c9\n",
+      "70 d29925d6bef3c5e318a6ef9461281ae2\n",
+      "71 d2914df9dabb24aad3b9a14f76bcaaf5\n",
+      "72 d290aae57d883ac77e882f082725753\n",
+      "73 d29d63a276a23f42f2011975bbb1432\n",
+      "74 d2914a2414672f2edc55afe090faa68b\n",
+      "75 d2943dd2a17e8b96766f47a75beceb6\n",
+      "76 d2915b7155c94423748e2a5f102d273d\n",
+      "77 d292f2a2d138a6606ec6acbea4d4b8a6\n",
+      "78 d29bf6fa58a7401fb2efd8f7b55473e\n",
+      "79 d29a7085f4c7a3ef1caab1d33c7772\n",
+      "80 d29e29aa2149be589126184fa6ba95b\n",
+      "81 d29d6e2963309e7b5c5978ea71f593\n",
+      "82 d291adb1933c79228e1fdfe1762f5b9f\n",
+      "83 d294bcf7cc7eeb78fa6439c66c135359\n",
+      "84 d29d12c15f67b3b6968abe771a5fd0\n",
+      "85 d29ba798d4f1c1e8cbcb46beda14c8f\n",
+      "86 d29056682c23fa206f7c952e512499ee\n",
+      "87 d2969725825c9da68e49c0e7be8daf1d\n",
+      "88 d291bc5a6b35c53f4117ad2415baa3\n",
+      "89 d29d45ca9fb464c9c60561fae2948\n",
+      "90 d291d29ff97c66bf1c6b4c956ed81\n",
+      "91 d298d5e0d479c7ec8add78c4aa80\n",
+      "92 d2943f69e59528b6e4a9a696763545\n",
+      "93 d299e68a3d7d50448da951f8792693b\n",
+      "94 d291ea441ccf3cc8f34073d4ce4d8d20\n",
+      "95 d29680dcc84f825828708b3d9427a8ae\n",
+      "96 d291201e72010356e91c88917aacfc\n",
+      "97 d297a5b71895f7bfbfb3d156d8ed3b83\n",
+      "98 d29d7e64a92ea1bd6ba08ff76c1a3bd8\n",
+      "99 d291f75dc99bee639fd680af7e7a4fb0\n",
+      "100 d29ddb6ca8a49f3369733f4d3a8887\n",
+      "101 d292ff4251b9a36b19fa6cbbc87851f\n",
+      "102 d29ca1839cfbb1c40ac2ba9cb30dea3\n",
+      "103 d29919029e2eeec95d7a41e9472e86e\n",
+      "104 d29b393acb76bf3dd45f1b4ef4f513\n",
+      "105 d29bcff1a7b2f4b109b42e99a49cb7\n",
+      "106 d2909c7f45b39247ba0c2ed811067ee\n",
+      "107 d29964417f32c1420bf235613af9c9\n",
+      "108 d29a9e825141d6605e6d9c4e658ae7f0\n",
+      "109 d29f2e7fea71dc911403e8c2b12414f\n",
+      "110 d2999b51a0e9a37f7d670ac979ae8c0\n",
+      "111 d2944c2f78b758c1b141104492f3f2\n",
+      "112 d2934e0baba9f344e85f7326e902b97\n",
+      "113 d29bda80ffd5d46aa798d715ff2c0\n",
+      "114 d2912c43811337b61f9c119f35781e56\n",
+      "115 d2945a077f98fdbde0fca07e22a41\n",
+      "116 d29aa68847eb4a66e71ab2626f84e\n",
+      "117 d291edb7ec46c41048967ab971c3c29c\n",
+      "118 d29ba93b2642d9f937316694da15b22b\n",
+      "119 d292aa457e9d7c8ffe8e55761f7d4f5\n",
+      "120 d29da11457a35654323fa3d93834a34\n",
+      "121 d295ba252458b15eab8957ac679509c\n",
+      "122 d299e43dd7d1ea48ddc02a823ddd7cc\n",
+      "123 d29d94f1c378e42b6fe990f76b94fe6\n",
+      "124 d29c52f12fede24a2caf6170655b558d\n",
+      "125 d2927cdb877b23f7c8356f8619aafc\n",
+      "126 d296d62fbd28d32a4d6f9f42a336ffdf\n",
+      "127 d294691643464e92e6afea29b7e6784\n",
+      "128 d29e9baa87e9b85136f85a397bcd126\n",
+      "129 d29a539bdde71c478e3655983d1ea\n",
+      "130 d29ff2366b398bac938d46e82945141\n",
+      "131 d29998f21f7db92b129b3567da7a546\n",
+      "132 d29c9c1908eeb3fd0fd8737ba53b59f\n",
+      "133 d29ae0f149cc6b67fefcaa79add5b873\n",
+      "134 d2988df2a66eccca85d12020f1896558\n",
+      "135 d29f85ed0efd46c7a46539e18a622\n",
+      "136 d296bfdc77c0cb5479361fdee3fa6\n",
+      "137 d29479e2d8f288ea2445659243bea37\n",
+      "138 d29bed60bd16db97ef5ec5f65cb7f1\n",
+      "139 d29595ef1a163f68659b56815b2fb21\n",
+      "140 d29f6ef4562c75bdc5d82dfd49394a\n",
+      "141 d2911c3e4ed5a8574991583e9e713f49\n",
+      "142 d29094b0977ffff0ff18a7715a96bda6\n",
+      "143 d291a0104cc6b978ddfb6d9eee0ec7f\n",
+      "144 d29d542a1dd0a0682c355e7ad45db14\n",
+      "145 d29f79dd32c418886791cbdb4ebe90f1\n",
+      "146 d29aeec1dced477c94bd396fc3127\n",
+      "147 d293eaabfefd938d71278825339494\n",
+      "148 d29e64339b9568b37fd4bdd7edb9836a\n",
+      "149 d298a529ba8a82918fe096e81110981e\n",
+      "150 d29ce5cf8539329ae4204e6ce327b2b\n",
+      "151 d29142b3b0e68e94a8522dfd3a2b690\n",
+      "152 d29e96e6b1b7eff768dff0ed8955edf1\n",
+      "153 d29265dae85a59cdd9615daffbcefe5\n",
+      "154 d29d9b1d50f747cb9adfc2b32fc32c\n",
+      "155 d29c5e12470d4e366c4bb8fa9baddc\n",
+      "156 d294fe93887dd377520c64839dc139\n",
+      "157 d29371933762af2ed19cafc3dd4fabd9\n",
+      "158 d2981a42980199daf93915c1eef93e\n",
+      "159 d29ea4ef385cf37b9045b06b4ba7252\n",
+      "160 d299a091d31f65129d3396a58f92642d\n",
+      "161 d29f3628139676f4ceb6bfe0cbc9aaf6\n",
+      "162 d29b114fa2616f46bcebb37f5d9f59d\n",
+      "163 d299a8d174e1f54ec383a241a8bb50ae\n",
+      "164 d2959843a6d4fc265637b63543a9419a\n",
+      "165 d29638c081713aa4dc5cd32ce88c158d\n",
+      "166 d2967886b6b0cbfb6b98b175822c956a\n",
+      "167 d29a6bc47f95f7668fbf43a1a1e947\n",
+      "168 d29e6265a813f02f2dcd114e5f2748be\n",
+      "169 d29960478ff0d270e822e84644638f7\n",
+      "170 d290a8ea4c1d4fda164e6ba0f978876c\n",
+      "171 d2934645777ae1ec25b2a4dfeb6036ed\n",
+      "172 d299ecad1261a988e0dcce7d1c7f6c42\n",
+      "173 d29b6eb6dc77c7be34481fa54754c\n",
+      "174 d2956739c7f8791ed7d4aff4f92b948\n",
+      "175 d293aa871b9e4c6913c43e91cf48040\n",
+      "176 d292d08afc78df9d38f6f1f1f28ec0\n",
+      "177 d29f448b6f32d4b9b64719beaaab3e2\n",
+      "178 d29e4bf8ed852972ff75aa6e4e6964\n",
+      "179 d29a2088d388e1799d144cdd88025ff\n",
+      "180 d29ad328dfb255da22af6aa72f15888\n",
+      "181 d29cf0f71eee59ccdecc8cb9e1aef482\n",
+      "182 d29a84ccf19a324aff1ece8d1831c8d\n",
+      "183 d298b98b2dcd287d31b7e5446ba8284\n",
+      "184 d29055e4ee873415ae12477e46fba\n",
+      "185 d2922e6430bb75a26072149343cc191a\n",
+      "186 d2955ecab0c76472d6f9dad25d165b8b\n",
+      "187 d29113a376ed16c6c1a65370cb845e65\n",
+      "188 d2944e25fb89b2466feb547d29e63975\n",
+      "189 d29d40ee1841d4dcf143925564f347\n",
+      "190 d2988f40eb3522d26574e7947840e116\n",
+      "191 d29fbc1e7c3c137f783ba57222cd98f6\n",
+      "192 d29fab615e3124424ba7eb1726c64b\n",
+      "193 d29e8e17d964c0aea5e52fe3c8c28270\n",
+      "194 d29b2838685267fc64f64a41211df2f5\n",
+      "195 d29a8a9d8edacfece418c8ea31eaf6\n",
+      "196 d29d6d371f46b5a31dd175a8a7f41\n",
+      "197 d29fbb7cc17c4e374232af541d4e4240\n",
+      "198 d293279e99aecab63ef09872b93f8d\n",
+      "199 d29fb59416e4c74dbed06129b130d828\n",
+      "200 d290dd1a636d24fa2fc96ef667f91a3\n",
+      "201 d29dc37487691ba4417611ab9b1187\n",
+      "202 d29f93114ac4dc3baace4d5e0bd3e57\n",
+      "203 d294259c4b3c35a1559d7f742a36c034\n",
+      "204 d29470f8213e53e2aa7164f3623ccc1f\n",
+      "205 d29faf1d9d6b7f77deb5385ea85111f\n",
+      "206 d29b18455b80dc37d81024d1ab6a99f\n",
+      "207 d29eb148320276099cd37f74b5b9ed8\n",
+      "208 d2961799bc3a8866450f9dedf3753da\n",
+      "209 d29826568875bdfe2f9b479a7e64ac4\n",
+      "210 d2928972156f2e5372cce81c5434655\n",
+      "211 d2992f331841f8c161554ea0f41faff\n",
+      "212 d291c33cbc5075d4be40b790c7435e\n",
+      "213 d29c655b787cb0e02be7272a57686cfa\n",
+      "214 d2939d29b1d1d5c3aa21d3b46127a1f8\n",
+      "215 d29839b090823f6d915a2bf4338b5b\n",
+      "216 d29334188dda37bfc48b84d1c2ad9f\n",
+      "217 d29e7e8ebfaae9b98b4141eb5242fb1\n",
+      "218 d295057d1bf5e65929485f92a4c1ab\n",
+      "219 d29184e22ba47ae9c449fc21086a82\n",
+      "220 d29e384d3221b25ec6221376c1a025d8\n",
+      "221 d290b2c211d229a262082c646856d57\n",
+      "222 d295472dea1c94a5a4a2443f83b82\n",
+      "223 d29ca6ac3858de4c4a0b6fd88547aac\n",
+      "224 d2956a39fbc3081f9fe394aff50aa\n",
+      "225 d29dc4779f78f63bca6252c4bd65bd46\n",
+      "226 d29960f8b3be7860b0af7497bd16d6bd\n",
+      "227 d2912134838dc88b389b60571f73c358\n",
+      "228 d295fc5a26d7da3cfa4fd4a824bf0\n",
+      "229 d29b4efbfa2b68093ee70fb2381fbfc\n",
+      "230 d293694f84e8ffafa8f42d55941bbd76\n",
+      "231 d29833fb44ce2852ef764a8fa87631d4\n",
+      "232 d296962ad593259fc695a76170c4f097\n",
+      "233 d2915c686462926e7e53b112ddcd1ac2\n",
+      "234 d298a5dcab9e7da513cf278d3e19f2\n",
+      "235 d29266df8098c511507828f4632363\n",
+      "236 d296413b2724e3c51954834ded46211b\n",
+      "237 d29514a1ed5b50893427ebefe1e5f8ab\n",
+      "238 d298bce28583de4cb34f279882d66bd\n",
+      "239 d2906ed28478ab13689e4ef0165376a4\n",
+      "240 d293d91cc7bad938f8cf20b372ac93b\n",
+      "241 d29fc131a3fe8ca955e6b59d2c8c981\n",
+      "242 d29215ab9be9ff9185479e24cf4ce56a\n",
+      "243 d29f9bc02cb942b447e9f99db525c\n",
+      "244 d29149d031f6d431bd3015219f275bd5\n",
+      "245 d29aae5bdb354bd96e46cf598dadb0\n",
+      "246 d2932362faf4ca9ac8697cbdd6c34bf\n",
+      "247 d2929a57d3d8a6bea06b5f4f9afb452\n",
+      "248 d29a5ac5d9ac764db856509640a5142a\n",
+      "249 d299c482f3314c3dad947db3a3156\n",
+      "250 d29d1183e0959ae7ac6c8e7b66374db\n",
+      "251 d29953bd569ce38442fc69fbcd8bd83c\n",
+      "252 d291a8ac4849b85a59b8c68dc752d0\n",
+      "253 d291e4aed5fb43f4f325d11ad66af251\n",
+      "254 d29675b4a3dfee18769a3ac674c1f3f\n",
+      "255 d29e648fed5b29ad1c152aa9cea0c6fc\n",
+      "256 d29b629bab26f59f9117f69a679db8\n",
+      "257 d29c593a5f55b48a9aefc728a6ccda8c\n",
+      "258 d29a4fe79f1cf5298b9feac44547d56\n",
+      "259 d290c12092a99314f219c6a4e6387eb1\n",
+      "260 d29fb6bcfb2d1e53fe919c26df36b578\n",
+      "261 d29482e12c94103037566fa2b227b21\n",
+      "262 d29e6ccc3336f45a4f216c83d5f72f1\n",
+      "263 d2931ac79c95d7526bf543407a54c3\n",
+      "264 d29450397c2bda2c15071e551dc4cfe\n",
+      "265 d29bd6f895319993cfa28439d6f561e7\n",
+      "266 d29038e417e3faf2e364fbae694f1a5f\n",
+      "267 d2958160bdd4a498a8fdbb78f464b7\n",
+      "268 d29b8defd5fc3ddf3048cf72176f19b\n",
+      "269 d296bb80e1db5d5fd3379dc8d92122a\n",
+      "270 d294c804331bd4d1c57f01642f61f7\n",
+      "271 d29461e7a136d19d529d286f5f2fdf3\n",
+      "272 d291d4646e9cddd811b3944076644810\n",
+      "273 d299760693b62a5eae3f1b3b075d75\n",
+      "274 d299bbb989b35ffd8f1ce25446f0d78c\n",
+      "275 d29962a38b71686cf1b772e97325a78a\n",
+      "276 d29f63192e11d1199628d01420131fef\n",
+      "277 d296de83a8761980102db948e468ae6\n",
+      "278 d291ab4d3f535975431e6af3b4640e6\n",
+      "279 d2978ef0cb5ef834b43202e51e33dc1\n",
+      "280 d29258d726278a56dd3919cc188a896\n",
+      "281 d2915e331f6388e5ca27935d52b9148\n",
+      "282 d29c604cc496ec9f12437c94fbac2864\n",
+      "283 d29a46723eb8d9632698ca567bd5568d\n",
+      "284 d29659d98358b82f7f5db4ef5a5e\n",
+      "285 d296207f2b1399f9f1af5dfcda227947\n",
+      "286 d2919ae436a2fb4c57d3b033b335a9ca\n",
+      "287 d2934474c6a34db57e3ff64a1845aff\n",
+      "288 d2992c53edf3df75ba139316ee933b\n",
+      "289 d29c21b8de09a7635e787d148742371\n",
+      "290 d29dac31e0d9ae8a8b1b9dc4463fd891\n",
+      "291 d29f958ccbae9457c11cb31f7d96bb5c\n",
+      "292 d2928d424132804e6fb2732910640\n",
+      "293 d29947c65eb219959c9ca5e701e75ce\n",
+      "294 d29c5d39572d4358239862d22b36ec6\n",
+      "295 d2925f6193f32511b1afd8a1508a6179\n",
+      "296 d29a91e0249311e7a9d58675153b1d33\n",
+      "297 d2997d5a8416e7371663096084e3\n",
+      "298 d29519b713f78f3d8fa21622c7681af2\n",
+      "299 d292f7f7a6f8b994f62993eed8482f39\n",
+      "300 d29b243ee2f2b8b043d2f8f09d611ce1\n",
+      "301 d290d43bb17551605962218450a0f179\n",
+      "302 d29ffa62178f054ec2094ed374db1c1\n",
+      "303 d293de44bad13998ce435eb50b7bf\n",
+      "304 d293e015cf9f815306545116ad229ab\n",
+      "305 d2919a98989c3b9758b76ca2f1c7379\n",
+      "306 d293ecf2a8a2e3e2074c45719737e91\n",
+      "307 d29fa524eb54e62d943e9bfa7c489\n",
+      "308 d29d70e5f39dd79c41bfa01367b3d96e\n",
+      "309 d297a1b5faf9c99b1a862c8cde3a0d5\n",
+      "310 d2954ff7073f7b8889b8fdaf9bf803\n",
+      "311 d2934a55774d6351ca6f0bf85ae6bbd\n",
+      "312 d297d2161bdeda48298bdf679089240\n",
+      "313 d29dfbb7e8746e19c769ed08cf9ab4\n",
+      "314 d29eac9d5137123aa873d7872a266572\n",
+      "315 d2989a38a0351440c9c3ea531f48b782\n",
+      "316 d294c2e6796f8f38524c4f36196f7d6\n",
+      "317 d29e843d1b3e22f97c2121d849eb43\n",
+      "318 d29b8763a0409c173651f57d2cc732\n",
+      "319 d29fe485da174c44309b2d2893666583\n",
+      "320 d29fbf9d5b504081915276afb3ff171\n",
+      "321 d294b740bb69be81a4a05e945afcd26b\n",
+      "322 d299cda76b9a1deafd15c8d029c814ca\n",
+      "323 d29f87767b65523073bbc0d62f4a1137\n",
+      "324 d29a59c28c2a20c9ea582bac4f3d803c\n",
+      "325 d29292389f7dfef416fcc59f3dab3445\n",
+      "326 d29f3eca85aaac96afd5b439733ce3a\n",
+      "327 d2924afd49df91267aacbcd3a55d87e\n",
+      "328 d299228447c3965210c9e2a0287d62bc\n",
+      "329 d299dfd6e5ed848f24adb34a1e66fa81\n",
+      "330 d29248b6cd344ac354ed3573c2ead3\n",
+      "331 d2917a5b3d58c3a1bde2f19029d4db1\n",
+      "332 d29a88af89763c811ad42f71e1adcd\n",
+      "333 d29fad4f12b05bc6316078f393c594fc\n",
+      "334 d293f6cd8ec2298d61c4c51bb43cb71\n",
+      "335 d298e6a75eadc0c941252a9f2a7d53b1\n",
+      "336 d290e285c5b531f06c921a5f20874ec6\n",
+      "337 d29d3cb79958d1e43568f1c63d34e39c\n",
+      "338 d29bbde017cfdde5da2e3fe91dc17e15\n",
+      "339 d2992c66109486fe8e2585e81e547c94\n",
+      "340 d29e5c8da4a9c63a963321b74cbe1b9\n",
+      "341 d290552ac332bf7fe8925333cf3a132\n",
+      "342 d29d596ffc266f1bc5f0637472499d55\n",
+      "343 d29beee88cc15b282a2dda98d6331f5f\n",
+      "344 d29357e935842c476dff143abfc5d1de\n",
+      "345 d2968522cfd5bf7d71331553902af0\n",
+      "346 d29b589144c4cef74c5888dd073b4\n",
+      "347 d29f6794bb3380d5388abc10c1d18b68\n",
+      "348 d294f6f9b9d1f9aa1cb516a095199c25\n",
+      "349 d2977b12141f2a60378f424936328c2\n",
+      "350 d2909ecedecb8fcfb434e1e563ac\n",
+      "351 d293365d7f87b7390888e6df1e8b2b8\n",
+      "352 d296aea77833d92f0a596fe811b28f9\n",
+      "353 d29fc025178de971df2b671296e885a8\n",
+      "354 d29d3e58a9f013c875a9728dfb7bc5b\n",
+      "355 d29033a3c4e8957464ad3290a213e8\n",
+      "356 d29c12a61ed81d6ec44664df4dfcf\n",
+      "357 d29fa842cadb65b73211e71a91f160\n",
+      "358 d29f5d646fc70ded5b8147931cc42bc\n",
+      "359 d2929e6a9fb9d2b0dd2d6ed6212e1c44\n",
+      "360 d29719b14e693377102b3c4153208158\n",
+      "361 d296f8b2a05c5c840fd3bf274d24de\n",
+      "362 d29d8ddb8d04a849453aebafb197ff6\n",
+      "363 d29923d45ae9d1ce1b2f013d93cf076\n",
+      "364 d29d77467587445028d29d9bb078739\n",
+      "365 d293ad8814801a96ef38962c19c2fe2\n",
+      "366 d299588fa03492cd5dc1f4699f66c6f3\n",
+      "367 d291c3e7637a0827245c44ad9afd09a\n",
+      "368 d29599987170f4af26232b57acd2\n",
+      "369 d299c7086bd48d25ff6ed1bdbfd2fc6\n",
+      "370 d2926f81cde09deba0c1c6267496ead3\n",
+      "371 d296ae4789c8813ae39495c3d6c574\n",
+      "372 d29f75b10f3ffbafced8c67fa76fb55\n",
+      "373 d293cec43d8db67d452f15c8796c3fd\n",
+      "374 d295132ca6905e8ed6ad506833d2061\n",
+      "375 d29cb2d78a6c4714148eb5bbe9b5b64a\n",
+      "376 d29ee1d7ab8a5d5246edc558c6faea\n",
+      "377 d297cc2b8df01184d81cc24b0ccf450\n",
+      "378 d292cb986c671084d12f6036607af6f\n",
+      "379 d29dc92b9e1f40308b373c1feaba64b6\n",
+      "380 d29f68ab824de5c9641813ca85ac691\n",
+      "381 d29c1d9c597176505d43699a47d7ee2e\n",
+      "382 d291acdbd759cb8da5cb32ec36ff7c8\n",
+      "383 d296d76b5e39f47645de56169e8b75a\n",
+      "384 d29cdb95cfa957d17f5c598e9a7d1fd\n",
+      "385 d29afe6fc2ba2256a1d54d1841fd09a\n",
+      "386 d2926e08a211c431631b9e3ad1ff3a\n",
+      "387 d29c978c20ada05bbda59b5b4df01946\n",
+      "388 d2905d2da31be4d896e01cb989b7a33b\n",
+      "389 d298a3fcabed7531fa21f6e31ffa44\n",
+      "390 d2912f36916842608b6f026888558\n",
+      "391 d29979bec95bd937836ad8f513c4343\n",
+      "392 d29a198573f1bdd60deabe1a1d0e669\n",
+      "393 d294a57d4c3b634cba3c9786f83d746\n",
+      "394 d29876388196118296732cc9de5c4c3\n",
+      "395 d2904fcf257ab1d071adb3d0197fbea6\n",
+      "396 d29de6c749995a657f686cc7312a66f0\n",
+      "397 d29c1db1e79d573aaee391d16536b764\n",
+      "398 d299d9295eec3275ac6766c777828a59\n",
+      "399 d2921a88927731d2f586f923e8a83fc\n",
+      "400 d29e39a136dad6ad234debf2f8d4facf\n",
+      "401 d294cb1721f626926b83e263f18b55\n",
+      "402 d2905bc59d4c1b55d3cd1bd2c5abd15c\n",
+      "403 d294d51d4d2df9f7bd7ecc5a7fb8011\n",
+      "404 d290c0f68de49420548fc674cf29b01\n",
+      "405 d299d4a536ccb9f0722cc4fa44a32f13\n",
+      "406 d29a3db0e17adda5d1b81986dbdbaed\n",
+      "407 d29a8087a55cb855bf4019a44bb1dcf8\n",
+      "408 d29fc641d016cafe244f5e1cd96ee2ab\n",
+      "409 d29bcf7bfe6309e8f64a65d1b4f8e51\n",
+      "410 d291bf19c28b57b8b86e43d04f2bcac\n",
+      "411 d292bf6fa98d48cd5549b022328fef64\n",
+      "412 d290644911735ee948b705a85e35c8\n",
+      "413 d29df0f5a31d7cdc1b8d37ff824c9832\n",
+      "414 d29450e886b5b1f825a9ebd50693638\n",
+      "415 d293554f84d67d1d40fa834f2cdc19c5\n",
+      "416 d297f65eb6778396ffa0fbc4a19217f4\n",
+      "417 d291f16b3f979b1d8f5b6e90ac289073\n",
+      "418 d29a9d3159b08ef8c7c0c59e90579758\n",
+      "419 d29af2f570815fc2ca7475d4c994e28\n",
+      "420 d29915b11f3faba2ef25837b5e2f3\n",
+      "421 d292ba3bdbce131b4b8eaba527568c\n",
+      "422 d29d4f93ee7b46b311aa51239528f\n",
+      "423 d291a3a9822042b33f6fcb1c9784923b\n",
+      "424 d2994c646d1a58a51e059e4866794f\n",
+      "425 d29f5ae959310c31a201342a95ad0be\n",
+      "426 d29b56938e0e851fea353dffcf7bdfa\n",
+      "427 d29db91d89a1a8768ad269a57768755\n",
+      "428 d29af5cbce13275d12295feb3c634838\n",
+      "429 d29cf5c5ab5aa85e642e48e6af7fd21d\n",
+      "430 d29fc380ed8a16c5ac9166a2b7a5587\n",
+      "431 d29b43aaea1f53f10fe7a6dbada\n",
+      "432 d29f689aa8dcc25f3e3633426a192a\n",
+      "433 d29053b6bd4b624167afbd4f517b4b95\n",
+      "434 d29145cf1aef12781ba46fd06bf974a\n",
+      "435 d29dc0e926c665e23678acdeea24aaac\n",
+      "436 d29c8e62cbfe4d14a646c9bbc8fcca5\n",
+      "437 d29e8837c56f5267630e5ae169bfd7\n",
+      "438 d29b74122114c49f36ab28ee4b97a2b\n",
+      "439 d2987c6c6453c84fb481f9cf337a3c7\n",
+      "440 d29173c29c7c5fcc92299e4b26f5efa\n",
+      "441 d29a3731ab79ce73a4d593b9fec56cd\n",
+      "442 d29aa2edfcd3bb4fb534027c494881\n",
+      "443 d29d6222f59352c7883c9851cbb8d8d\n",
+      "444 d296ed54c0a533659da6d5c8cdeed961\n",
+      "445 d29f7ae54b96b167c4f3eb5772ba67\n",
+      "446 d2962af850a2dcfc528eb58175e373b\n",
+      "447 d29ce7f2934f6874fcebd63986193911\n",
+      "448 d29a34ff699ecd4f359039902365f2ac\n",
+      "449 d29da8178da009efa3663ef49a5bbbc\n",
+      "450 d2986f88e06afb263353345729bc9fb0\n",
+      "451 d297313017d2278a6b5414f0c32045ed\n",
+      "452 d29492bd2642d4c1cb38cf556ba15141\n",
+      "453 d292040f0d9472364211ba7f61858e\n",
+      "454 d29b7145e356aa5c5636e0183ddb92e\n",
+      "455 d29c45ab915f4d22ea37bee6b340c43a\n",
+      "456 d29854a78878f17d04a4465ef1c1d33\n",
+      "457 d299c9d4b94b8a6aca9a8c59a1a3d92\n",
+      "458 d2994c6132ce56145bb5cb1040668360\n",
+      "459 d2957b49289677c103bfa3c8c426ee2\n",
+      "460 d29f6a26cd131dd82b3b54151431a4f8\n",
+      "461 d2931df6c55137e6aa899bad4e4b02\n",
+      "462 d29955bc44e4a34b495b72a355cb88\n",
+      "463 d29c1dc294983dee299fc1d81caa72ad\n",
+      "464 d29b5b3e656eb0ffc0494e01532cdcb\n",
+      "465 d29fceda623a8e845eec231294b34289\n",
+      "466 d2981ba75e6c7a85fe7dfceaebf659\n",
+      "467 d2987a6bc7bfc7a2c6aff6fb466652e\n",
+      "468 d29ab7c2bc5df4e1eda994590eeb294\n",
+      "469 d2925d7b49ad90233c57b664e6ed6879\n",
+      "470 d29ab251b6ea37c9868bf7724850e0a1\n",
+      "471 d299902666d823548fe9cce52d60bad0\n",
+      "472 d29851d275ac962358b158e0de1b13\n",
+      "473 d2944768b1e5c897c545cbf240ea5367\n",
+      "474 d29c87dac6749e7ecfac0860edab4b\n",
+      "475 d29bf455ae5baf7df36aea784cad8edf\n",
+      "476 d29d9e3fcba5e57e95b8a091c11a4e18\n",
+      "477 d29de84d67cb66a9b4f68a4e36244d\n",
+      "478 d297f29c3c57ba46c5e4759d7e275b92\n",
+      "479 d299864d4459c82c9d417123fb7ed0e0\n",
+      "480 d2956a07cc7b735cb2fc39940e079c2\n",
+      "481 d29e8a8d225666058d1519cd1d9bcb\n",
+      "482 d29ea1a95113e741e995195bef381eb\n",
+      "483 d2924d93a760bbb9fdaa3580d6fb40\n",
+      "484 d29a83c0ff462b1bee23b4914e28470\n",
+      "485 d2933dd155f25bacd950a3e77785ae53\n",
+      "486 d293a48ce261f88a756931de8fe812a2\n",
+      "487 d2923e4fb7105a50964af5f793bb312\n",
+      "488 d29ba0eb4b1aed94e2f514fa22d3144c\n",
+      "489 d29bb961e35c744ceaa8d934f56c38\n",
+      "490 d2996b3456a45a3d41238d9776d2e8d1\n",
+      "491 d290558fbcd5a1c42c88e6a688391d5\n",
+      "492 d29ca6fc962ba76eb2f6dd94dfa1b51\n",
+      "493 d292e6dae1547d0c1771389fe9454d5\n",
+      "494 d29fa8e372d613b21f876bc4e861164\n",
+      "495 d2954d2b6a9d4e255f22dd209678\n",
+      "496 d29cb4599b081c0fb829e394c3e3b67\n",
+      "497 d29b8a9a436ed6e2b57f994238cfc3f\n",
+      "498 d29dd04bba37b924be68b994d1ee2e4b\n",
+      "499 d293db1c82edad55c8a8299e6b438793\n",
+      "500 d29886f24fe094ddb7e46aaf5e0dab6\n",
+      "501 d29dc2bde2c4827c70ec2be14799bc\n",
+      "502 d29e5578df4d7f59092a02be4bce080\n",
+      "503 d29d7a2f8b3f1bba983d6f2966f9e7d\n",
+      "504 d29e81db43bf6a7161d2ec361371ff\n",
+      "505 d29744b420d887a067757a3545822e\n",
+      "506 d2907f4872a14b616ad6559a6bd68414\n",
+      "507 d2919bfb4baf1ce16c76b5cffebd4177\n",
+      "508 d29dc9ce25a91e984119614aba432934\n",
+      "509 d29a39a665d71a1f95bd630a8f7b798\n",
+      "510 d29e5e4fe2bb33d13aca84411dfe\n",
+      "511 d2923a1af99c914ca2332a25cfc219a\n",
+      "512 d293a3fc5bebde784f86994dcaaafcc\n",
+      "513 d2937777a67145a58baf14705ad36ba7\n",
+      "514 d296453ce16df5aab685a113b28d1a0\n",
+      "515 d29e9754b3c64e1a4403898d9c91893\n",
+      "516 d292a6e2c0d1928ddb1e773cbfaa4b6\n",
+      "517 d2994c52c4139a85acb2643be15ae92d\n",
+      "518 d29e843f3a842ff08412ad598a6c8f72\n",
+      "519 d29b8be313f882b734a4c15de6578\n",
+      "520 d297cc9a5828aaa149722a0a99bd31a\n",
+      "521 d29f53d4d383ec62ede38be2624ca47f\n",
+      "522 d291a8d61746094db6cfb5a37a5e811\n",
+      "523 d293f937fe7825ef9266c34b0ab47e\n",
+      "524 d29bc0c163c4dbf2c82b3648cb9ddd\n",
+      "525 d29a2340f91e4bb4c3982245f5abe91\n",
+      "526 d29e72d22e7849766784a5c1d57e1898\n",
+      "527 d299ae8d8e8e5b1479981ba7e3d620d8\n",
+      "528 d29fd3352226e4cc839cfb19ff05a61\n",
+      "529 d29e74ce55d334f67db6ec34837b33f4\n",
+      "530 d29c3d44fb84d8f711b37b742d957b\n",
+      "531 d294236f1cd30b04c665afbfefa1a3\n",
+      "532 d29ac034f114febf34847edf9d36f5\n",
+      "533 d29eb139aeaaf7fa3d1bca4e44ab477a\n",
+      "534 d295e75c7ac697329dd61a4ed47b64a5\n",
+      "535 d29f1334244f1b3db941e3e410412d18\n",
+      "536 d29e897cb34d9749afa9b714d36fef9e\n",
+      "537 d29e5aad6acd594189b6cec37ca5b4\n",
+      "538 d29925c2742e847a96765883d5df7f6\n",
+      "539 d294517196a46fc47b02564e3c521a\n",
+      "540 d2924adc342d2534f5eb7c53c110d482\n",
+      "541 d292ce95c7f45a3c3493b8e4ba4268f\n",
+      "542 d29f5120673264b9e6351343569c8643\n",
+      "543 d29da888145eab653f71aaaa3e3b721\n",
+      "544 d2915e2dc9768d7c9e293ba8ff78ccef\n",
+      "545 d293e1e962479acb87de21b7b4de047\n",
+      "546 d2959ed48cb926ba9835f41778c29d6e\n",
+      "547 d295281dbf716fd08f070a0629d5da\n",
+      "548 d290a6b81821ff7f9ad19afa2fd84ac\n",
+      "549 d298531ae57af88b8433a7f282ea2b\n",
+      "550 d297f18142c314a35af6c63b14b9f9e\n",
+      "551 d2962446fd5abe52567fe06a1b38cce7\n",
+      "552 d29a0cac08f5ed2323feda06c8bea28\n",
+      "553 d29e9e1e2949c704454b2b6611d6d8c\n",
+      "554 d29cd042363ce647ef1b76b1b391fc\n",
+      "555 d295873185d9a9c1792b8b59d4bae4\n",
+      "556 d29a4bb262d1b0729424e7f1bc7b44b\n",
+      "557 d2908589c031ebf2abcd76a75fc11a0\n",
+      "558 d2949f30dbbae0207b4a391fe658bb\n",
+      "559 d297ebbd26c1ffc19841283c3da33b5\n",
+      "560 d29e913733a376e1d68dc1d98937659\n",
+      "561 d29d0cdfd7d93dd29577df869a06dd7\n",
+      "562 d29af95d81eed483fd7fe25ef7f68a2\n",
+      "563 d294afd91d3f92eeac312f73cc12395a\n",
+      "564 d2952a96e1218338870efc098f19e\n",
+      "565 d29c8e6ecad3413f7cb21c3f1f26e0\n",
+      "566 d29a77eec1c5179561cb35204f6bae7e\n",
+      "567 d2904a3dc19b4e8ab52f24f0f49ed576\n",
+      "568 d292ea27ddc8522669b2aa0ab2a401a\n",
+      "569 d29ed6ae6be3816d379b98efc8fb8b3b\n",
+      "570 d29b13aaa457d8bacd6806919dfc1e\n",
+      "571 d290a2953354a07cc6711c284edd2a88\n",
+      "572 d295f3be27879c4aab716ef8d20156b\n",
+      "573 d29748307bcfec96dd14fe6ba36abc56\n",
+      "574 d295b1b47195707cb92faa7d952627\n",
+      "575 d29b1acbe5886cdaccb0bafccb27e024\n",
+      "576 d29b44b8afcaefa5769cd3bfdb96997\n",
+      "577 d293c0c7285c877e805d50c2196d3c78\n",
+      "578 d2997dee4dfcc7bc62bfe5db828c5\n",
+      "579 d29399cc8460364389d2cd1a6392e54\n",
+      "580 d2944cf6f785198d86773c4b44cc5c0\n",
+      "581 d29285ab48f5a6ecd9c6a45b229b335\n",
+      "582 d29d8536b0504ab1ac1eed17f1ccaebe\n",
+      "583 d2988eddef757af7d9814858e4d980f6\n",
+      "584 d296cda8b065f1315df7d284722773f\n",
+      "585 d29577d5147cbbca8a97cf13fe1ae27c\n",
+      "586 d2927f9825ab537f697784afbdc9bf9\n",
+      "587 d297b94ebc79c5d68bd1bf46999cc8\n",
+      "588 d2981b64d66b3ddae5b9d547df7f4bc4\n",
+      "589 d29b4e8491a1454ac5b4effde5f29\n",
+      "590 d294da8d22ec4c14683d3fbcb346298f\n",
+      "591 d299e568ccd4bfe473214f116f16cd51\n",
+      "592 d29d9be9054b13d6d9391419c09b1d\n",
+      "593 d29067138de3e6fc96ac8837bc4d217\n",
+      "594 d29d8eced5755ee1574b85cb0dbc390\n",
+      "595 d294918dea66792e53ef27496fb42a\n",
+      "596 d290c72ed624368549d9da1b296f127\n",
+      "597 d29fdc2a87e9197967743cf0afc02b2d\n",
+      "598 d29f5e962f897871323880579646c1c\n",
+      "599 d29cd4a5555bde50a6c9cc3d6eaa93a\n",
+      "600 d29da0cb710d6764f708cb3a16bd058\n",
+      "601 d29938dd4780fef53cb4502944c2ae1\n",
+      "602 d29dd6fd1e45279df9f83a8d16b481a\n",
+      "603 d298a69ab72fc0784b72e43e42acd\n",
+      "604 d29dfb75856f9761de28eec281a4412a\n",
+      "605 d29f50837e6202bb0ff8db95626eaa3\n",
+      "606 d2911cb3cee24be5abb2c7ebd92ea9b\n",
+      "607 d29a9927caceae6c14dc6d7ec3eed2a\n",
+      "608 d2937bef46c0e329fc3465dbf7d4b3e3\n",
+      "609 d29af7e273912a5b4217277c54b8e26\n",
+      "610 d2916e2d11436160344632db0dee96\n",
+      "611 d29fce919dc47333a5c20f84960403f\n",
+      "612 d295dc6f26f8e6379e2de3584d8a5fa5\n",
+      "613 d2941541ad8d453ff7ffd6301e4d2818\n",
+      "614 d29d6bb724e9eaf716f34c5c9f3690\n",
+      "615 d2944b67e4a43a6362214585122ba\n",
+      "616 d292493e4de6937f7515a7747e1ecd3d\n",
+      "617 d29b656c1314f13cd9761279f1afcaa\n",
+      "618 d29036667479f953455c613c75b274a9\n",
+      "619 d29181e24020b418ea3ea3b8fc44f4d6\n",
+      "620 d29fa49f6edefd91b8e854cc89b1d284\n",
+      "621 d291e0cb8aa28d3b11a03e513da6d3\n",
+      "622 d2992973f5ad4d39a1f19328663af85\n",
+      "623 d297297a667b971328636252dd708\n",
+      "624 d2998240a3682259f1f6ed2b36824d58\n",
+      "625 d29b375f07fdd1976e6f41bc7d86559\n",
+      "626 d29558b35bdcc2453d7a3067adcac60\n",
+      "627 d2946353e753729d4a5127ed6b3b898\n",
+      "628 d298165c933ec382a997c6c9665824\n",
+      "629 d29c273bea347330addd7eb689651f8b\n",
+      "630 d295a0f3693929e2c8ba2362f9b0acca\n",
+      "631 d2998924cf5a2c1cfdfee129d4f5a46a\n",
+      "632 d299d317d99a45faaff48b2772b0f6f6\n",
+      "633 d29858f839fa221075a9bb494374bcc\n",
+      "634 d29cd7edc74f467aed9f6321528ebdf\n",
+      "635 d2992f3279c543fff277528149c59f3\n",
+      "636 d29694d25d472eea115117b871659\n",
+      "637 d29b51fa7b0b19c8557e94a6687e8a1\n",
+      "638 d29015f0e054a3efbb9be424e58dbf4\n",
+      "639 d296ed355cbf57ab8b69f99c70f0f3\n",
+      "640 d291f57f60fb372b2ff12a8ce8704569\n",
+      "641 d2932924f67936b51bb26840a99c6539\n",
+      "642 d29899f48469ec2054cdd9a1a2918468\n",
+      "643 d2941558d8eb2cd1dd944137bdfb8b29\n",
+      "644 d295582dbe4e6a7243d6bc3a2833488d\n",
+      "645 d294bd593ed959668e48c92881f73ffc\n",
+      "646 d2956ff88a4af4ec6bb1eb08b8df32a\n",
+      "647 d291ed4d52f2596f310212beac7bbd\n",
+      "648 d294bff890fdb108792581b118c82\n",
+      "649 d29e7e09384d5b8ea2e80d12ef553\n",
+      "650 d297bf2df519cb82918959c3ccc0cb55\n",
+      "651 d29737c8bac680c1a4b820f6e21457\n",
+      "652 d29d705d34814746b26010a9cdaafb4e\n",
+      "653 d293e464cbd68fcb8343eca51fbf8194\n",
+      "654 d29996eb36de8886d68dba475d21c3a3\n",
+      "655 d298cc71c18291df579944ee3cd93ac\n",
+      "656 d29d81c9f42bfb5f5dd3321c5e9fb671\n",
+      "657 d292e52e76e8ba45356e47255cdfe350\n",
+      "658 d29b5f55f4116aa6de6f713c9cc2582\n",
+      "659 d29ed238e14a28b3f41e33b454abb15f\n",
+      "660 d2906b3162ee858fc6add2cb5f4b276\n",
+      "661 d29830b5f42099f12c90f830a2ade11b\n",
+      "662 d29215bf492daaaf2d54aa8022b6165\n",
+      "663 d290668a4ca023fde6853e8b48ec9b4f\n",
+      "664 d29ab760cfaf4b8232dd94e8ffe71b25\n",
+      "665 d2912760a642287432653899a490a229\n",
+      "666 d29d911a3e0b877c95a2962b7377a79\n",
+      "667 d291a7e575ad7f83deade677c4f27246\n",
+      "668 d29c7f972b1716113edc6a68c64e79\n",
+      "669 d29fc373b6c588d3c81fe753f88257ac\n",
+      "670 d29769564e8289f43c8ff769359ea9c6\n",
+      "671 d29d92153f44fdba926224ac7b8bec3\n",
+      "672 d297b8d58478a65c5ca79df8ad88b\n",
+      "673 d29351fb392d71e61da1ffac96a29ae\n",
+      "674 d29194b067e9dab16f189287ffc779ad\n",
+      "675 d299f4c2f6fa7ef665b935f6d3bbd3\n",
+      "676 d29af155d2d7c947b5e77397b64a544\n",
+      "677 d29b42d6ece2ecac3d784a915b7485ac\n",
+      "678 d2982667801c9f5171c6c1f04a797498\n",
+      "679 d298e8bff0871d2249bf487971c377ca\n",
+      "680 d2979222961dafe3cb68dbf3f0518b\n",
+      "681 d29a6752837d58368e53925569908f1c\n",
+      "682 d299d0b46a9eb5c41c4469e189a7eed\n",
+      "683 d297b34212b5b49e1a1b28a8b3866773\n",
+      "684 d293f2acede4ebe470106a623b6a515c\n",
+      "685 d29a44a041d991b77794fb01167395\n",
+      "686 d2972c64bf3d7131b17912d56db8\n",
+      "687 d29e50c918f8b4b4c8bcf34364afbdd8\n",
+      "688 d29ea7580511e498046f389bce9d53\n",
+      "689 d29025f886bcdc88e3cf3795c96262\n",
+      "690 d29ffc11409938bbb8eb13bde312d777\n",
+      "691 d29ee0be2178e838fa8ea3861d9b7172\n",
+      "692 d2904623523d8119d2b1a28364cedfc9\n",
+      "693 d296e64238df7f7f47a2d664d34ed85\n",
+      "694 d29b441881275c44f3c2c62e8e78c\n",
+      "695 d29cdbb47a6b9db377c38f6666cf526\n",
+      "696 d294e9faa752812dfbfd612fe6e8f3\n",
+      "697 d29637c1fb7be3ba592c873ccfb3fac\n",
+      "698 d296164d9647a69fc2a22855ea2207b\n",
+      "699 d292bbd49cfbc285ae38abc7f581f37e\n",
+      "700 d29b8659d7fcc6bfa6ab48a7eee32e96\n",
+      "701 d297388166d5f101c5c3496d223c48b\n",
+      "702 d291ceb7ef4442ca3ff9f6437c2460\n",
+      "703 d29accbd24c24f7621e05afcb067e\n",
+      "704 d29da88fc29a3bca4dcfdc190f9dff\n",
+      "705 d2987a60afd375f03612b54f3b9cd93\n",
+      "706 d29517c88f943233dfbb38b5bfe7245\n",
+      "707 d296e19f3d1bc89a4f5aa0e55bf190ae\n",
+      "708 d29a651edf99216156e1b366ad3f9469\n",
+      "709 d297de9d0b1b1a58fb375c5ce4a1f6\n",
+      "710 d29191dc4ad5586eb86162e8c6ffa8a6\n",
+      "711 d2926d26636aa631e6ca74ff32ec99a\n",
+      "712 d29a13c54979d5667bd0a06c231587cb\n",
+      "713 d294ad142f70de4eee6ef9aabd051f7\n",
+      "714 d29e758920ad853db60a175dbf16\n",
+      "715 d2941b4a896ac7205845efc1e3d311f4\n",
+      "716 d2979692498dda6be501a2c19abbec1\n",
+      "717 d29035fa19ce8d2db8209c6dd9cf18c\n",
+      "718 d297902857ecceb79df42eb2b8286830\n",
+      "719 d29ebd7d6556655ba798a563c3c81b4\n",
+      "720 d2996ccbc7aef032ddc9c40c9f4e2\n",
+      "721 d29f3bf78d3e4e659e8c65b2a06063ef\n",
+      "722 d2973089b678f4f783f7687fdfd3c413\n",
+      "723 d29e239ea96cdd0f59bc7e3d299a723\n",
+      "724 d29e50b79ce062e5e8bc6c1b2622ed1d\n",
+      "725 d297a8f1a68efcf7f6a3747f9b99bc16\n",
+      "726 d29463a2ef7ee7ab996621fff62cce4\n",
+      "727 d29977ed567f23fe5d5b13f5186ea89\n",
+      "728 d2985efb90f03916d7d36c121cec28\n",
+      "729 d29842dcb0b1fccd70c834c2e92e5a1a\n",
+      "730 d290aa2a5091fac7d791ed22707cdcac\n",
+      "731 d29e107f6963ad47284ed6a47366a3f\n",
+      "732 d29711f5d625e953559d1a6b33462bd4\n",
+      "733 d2955ca3d3d6f6cefdf20b350ebb9c7\n",
+      "734 d29c3a4c14fc71c7d050fd293c2737\n",
+      "735 d2955b7c292299d843e3a2bd7a631816\n",
+      "736 d29b3807fcbecdbc37d7a11421a612e\n",
+      "737 d29cec30dfa4a2bb22b2785522d8bac\n",
+      "738 d29afc07a96bbdc641abdbc71862f7\n",
+      "739 d298116eb1cdf4135a5761a4774c5e59\n",
+      "740 d2905e5f8949982586204e633e322fe\n",
+      "741 d299fe3aae57bbc76f66867395063\n",
+      "742 d29d987f35742fa36bc11ea02ef6c395\n",
+      "743 d296257f78985bee2f732d4bdcd9787\n",
+      "744 d29aa8dcf7d7bbb373adf71c8f05418\n",
+      "745 d29e4124dacad46ca69bb3952f1654\n",
+      "746 d299bfd25f0b38f9e9b1cb076999968\n",
+      "747 d29cb18dc082b9abf777775c244bcb\n",
+      "748 d295ea8e422a5599c2e7865ed482eb5\n",
+      "749 d293878ad47054666535fbc2bb4783d\n",
+      "750 d29c78cf63ccaa475b9be293c50dd48\n",
+      "751 d292bdf79b415e4fdc426be3a29eb62\n",
+      "752 d293c4bad6e52c345a5c6a37168672\n",
+      "753 d297224294f55bc425603b6e261e88c8\n",
+      "754 d2924655f5f018d7f2a153691444f1bc\n",
+      "755 d292eea19488127d5c4247ff3832c7\n",
+      "756 d29ee24ef7d324f6c94df7ac4f534b3\n",
+      "757 d29ac7c01ebbb7d56b12929253a2e711\n",
+      "758 d2925f53be7ac95079be431cfb3393e\n",
+      "759 d29ad94d7541ad3f4569386c1311570\n",
+      "760 d2982ce8d6fd3d7891d7a5244d6e46\n",
+      "761 d298dc1f3a5423b0c848d648f3578de\n",
+      "762 d29866f8452bc5e5a93495d309824f6\n",
+      "763 d29237ee31a09c1143df8ad7e5edd7e\n",
+      "764 d29f1036bfa749f2844986e4ec2b08f\n",
+      "765 d290c145d34ccf80b624421a3a9862c\n",
+      "766 d290c5efc4398d2f99b8806260b3bd5\n",
+      "767 d29bce9d439282122cacfd9a3994ee6\n",
+      "768 d29487ffb1f785b1dec81bcb1644a5\n",
+      "769 d294c93212c56d4c638e1015c7a4cd\n",
+      "770 d29ca0b321bf709c9314bf546435a014\n",
+      "771 d2944be7a48692e91c12e44924e8c378\n",
+      "772 d29599c0222fe18127894979b2d9e195\n",
+      "773 d291b5269b7e2be302bfc2e35cd9493\n",
+      "774 d29ad7050d32f987bd1ec29956537e\n",
+      "775 d29e847361db8a6d95dd31e02656adf3\n",
+      "776 d2938c8eb48754392aa2979628517cf8\n",
+      "777 d2985b3835dd12d38b4ee2fe44f5d51\n",
+      "778 d292bc4f15b8a333420c036e552dbef\n",
+      "779 d29fd3b52d428dfcbabc1e5dce0b218\n",
+      "780 d2925265e9c682772c6fcc801fd567ed\n",
+      "781 d29385b2d29b13dc8531bc0e3ae3c3\n",
+      "782 d29862aaf54f52438142c03ec5fd9\n",
+      "783 d29517565aef432bca2c56111eb6bb8\n",
+      "784 d295e1fea5f25c2ae638080cd4fd986\n",
+      "785 d29d7de1daa871be5dcea8b895860c9\n",
+      "786 d29b2dfc8c1aa0b5d35e12298ece91db\n",
+      "787 d29dd7cf86216b8880f4bcac6c5ac89e\n",
+      "788 d296ea2bba6573f1a42eb1f4b861ee\n",
+      "789 d29db474e03b44f26b1918293eb880ab\n",
+      "790 d29c5bba5ada4ff67ea059ec58bec4ff\n",
+      "791 d29f8eded8bd5e5b39384682b2de5\n",
+      "792 d2983d5eb4a476b5d821989ac7f7c99\n",
+      "793 d29cee44da3e77889c2fab8a3145aab9\n",
+      "794 d29ba9669bc9321fecefc1d11f498d2\n",
+      "795 d29825d027a364754199952ca1334046\n",
+      "796 d29aad8d9afcaf7e6ed0fae648833447\n",
+      "797 d2975510d6eefcc0eda919f86f148b1c\n",
+      "798 d291a0ce1a15ea52f6eeaed1edf58dce\n",
+      "799 d2904816b05dc67c8537bb6582de982f\n",
+      "800 d29fab6e11338ab587f16136cf9444\n",
+      "801 d296acd93e6038c342a36b18a7909fd0\n",
+      "802 d2982cd07bbc8e64c6e135232b8b6925\n",
+      "803 d299dd6b21d0c6f1cbd19185d579dc\n",
+      "804 d29544126d3e23f648bd112ef6a61ce\n",
+      "805 d29826cc8d2dd83b92e86337d0c031c\n",
+      "806 d29dd510e5e7614b5ca4379d6ed707\n",
+      "807 d29ad167c0ce4337b9589eafe03494af\n",
+      "808 d29960b34deb26acde68498c9f6f9cf\n",
+      "809 d299c3dd50eff7982fcfb87421d424f\n",
+      "810 d292f74f748e4d1973c5b9515e96f\n",
+      "811 d29aab4e896a43874eb098ff95b7d359\n",
+      "812 d29750922a9834d5a9dd10bb748832b\n",
+      "813 d29fa86517414edfbdf9a96e18ac67\n",
+      "814 d29aa42bf76a9f2cd1c51157b2368a2\n",
+      "815 d29f267452f46384c1eee391630f\n",
+      "816 d29ff196e725e8639fbc3a09899e8\n",
+      "817 d296c63571356a27fbdf0afd31afb\n",
+      "818 d29be82641878d0b8f02762222f8256\n",
+      "819 d29f3c32b0e180c8636dbe96571c47ca\n",
+      "820 d29c3829d6808ff566be34ff1f3f3a21\n",
+      "821 d2966dfabc587217328be7627c64a4a\n",
+      "822 d299e265e3e3e6535a294dc4adeddf\n",
+      "823 d29e7ba26ed08b7b5499d2853bbe1440\n",
+      "824 d2943428447dafe9c198f43443dec1\n",
+      "825 d29aea1d82818daffabb7f916ca6e80\n",
+      "826 d294b396de33e8ec9e19064faffa6c7\n",
+      "827 d292357ac7c73ffe69394836cbe6c81\n",
+      "828 d2946d777ace8d523759eed85c97d\n",
+      "829 d2993a76ad5a82137a158a3cd69a3498\n",
+      "830 d29ced67ec73d4466990b2a0c39dc952\n",
+      "831 d298d6a3641f7a81d7e810f3178c3ffb\n",
+      "832 d29c3acabee676411107c46199976a\n",
+      "833 d292f399f7e6c1748a6697fbdb8a3b0\n",
+      "834 d299cd99a36be18b25f23d3578a819cb\n",
+      "835 d293983a451ca4bb7ded3ec6ee2e934\n",
+      "836 d29a176ba9b216e746d4fcd147dd01e\n",
+      "837 d29c48fc5e615a222bf8f16cc683d217\n",
+      "838 d29f3f15a8513c1281e157c6e990d5bc\n",
+      "839 d299e63ed9cbd0dd4279a0c1b8406f2\n",
+      "840 d2913834d6c820d0cae85aab464764d\n",
+      "841 d29528f1fba2a9365522f0528dd69c4\n",
+      "842 d296d5361b93faa231da4da432b8720\n",
+      "843 d29ebee84c287cd5ed27dfbe37df79a\n",
+      "844 d29cd3f3444df8673be61bd7499332c\n",
+      "845 d290855d546a7a55da3548fe7deb5c2\n",
+      "846 d29425170a41b4b75177787b5c65ab\n",
+      "847 d294fb4a65a5d037676413a1d95cee12\n",
+      "848 d297cd83a3c4d24264d8fad18c4ab9\n",
+      "849 d29c1ee453c62685c16851bb257f1b7\n",
+      "850 d29da361eded18e5845516efa1c146\n",
+      "851 d29316f1b419cca78b8eb21c91d094f2\n",
+      "852 d2907ecfc6276dca050fc71a479048\n",
+      "853 d29e9a4d1b55913b632a057c282add\n",
+      "854 d29d5849151bfa133bacad8ab196d7\n",
+      "855 d29f52a596a66fd22c99f7292fcbd4\n",
+      "856 d2995314458dc296fa8a50d578907f63\n",
+      "857 d299729169810aee8cf863b98885472\n",
+      "858 d29098775cb7d920da1dd2d5d444df1e\n",
+      "859 d29dc3f4bf446d5bff72ccb27c631\n",
+      "860 d29ddc6413cd9fdaa9b97bfb38697166\n",
+      "861 d29a189d80fb1b8d64b2b92d6e25576\n",
+      "862 d297d4df9a1ec83dbf159293653ff71\n",
+      "863 d290b698b6489f39e8f2ad3ce4261594\n",
+      "864 d29fa9ad34a40779b3736a71e3e8956\n",
+      "865 d295d0175728998b8560507b74416d81\n",
+      "866 d295b028a2ba30598e75b46cc28feafb\n",
+      "867 d29a33b2188451fca68da55e201b57d3\n",
+      "868 d29273375d45edaa581a111ce22e5482\n",
+      "869 d29307629e27a419c4c9459a918ff6f\n",
+      "870 d296de62b4fa21342a695aebf897b4f\n",
+      "871 d29691391d561ed43b68c891792e946\n",
+      "872 d29e81413ac99fb57626e48278e783\n",
+      "873 d2925c82aa7fd47f6ea9e20f5715719\n",
+      "874 d29f6adb7938f5a11ffcd43fa17be269\n",
+      "875 d2956a499e1e38b4538a34c88af8438\n",
+      "876 d2941bd03c33d2bb4921405e52677319\n",
+      "877 d294252778388bab43954be9d64bf111\n",
+      "878 d29fd97555f88c8f56515d5ef6aa35a\n",
+      "879 d2966cf6fc7be1abf528520b894d077\n",
+      "880 d29ee684b54476792bddd1dcf5455a5d\n",
+      "881 d29be915e4b91c211a954dea9af625\n",
+      "882 d297fd4966bb91d5809164c3eed20f3\n",
+      "883 d299c65b6d9e92f0e34f3ca5452f2142\n",
+      "884 d297bce473a261d5a8fdc04b62a9b9a\n",
+      "885 d29ebb6fb2285a61486c87fff8d38e\n",
+      "886 d29f118564ee222d29141c5670a3f3cc\n",
+      "887 d29152bcecd9b6abd8cc15c33d28948\n",
+      "888 d29b5b2817bf309dcf124afcc41096\n",
+      "889 d298ebd5120ab3cf7a4184a43c5cf\n",
+      "890 d29c328293364cd58d99c33f46a7c1\n",
+      "891 d297c62a9eb99ba18b46df6c43ef0e5\n",
+      "892 d296d4312ccba6f29387ac3a67a0ce\n",
+      "893 d2972baeb06f9d30176a6a2b8f5c535\n",
+      "894 d29cb9ffbaa7fca2aa38c5aa0050a8\n",
+      "895 d2941e7858af7221c16d1bfe1792b3a\n",
+      "896 d29ed990f5cbb0eaeb1cdd1645e216cb\n",
+      "897 d29f8616ffa1c0e16437ecda5833578e\n",
+      "898 d29f264016fb567fc523f16c99dd63d\n",
+      "899 d297f883cdb8fba51734bfcb78ac\n",
+      "900 d295f5e269934ac4bf355afa329b0ac\n",
+      "901 d2961171e684b9d624ecf5439932a537\n",
+      "902 d291c634c899a6c3fe5f28c54680f4ac\n",
+      "903 d29cd457bd71c490bb53521b1730eb2c\n",
+      "904 d292daf9834286ad9850fc67f4c3d69\n",
+      "905 d29effc9b2125ce69c928f41dffefb3\n",
+      "906 d298d24472564871585443c2ba9f6625\n",
+      "907 d29d42f9314878c6bb302c1a73d6f1\n",
+      "908 d296bcc710b43b719bcdd93e2cdaf29\n",
+      "909 d29825d81e57c0c317ae93b5dbe78\n",
+      "910 d29b39adc7a9e4ef33cbd8a6ef32879\n",
+      "911 d298f52822e8a1b966f359eef53869ed\n",
+      "912 d2909f79c3d51e9b8b41366d851791\n",
+      "913 d29f4b4ed084c42652b62b0b6182269\n",
+      "914 d2952bad222361a6263b53cef5c08fd7\n",
+      "915 d29e95a3a24e6f548dc5bc66e9534ef9\n",
+      "916 d2983e18a5eb44da8b9cd2955d2598\n",
+      "917 d292380b6f7a4791e6829215b4df483\n",
+      "918 d2966824b928a21da99f327dcc25b2c\n",
+      "919 d29690a89ccff87641923adb266ace\n",
+      "920 d297d320fc6e1221f2939dead1829f1\n",
+      "921 d297dac5dd61868c16393413e9df419\n",
+      "922 d291f0fce6e2b09f27c637d1def6fda0\n",
+      "923 d2937ba495db231c9f863bdd5e2efc2\n",
+      "924 d2945f89f43315f3fcee9ccc5f14fde\n",
+      "925 d294d41c45ea9fd8cf1df22214f7f65\n",
+      "926 d2923eda9fd98f1fa0fdc85c2c6a8f58\n",
+      "927 d2919a159571de2c8ae87fcee7f72\n",
+      "928 d294bd134345a46391dcec1cd27248fd\n",
+      "929 d299b714d28830663458662681e041e4\n",
+      "930 d2994917ae56468019ace55110693b\n",
+      "931 d29638364c5166dc5ce5040424db5\n",
+      "932 d297a7b0a91ff4e9d999dfad446501d\n",
+      "933 d29883a44e13226a369554c0f826474\n",
+      "934 d291879da81e887f31e11fe0c54b69ed\n",
+      "935 d290fd3d51f8d62324b0338a84278ba8\n",
+      "936 d29465e1fe608a4bdd4b3cba5f985129\n",
+      "937 d293d623b63e47b96e812ac2fe5565f\n",
+      "938 d29fffdf16211b8d5aa41487a8daa5ca\n",
+      "939 d299fc7fb7f458ec1b976a5a52b8b04a\n",
+      "940 d296a995f653a0335e447e0f9f8804c\n",
+      "941 d296f252693c6130da6fbaadc08469\n",
+      "942 d29cc9dcde13c9371a28cc1bf9836e3c\n",
+      "943 d295918d4f51d352b3c83bdf3d16f861\n",
+      "944 d29832ee32acfc4c7b56c4d1eed42\n",
+      "945 d296ef3360d4f5ddfbd530d479d2992c\n",
+      "946 d2965113b74b1a9ec3cbc33602811e9\n",
+      "947 d2956451b5c77299969f87aea3621e3\n",
+      "948 d29ab427ff507dbbe13ae25ebbbace6f\n",
+      "949 d29a5ba29763bc916b853c15293689f\n",
+      "950 d2927f7a6056ab6be96cd0812640ce\n",
+      "951 d29ac16ee01e78164acdd4e9ae56b65c\n",
+      "952 d298f1ab24787baabadc2c79489857b\n",
+      "953 d2934db68cdb24285a4bfe4c45de83\n",
+      "954 d296a2c4fd479d35942e20779121cd2b\n",
+      "955 d292aedad670eb23c0de67d754c9f\n",
+      "956 d292f67c97843c616fe91b24b833e81\n",
+      "957 d294b46b302a24644766c7449594721f\n",
+      "958 d292961146b9cbbb547223db2a8a9\n",
+      "959 d296a012631260f8f4d62a553b79b2d9\n",
+      "960 d296fe9aac4d48e7bf61db9aac5bcb8\n",
+      "961 d29c64939a3116d25d2baea9fa5ca2\n",
+      "962 d2921bc19d4534ab7fa7a85bf67e1faa\n",
+      "963 d29e44e97f49146198417e4ab07cf7e6\n",
+      "964 d29e7f55fdb62ca7f29191e6f3551ebb\n",
+      "965 d294807c2d6877a01b863757ccbf\n",
+      "966 d29399f926878adeae85b9126c9c545\n",
+      "967 d295684772ee4705d79a7ecfa44572\n",
+      "968 d299e639d6e22972f6789e1f7613dee2\n",
+      "969 d2955e19f597df6c42b37859b59b4a\n",
+      "970 d295648026dce77c96bb4f94cb1b6ae\n",
+      "971 d296b192e72f956789e68dd798faecd\n",
+      "972 d2927984b7b4badce29cbef261244\n",
+      "973 d2981e54d04b40b869399c3ae30dea3\n",
+      "974 d29ace284cb77abebfe84a87eace985\n",
+      "975 d29f28f637ff8952889657bebddfed5\n",
+      "976 d292e945bbd333b72c4951321587958d\n",
+      "977 d29b28c6e5e48c4d898cb786c3ddc\n",
+      "978 d2919df6a0b0c198a55db2b82c9e8a\n",
+      "979 d29d73b4807db874afb1951d5c6fe58\n",
+      "980 d2998145f1a42e419e9c669f3ce36f5\n",
+      "981 d2967bcc651b29e9e7bd65fab12d5a3\n",
+      "982 d291736293c558225a0cebe457a6f2\n",
+      "983 d29e9483c1c73bda7d7d74e869b4e7e\n",
+      "984 d299d5f6b506c6236dc858da34f1cc\n",
+      "985 d2913ad1734310694a6c2c35a1c569e8\n",
+      "986 d294bdca75f6d53d497559412a7a3d\n",
+      "987 d29aecc65b7df1f508c83df595ff4e\n",
+      "988 d29cda9cb047b6bdbcd4d3b50feec7e\n",
+      "989 d29739396b17f9e255c7726de428c5f\n",
+      "990 d29b475454526ecffec9fefcf8f01c8e\n",
+      "991 d29667e51ed875183825ab53d44fa70\n",
+      "992 d297e8ed757593d67a2771257a27be4\n",
+      "993 d295c322fc9ee4dca758544c942f2d53\n",
+      "994 d298372c48d5c8aaa16ee2f3a5a5380\n",
+      "995 d2946559a807388662cd0308ad666dd\n",
+      "996 d29dcc2038b89c365b3aba17f94bf52\n",
+      "997 d29fcaee2537fda115ad172ed10778\n",
+      "998 d29ca7d044203e0242084cb958ef464\n",
+      "999 d299349d8bd55ccae1dcea12b2b7ca73\n",
+      "returning 0\n"
+     ]
+    },
+    {
+     "ename": "IndexError",
+     "evalue": "index out of bounds",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
+      "\u001b[0;32m/tmp/ipykernel_617634/3764770081.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m filtered_dataset = dataset.map(\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mselect_existing_rows\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mremove_columns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumn_names\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mbatched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mnum_proc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/code/hf_jax/datasets/src/datasets/arrow_dataset.py\u001b[0m in \u001b[0;36mmap\u001b[0;34m(self, function, with_indices, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)\u001b[0m\n\u001b[1;32m   1655\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1656\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnum_proc\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mnum_proc\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1657\u001b[0;31m             return self._map_single(\n\u001b[0m\u001b[1;32m   1658\u001b[0m                 \u001b[0mfunction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1659\u001b[0m                 \u001b[0mwith_indices\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwith_indices\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/code/hf_jax/datasets/src/datasets/arrow_dataset.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    183\u001b[0m         }\n\u001b[1;32m    184\u001b[0m         \u001b[0;31m# apply actual function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m         \u001b[0mout\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Dataset\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"DatasetDict\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m         \u001b[0mdatasets\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Dataset\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m         \u001b[0;31m# re-apply format to the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/code/hf_jax/datasets/src/datasets/fingerprint.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    395\u001b[0m             \u001b[0;31m# Call actual function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    396\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 397\u001b[0;31m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    398\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    399\u001b[0m             \u001b[0;31m# Update fingerprint of in-place transforms + update in-place history of transforms\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/code/hf_jax/datasets/src/datasets/arrow_dataset.py\u001b[0m in \u001b[0;36m_map_single\u001b[0;34m(self, function, with_indices, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset, desc)\u001b[0m\n\u001b[1;32m   2022\u001b[0m                             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2023\u001b[0m                                 \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcast_to_python_objects\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2024\u001b[0;31m                                 \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2025\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mupdate_data\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mwriter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2026\u001b[0m                     \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# close_stream=bool(buf_writer is None))  # We only close if we are writing in a file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/code/hf_jax/datasets/src/datasets/arrow_writer.py\u001b[0m in \u001b[0;36mwrite_batch\u001b[0;34m(self, batch_examples, writer_batch_size)\u001b[0m\n\u001b[1;32m    386\u001b[0m             \u001b[0mtyped_sequence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOptimizedTypedSequence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_examples\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcol_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtry_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcol_try_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    387\u001b[0m             \u001b[0mtyped_sequence_examples\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtyped_sequence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 388\u001b[0;31m         \u001b[0mpa_table\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pydict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtyped_sequence_examples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    389\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite_table\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpa_table\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwriter_batch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    390\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/pyarrow/table.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.Table.from_pydict\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/pyarrow/array.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.asarray\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/pyarrow/array.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.array\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/pyarrow/array.pxi\u001b[0m in \u001b[0;36mpyarrow.lib._handle_arrow_array_protocol\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;32m~/code/hf_jax/datasets/src/datasets/arrow_writer.py\u001b[0m in \u001b[0;36m__arrow_array__\u001b[0;34m(self, type)\u001b[0m\n\u001b[1;32m     98\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m                 \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mtrying_type\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_py\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    101\u001b[0m                 raise TypeError(\n\u001b[1;32m    102\u001b[0m                     \u001b[0;34m\"Specified try_type alters data. Please check that the type/feature that you provided match the type/features of the data.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/pyarrow/array.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.Array.__getitem__\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/pyarrow/array.pxi\u001b[0m in \u001b[0;36mpyarrow.lib._normalize_index\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;31mIndexError\u001b[0m: index out of bounds"
+     ]
+    }
+   ],
+   "source": [
+    "filtered_dataset = dataset.map(\n",
+    "    select_existing_rows,\n",
+    "    remove_columns = dataset.column_names,\n",
+    "    batched = True,\n",
+    "    num_proc = 1,\n",
+    "    desc = \"Selecting rows with images that exist\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 109,
+   "id": "7060ff8f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df['image_exists'] = df.apply(lambda row: image_exists(row['key']), axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 113,
+   "id": "fecc9a00",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image_size = 256\n",
+    "def image_transform(image):\n",
+    "    s = min(image.size)\n",
+    "    r = image_size / s\n",
+    "    s = (round(r * image.size[1]), round(r * image.size[0]))\n",
+    "    image = TF.resize(image, s, interpolation=InterpolationMode.LANCZOS)\n",
+    "    image = TF.center_crop(image, output_size = 2 * [image_size])\n",
+    "    image = torch.unsqueeze(T.ToTensor()(image), 0)\n",
+    "    image = image.permute(0, 2, 3, 1).numpy()\n",
+    "    return image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 98,
+   "id": "1a065700",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class YFC100Dataset(Dataset):\n",
+    "    def __init__(self, image_list_path: str, images_root: str, image_size: int, max_items=None):\n",
+    "        \"\"\"\n",
+    "        :param image_list_path: Path to a file containing a list of all images, in jsonl format.\n",
+    "        :param images_root: Root directory containing the images\n",
+    "        :param image_size: Image size. Source images will be resized and center-cropped.\n",
+    "        :max_items: Limit dataset size for debugging\n",
+    "        \"\"\"\n",
+    "        self.image_list = pd.read_json(image_list_path, orient=\"records\", lines=True)\n",
+    "        self.images_root = Path(images_root)\n",
+    "        if max_items is not None: self.image_list = self.image_list[:max_items]\n",
+    "        self.image_size = image_size\n",
+    "        \n",
+    "    def __len__(self):\n",
+    "        return len(self.image_list)\n",
+    "    \n",
+    "    def _get_raw_image(self, i):\n",
+    "        image_name = self.image_list.iloc[0].key\n",
+    "        image_path = (self.images_root/image_name[0:3]/image_name[3:6]/image_name).with_suffix('.jpg')\n",
+    "        return default_loader(image_path) if image_path.exists() else None\n",
+    "    \n",
+    "    # TODO: we could maybe use jax resizing / scaling functions\n",
+    "    def resize_image(self, image):\n",
+    "        s = min(image.size)\n",
+    "        r = self.image_size / s\n",
+    "        s = (round(r * image.size[1]), round(r * image.size[0]))\n",
+    "        image = TF.resize(image, s, interpolation=InterpolationMode.LANCZOS)\n",
+    "        image = TF.center_crop(image, output_size = 2 * [self.image_size])\n",
+    "        image = torch.unsqueeze(T.ToTensor()(image), 0)\n",
+    "        image = image.permute(0, 2, 3, 1).numpy()\n",
+    "        return image\n",
+    "    \n",
+    "    def _get_caption(self, i):\n",
+    "        # We are currently appending title and caption. Should we use another separator?\n",
+    "        row = self.image_list.iloc[i]\n",
+    "        return ' '.join(row.title_clean, row.description_clean)\n",
+    "    \n",
+    "    def __getitem__(self, i):\n",
+    "        image = self._get_raw_image(i)\n",
+    "        if image is None: return None\n",
+    "        image = self.resize_image(image)\n",
+    "        caption = self._get_caption(i)\n",
+    "        return {'image': image, 'text': caption}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 99,
+   "id": "4ce2211f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = YFC100Dataset(\n",
+    "    image_list_path = yfc100m_metadata,\n",
+    "    images_root = yfc100m_images,\n",
+    "    image_size = 256,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 100,
+   "id": "cc922704",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "5000"
+      ]
+     },
+     "execution_count": 100,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 102,
+   "id": "6e47ba46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataloader = DataLoader(dataset, batch_size=32, num_workers=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 103,
+   "id": "c8a130eb",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "Caught TypeError in DataLoader worker process 0.\nOriginal Traceback (most recent call last):\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py\", line 287, in _worker_loop\n    data = fetcher.fetch(index)\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\", line 47, in fetch\n    return self.collate_fn(data)\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\", line 86, in default_collate\n    raise TypeError(default_collate_err_msg_format.format(elem_type))\nTypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'NoneType'>\n",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m/tmp/ipykernel_320049/1409168804.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1201\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1202\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_task_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1203\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1205\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_try_put_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_process_data\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m   1227\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_put_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1228\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mExceptionWrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1229\u001b[0;31m             \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1230\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/_utils.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    423\u001b[0m             \u001b[0;31m# have message field\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    424\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mTypeError\u001b[0m: Caught TypeError in DataLoader worker process 0.\nOriginal Traceback (most recent call last):\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py\", line 287, in _worker_loop\n    data = fetcher.fetch(index)\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\", line 47, in fetch\n    return self.collate_fn(data)\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\", line 86, in default_collate\n    raise TypeError(default_collate_err_msg_format.format(elem_type))\nTypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'NoneType'>\n"
+     ]
+    }
+   ],
+   "source": [
+    "next(iter(dataloader))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "62ad01c3",
+   "metadata": {},
+   "source": [
+    "## Encoding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 89,
+   "id": "88f36d0b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def encode(model, batch):\n",
+    "    print(\"jitting encode function\")\n",
+    "#     _, indices = model.encode(batch)\n",
+    "\n",
+    "    # The model does not run in my computer (no cudNN currently installed) - faking it\n",
+    "    indices = [random.randint(0, 16384) for _ in range(256)]\n",
+    "    return indices"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 90,
+   "id": "1f35f0cb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def superbatch_generator(dataloader, num_tpus):\n",
+    "    iter_loader = iter(dataloader)\n",
+    "    for batch in iter_loader:\n",
+    "        superbatch = [batch.squeeze(1)]\n",
+    "        try:\n",
+    "            for b in range(num_tpus-1):\n",
+    "                batch = next(iter_loader)\n",
+    "                if batch is None:\n",
+    "                    break\n",
+    "                # Skip incomplete last batch\n",
+    "                if batch.shape[0] == dataloader.batch_size:\n",
+    "                    superbatch.append(batch.squeeze(1))\n",
+    "        except StopIteration:\n",
+    "            pass\n",
+    "        superbatch = torch.stack(superbatch, axis=0)\n",
+    "        yield superbatch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 93,
+   "id": "2210705b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import jax\n",
+    "\n",
+    "def encode_captioned_dataset(dataset, output_jsonl, batch_size=32, num_workers=16):\n",
+    "    if os.path.isfile(output_jsonl):\n",
+    "        print(f\"Destination file {output_jsonl} already exists, please move away.\")\n",
+    "        return\n",
+    "    \n",
+    "    num_tpus = jax.device_count()\n",
+    "    dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers)\n",
+    "    superbatches = superbatch_generator(dataloader, num_tpus=num_tpus)\n",
+    "    \n",
+    "    p_encoder = pmap(lambda batch: encode(model, batch))\n",
+    "\n",
+    "    # We save each superbatch to avoid reallocation of buffers as we process them.\n",
+    "    # We keep the file open to prevent excessive file seeks.\n",
+    "    with open(output_jsonl, \"w\") as file:\n",
+    "        iterations = len(dataset) // (batch_size * num_tpus)\n",
+    "        for n in tqdm(range(iterations)):\n",
+    "            superbatch = next(superbatches)\n",
+    "            encoded = p_encoder(superbatch.numpy())\n",
+    "            encoded = encoded.reshape(-1, encoded.shape[-1])\n",
+    "\n",
+    "            # Extract fields from the dataset internal `captions` property, and save to disk\n",
+    "            start_index = n * batch_size * num_tpus\n",
+    "            end_index = (n+1) * batch_size * num_tpus\n",
+    "            paths = dataset.captions[\"image_file\"][start_index:end_index].values\n",
+    "            captions = dataset.captions[\"caption\"][start_index:end_index].values\n",
+    "            encoded_as_string = list(map(lambda item: np.array2string(item, separator=',', max_line_width=50000, formatter={'int':lambda x: str(x)}), encoded))\n",
+    "            batch_df = pd.DataFrame.from_dict({\"image_file\": paths, \"caption\": captions, \"encoding\": encoded_as_string})\n",
+    "            batch_df = batch_df.dropna()\n",
+    "            batch_df.to_json(file, orient='records', lines=True, index=None)\n",
+    "            "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "id": "7704863d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  0%|                                                                                        | 0/78 [00:00<?, ?it/s]\n"
+     ]
+    },
+    {
+     "ename": "TypeError",
+     "evalue": "Caught TypeError in DataLoader worker process 0.\nOriginal Traceback (most recent call last):\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py\", line 287, in _worker_loop\n    data = fetcher.fetch(index)\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\", line 47, in fetch\n    return self.collate_fn(data)\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\", line 86, in default_collate\n    raise TypeError(default_collate_err_msg_format.format(elem_type))\nTypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'NoneType'>\n",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m/tmp/ipykernel_320049/140243368.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mencode_captioned_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myfc100m_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m/tmp/ipykernel_320049/2954345319.py\u001b[0m in \u001b[0;36mencode_captioned_dataset\u001b[0;34m(dataset, output_jsonl, batch_size, num_workers)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0miterations\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m//\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnum_tpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m             \u001b[0msuperbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msuperbatches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m             \u001b[0mencoded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp_encoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msuperbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m             \u001b[0mencoded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mencoded\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoded\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/tmp/ipykernel_320049/4148450576.py\u001b[0m in \u001b[0;36msuperbatch_generator\u001b[0;34m(dataloader, num_tpus)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msuperbatch_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_tpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0miter_loader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miter_loader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m         \u001b[0msuperbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1201\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1202\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_task_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1203\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1205\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_try_put_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_process_data\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m   1227\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_put_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1228\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mExceptionWrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1229\u001b[0;31m             \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1230\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/_utils.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    423\u001b[0m             \u001b[0;31m# have message field\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    424\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mTypeError\u001b[0m: Caught TypeError in DataLoader worker process 0.\nOriginal Traceback (most recent call last):\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py\", line 287, in _worker_loop\n    data = fetcher.fetch(index)\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\", line 47, in fetch\n    return self.collate_fn(data)\n  File \"/home/pedro/miniconda3/envs/hf_jax/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\", line 86, in default_collate\n    raise TypeError(default_collate_err_msg_format.format(elem_type))\nTypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'NoneType'>\n"
+     ]
+    }
+   ],
+   "source": [
+    "encode_captioned_dataset(dataset, yfc100m_output, batch_size=64, num_workers=16)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8953dd84",
+   "metadata": {},
+   "source": [
+    "----"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}