updated file

view_data.ipynb

@@ -0,0 +1,619 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "df = pd.read_csv('/Users/ludovicaschaerf/Desktop/Data/omniart_v3_datadump.csv')\n",
+    "df.shape, df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df['collection_origins'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df['general_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for val, c in zip(df['artwork_type'].value_counts(), df['artwork_type'].value_counts().index):\n",
+    "    print(val,c)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df['collection_origins'] == 'The British Library [Flickr]']['artwork_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df['collection_origins'] == 'The British Library [Flickr]']['general_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df['collection_origins'] == 'Brill Iconclass Arkyves']['artwork_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df['collection_origins'] == 'The Met 17']['artwork_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df['collection_origins'] == 'DeviantArt']['artwork_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df['collection_origins'] == 'WikiArts 17']['artwork_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df['collection_origins'] == 'MOMA - New York']['artwork_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = df[~df['artwork_type'].fillna('').str.contains('book')]\n",
+    "df_filtered.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = df_filtered[~df_filtered['artwork_type'].fillna('').str.contains('illustr')]\n",
+    "df_filtered.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = df_filtered[~df_filtered['artwork_type'].fillna('').str.contains('unknown')]\n",
+    "df_filtered.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "to_remove = df_filtered[df_filtered['collection_origins'] == 'Brill Iconclass Arkyves'][~(df_filtered['artwork_type'] == 'image')].index\n",
+    "df_filtered = df_filtered.drop(to_remove, axis=0)\n",
+    "df_filtered.shape, df_filtered[df_filtered['collection_origins'] == 'Brill Iconclass Arkyves']['artwork_type'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = df_filtered[~(df_filtered['collection_origins'] == 'The British Library [Flickr]')]\n",
+    "df_filtered.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = df_filtered.reset_index(drop=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = df_filtered[~(df_filtered['collection_origins'] == 'The Met 17')]\n",
+    "df_filtered.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = df[df['artwork_type'] == 'textiles'].reset_index()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered.sample(5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from PIL import Image\n",
+    "import requests\n",
+    "from io import BytesIO\n",
+    "\n",
+    "\n",
+    "print(df_filtered['image_url'][0])\n",
+    "# Download the image\n",
+    "response = requests.get(df_filtered['image_url'][0])\n",
+    "image_data = response.content\n",
+    "\n",
+    "# Open and display the image\n",
+    "image = Image.open(BytesIO(image_data))\n",
+    "image.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import random"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'random' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[1;32m/Users/ludovicaschaerf/Desktop/latent-space-theories/view_data.ipynb Cell 23\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/ludovicaschaerf/Desktop/latent-space-theories/view_data.ipynb#X25sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39m# Randomly select 20 images\u001b[39;00m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/ludovicaschaerf/Desktop/latent-space-theories/view_data.ipynb#X25sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m random_images \u001b[39m=\u001b[39m random\u001b[39m.\u001b[39msample(\u001b[39mlist\u001b[39m(df_filtered[\u001b[39m'\u001b[39m\u001b[39mimage_url\u001b[39m\u001b[39m'\u001b[39m]), \u001b[39m20\u001b[39m\u001b[39m*\u001b[39m\u001b[39m20\u001b[39m)\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/ludovicaschaerf/Desktop/latent-space-theories/view_data.ipynb#X25sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m \u001b[39m# Set up the grid layout\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/ludovicaschaerf/Desktop/latent-space-theories/view_data.ipynb#X25sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m grid_size \u001b[39m=\u001b[39m (\u001b[39m20\u001b[39m, \u001b[39m20\u001b[39m)\n",
+      "\u001b[0;31mNameError\u001b[0m: name 'random' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "# Randomly select 20 images\n",
+    "random_images = random.sample(list(df_filtered['image_url']), 20*20)\n",
+    "\n",
+    "# Set up the grid layout\n",
+    "grid_size = (20, 20)\n",
+    "fig, axs = plt.subplots(*grid_size, figsize=(100, 100))\n",
+    "\n",
+    "# Display the images in the grid\n",
+    "for i, image_link in enumerate(random_images):\n",
+    "    response = requests.get(image_link)\n",
+    "    image_data = response.content\n",
+    "    try:\n",
+    "        image = plt.imread(BytesIO(image_data), format='auto')\n",
+    "    except (OSError, IOError,):\n",
+    "        print(f\"Skipped image {i + 1} due to an error\")\n",
+    "\n",
+    "    # Compute the grid coordinates\n",
+    "    x = i % grid_size[1]\n",
+    "    y = i // grid_size[1]\n",
+    "    \n",
+    "    # Display the image in the corresponding grid cell\n",
+    "    axs[y, x].imshow(image)\n",
+    "    axs[y, x].axis('off')\n",
+    "\n",
+    "# Adjust spacing and display the grid of images\n",
+    "plt.tight_layout()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered['collection_origins'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = df_filtered[df_filtered['collection_origins']=='The Met 17'].reset_index()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered['omni_id']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered.loc[0, ['original_id_in_collection', 'artwork_name', 'artist_full_name']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "met_data = pd.read_csv('/Users/ludovicaschaerf/Desktop/Data/MetObjects.csv')\n",
+    "met_data.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered.to_csv('/Users/ludovicaschaerf/Desktop/Data/omniart_v3_textiles.csv', index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered.to_csv('/Users/ludovicaschaerf/Desktop/Data/omniart_v3_filtered.csv', index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "req = requests.get(\"https://api.vam.ac.uk/v2/objects/search?images_exist=true&id_category=THES48885&kw_object_type=Textile%20design&kw_object_type=Furnishing%20fabric&page_size=100\")\n",
+    "object_data = req.json()\n",
+    "print(object_data)\n",
+    "object_info = object_data[\"info\"]\n",
+    "object_records = object_data[\"records\"]\n",
+    "record_count = object_info[\"record_count\"]\n",
+    "print(f\"There are {record_count} objects that have these aspects in the record\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import urllib.request\n",
+    "\n",
+    "def download_image(url, file_path, file_name):\n",
+    "    full_path = file_path + file_name + '.jpg'\n",
+    "    urllib.request.urlretrieve(url, full_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm import tqdm\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv(f\"https://api.vam.ac.uk/v2/objects/search?images_exist=true&id_category=THES48885&kw_object_type=Textile%20design&response_format=csv&page={0}&kw_object_type=Furnishing%20fabric&page_size=100\")\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "f\"https://framemark.vam.ac.uk/collections/{df.loc[0, '_primaryImageId']}/full/735,/0/default.jpg\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out_path = '/Users/ludovicaschaerf/Desktop/Data/VA_textiles/'\n",
+    "\n",
+    "for i in tqdm(range(100)):\n",
+    "    df = pd.read_csv(f\"https://api.vam.ac.uk/v2/objects/search?images_exist=true&id_category=THES48885&kw_object_type=Textile%20design&response_format=csv&kw_object_type=Furnishing%20fabric&page_size=100&page={i}\")\n",
+    "        \n",
+    "    df.to_csv(out_path + f'info_{i}.csv', index = False)\n",
+    "            \n",
+    "    for j in range(100):\n",
+    "        im_url = f\"https://framemark.vam.ac.uk/collections/{df.loc[j, '_primaryImageId']}/full/735,/0/default.jpg\"\n",
+    "        name = df.loc[j, 'systemNumber']\n",
+    "            \n",
+    "        download_image(im_url, out_path, name)\n",
+    "        \n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in range(100):\n",
+    "    df = pd.read_csv(out_path + f'info_{i}.csv')\n",
+    "    \n",
+    "    if i == 0:\n",
+    "        total_df = df\n",
+    "        print(total_df.head())\n",
+    "    else:\n",
+    "        total_df = pd.concat([total_df, df], axis=0)\n",
+    "    \n",
+    "print(total_df.shape)\n",
+    "total_df.to_csv(out_path + f'complete_info.csv', index = False)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image = object_records[4]['_images']['_iiif_image_base_url'] + 'full/735,/0/default.jpg'\n",
+    "image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for rec in tqdm(object_records):\n",
+    "    im_url = rec['_images']['_iiif_image_base_url'] + 'full/735,/0/default.jpg'\n",
+    "    name = rec['systemNumber']\n",
+    "    \n",
+    "    download_image(im_url, out_path, name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import undetected_chromedriver as uc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "\n",
+    "def chromeUndetectableDriver():\n",
+    "    driver = uc.Chrome()\n",
+    "    return driver\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out_path = '/Users/ludovicaschaerf/Desktop/Data/GWU_textiles'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "import time\n",
+    "\n",
+    "from bs4 import BeautifulSoup as bs\n",
+    "\n",
+    "def scrape_gwm():\n",
+    "    driver = chromeUndetectableDriver()\n",
+    "    driver.get('https://collections-gwu.zetcom.net/en/collection/?f=withImages&f=publicDomain&ff=%7B%22classification_en_s%22%3A%5B%22Textile%22%5D%7D&om=5')\n",
+    "    \n",
+    "    d = {'Filename': [], 'Title': [], 'Geography': [], 'Culture': [], 'Materials': [],\n",
+    "         'Collection': [], 'Accession Number': [], 'Credit Line': [], 'Date': [],\n",
+    "         'Copyright': [], 'Object Type': [], 'Dimensions': [], 'Structure': [], 'Used in': []}\n",
+    "\n",
+    "    c = 1\n",
+    "    # scroll down until full page is rendered\n",
+    "    soup = None\n",
+    "    time.sleep(10)\n",
+    "    soup = bs(driver.page_source, 'html.parser')\n",
+    "    print(soup)\n",
+    "    # while True and c < 500:  # c < 100 to prevent infinite loop\n",
+    "    #     c += 1\n",
+    "    #     time.sleep(3)\n",
+    "    #     driver.execute_script(\"window.scrollTo(0,document.body.scrollHeight)\")\n",
+    "    #     ele = driver.find_element(\"xpath\", '//*[@class=\"SearchFormResults-buttonViewMore\"]')\n",
+    "    #     ele.click()\n",
+    "    #     time.sleep(3)\n",
+    "    #     s = bs(driver.page_source, 'html.parser')\n",
+    "    #     if s != soup:\n",
+    "    #         soup = s\n",
+    "    #         continue\n",
+    "    #     else:\n",
+    "    #         print('reached end of page')\n",
+    "    #         break\n",
+    "\n",
+    "    for a in tqdm(\n",
+    "            soup.findAll('div', id=re.compile('search-result-item-object-*'))):  # soup.findAll('div', 'item'): #for old version of website use this\n",
+    "        try:\n",
+    "            link = 'https://collections-gwu.zetcom.net/' + a.find('a')['href']  # a.find('a')['href'] #for old version of website\n",
+    "        except Exception as e:\n",
+    "            print('not working', e, a)\n",
+    "            continue\n",
+    "        \n",
+    "        name = a.find('a')['href']\n",
+    "        driver.get(link)\n",
+    "        time.sleep(2)\n",
+    "    \n",
+    "        soup_ = bs(driver.page_source, 'html.parser')\n",
+    "        i = soup_.find('img', {'class': 'Carousel-itemImage'})\n",
+    "        print(i)\n",
+    "        \n",
+    "        if i is not None:\n",
+    "            i =  'https://collections-gwu.zetcom.net/' + i['src']\n",
+    "            print(i, name)\n",
+    "            download_image(i, out_path, name)\n",
+    "            d['Filename'].append(name)\n",
+    "            d['Title'].append(soup_.find('h1', {'class': 'text-primary'}).text)\n",
+    "            for a in soup_.findAll('div', id='CollectionDetails-Item'):\n",
+    "                d[a.find('div', {'class': 'CollectionDetails-Item-Lable'}).text].append(a.find('div', {'class': 'CollectionDetails-Item-Body'}).text)\n",
+    "        else:\n",
+    "            print('Link was none,', soup_)\n",
+    "        \n",
+    "        print(d)\n",
+    "\n",
+    "    # pd.DataFrame(d).to_csv(output_csv, index=False)\n",
+    "    # return pd.DataFrame(d)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scrape_gwm()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! python /Users/ludovicaschaerf/Desktop/Repos/ailia-models/background_removal/indexnet/indexnet.py --input /Users/ludovicaschaerf/Desktop/Data/VA_textiles/O25180.jpg --savepath /Users/ludovicaschaerf/Desktop/Data/NO_BG/no_bg_O25180.jpg -a u2net"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "art-reco_x86",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.16"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

view_predictions.ipynb

@@ -2,10 +2,31 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "fe7acfaf-dc61-4211-9c78-8e4433bc9deb",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Intel MKL WARNING: Support of Intel(R) Streaming SIMD Extensions 4.2 (Intel(R) SSE4.2) enabled only processors has been deprecated. Intel oneAPI Math Kernel Library 2025.0 will require Intel(R) Advanced Vector Extensions (Intel(R) AVX) instructions.\n",
+      "Intel MKL WARNING: Support of Intel(R) Streaming SIMD Extensions 4.2 (Intel(R) SSE4.2) enabled only processors has been deprecated. Intel oneAPI Math Kernel Library 2025.0 will require Intel(R) Advanced Vector Extensions (Intel(R) AVX) instructions.\n"
+     ]
+    },
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'sklearn'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 8\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mtorch\u001b[39;00m\n\u001b[1;32m      7\u001b[0m \u001b[39m#import ninja\u001b[39;00m\n\u001b[0;32m----> 8\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mbackend\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mdisentangle_concepts\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n\u001b[1;32m      9\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mdnnlib\u001b[39;00m \n\u001b[1;32m     10\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mlegacy\u001b[39;00m\n",
+      "File \u001b[0;32m~/Desktop/latent-space-theories/backend/disentangle_concepts.py:2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mnumpy\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mnp\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39msklearn\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39msvm\u001b[39;00m \u001b[39mimport\u001b[39;00m SVC\n\u001b[1;32m      3\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39msklearn\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mlinear_model\u001b[39;00m \u001b[39mimport\u001b[39;00m LogisticRegression\n\u001b[1;32m      4\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39msklearn\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mmodel_selection\u001b[39;00m \u001b[39mimport\u001b[39;00m train_test_split\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'sklearn'"
+     ]
+    }
+   ],
    "source": [
     "import pandas as pd\n",
     "import pickle\n",
@@ -150,10 +171,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "0eae840f",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/ludovicaschaerf/anaconda3/envs/torch_arm/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
     "import open_clip\n",
     "import os\n",
@@ -164,7 +194,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "4d776015",
    "metadata": {},
    "outputs": [],
@@ -175,10 +205,33 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "e3f917a7",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['Provenance ADRIA', 'Provenance AEGINA', 'Provenance AL MINA', 'Provenance ALICANTE', 'Provenance AMATHUS', 'Provenance AMPURIAS', 'Provenance APOLLONIA PONTICA', 'Provenance APULIA', 'Provenance ARGOLIS', 'Provenance ARGOS', 'Provenance ATHENS', 'Provenance ATHENS (?)', 'Provenance ATTICA', 'Provenance BARI', 'Provenance BENGHAZI', 'Provenance BEREZAN', 'Provenance BLACK SEA', 'Provenance BOEOTIA', 'Provenance BOLOGNA', 'Provenance CAPUA', 'Provenance CARIA', 'Provenance CARTHAGE', 'Provenance CHIUSI', 'Provenance CIVITAVECCHIA', 'Provenance CORINTH', 'Provenance CORSICA', 'Provenance CRETE', 'Provenance CRIMEA', 'Provenance CUMAE', 'Provenance CYCLADES', 'Provenance CYPRUS', 'Provenance CYRENAICA', 'Provenance CYRENE', 'Provenance EGYPT', 'Provenance ELIS', 'Provenance ENSERUNE', 'Provenance ETRURIA', 'Provenance EUBOEA', 'Provenance FALERII', 'Provenance GRANADA', 'Provenance GREECE', 'Provenance GREECE (?)', 'Provenance HISTRIA', 'Provenance ITALY', 'Provenance JAEN', 'Provenance KITION', 'Provenance LESBOS', 'Provenance LOCRI', 'Provenance LOCRIS', 'Provenance LYCIA', 'Provenance MACEDONIA', 'Provenance MARION', 'Provenance MARSEILLES', 'Provenance MARZABOTTO', 'Provenance METAPONTUM', 'Provenance MILETUS', 'Provenance NAPLES', 'Provenance NAUCRATIS', 'Provenance NOLA', 'Provenance NUMANA', 'Provenance OLD SMYRNA', 'Provenance ORVIETO', 'Provenance PAESTUM', 'Provenance PHOCIS', 'Provenance PITANE', 'Provenance POPULONIA', 'Provenance RHODES', 'Provenance ROME', 'Provenance RUVO', 'Provenance SALERNO', 'Provenance SAMOS', 'Provenance SARDINIA', 'Provenance SARDIS', 'Provenance SICILY', 'Provenance SMYRNA', 'Provenance SOUTH', 'Provenance SPARTA', 'Provenance SPINA', 'Provenance SUESSULA', 'Provenance TARANTO', 'Provenance TELL DEFENNEH', 'Provenance THASOS', 'Provenance THERA', 'Provenance THESSALY', 'Provenance THRACE', 'Provenance TOCRA', 'Provenance TODI', 'Provenance ULLASTRET', 'Provenance VALENCIA', 'Shape Name ALABASTRON', 'Shape Name ALABASTRON FRAGMENT', 'Shape Name AMPHORA', 'Shape Name AMPHORA (?) FRAGMENT', 'Shape Name AMPHORISKOS', 'Shape Name ARYBALLOS', 'Shape Name ASKOS', 'Shape Name ASKOS FRAGMENT', 'Shape Name BOTTLE', 'Shape Name BOWL', 'Shape Name BOWL FRAGMENT', 'Shape Name CHALICE', 'Shape Name CHALICE FRAGMENT', 'Shape Name CHOUS', 'Shape Name CHOUS FRAGMENT', 'Shape Name CUP', 'Shape Name CUP (?) FRAGMENT', 'Shape Name CUP A', 'Shape Name CUP A FRAGMENT', 'Shape Name CUP A FRAGMENTS', 'Shape Name CUP B', 'Shape Name CUP B FRAGMENT', 'Shape Name CUP B FRAGMENTS', 'Shape Name CUP C', 'Shape Name CUP C FRAGMENT', 'Shape Name CUP C FRAGMENTS', 'Shape Name CUP DROOP', 'Shape Name CUP DROOP FRAGMENT', 'Shape Name CUP FRAGMENT', 'Shape Name CUP FRAGMENTS', 'Shape Name CUP KASSEL', 'Shape Name CUP KASSEL FRAGMENT', 'Shape Name CUP LITTLE MASTER BAND', 'Shape Name CUP LITTLE MASTER BAND (?) FRAGMENT', 'Shape Name CUP LITTLE MASTER BAND FRAGMENT', 'Shape Name CUP LITTLE MASTER BAND FRAGMENTS', 'Shape Name CUP LITTLE MASTER FRAGMENT', 'Shape Name CUP LITTLE MASTER LIP', 'Shape Name CUP LITTLE MASTER LIP FRAGMENT', 'Shape Name CUP LITTLE MASTER LIP FRAGMENTS', 'Shape Name CUP SIANA', 'Shape Name CUP SIANA FRAGMENT', 'Shape Name CUP SIANA FRAGMENTS', 'Shape Name CUP SKYPHOS', 'Shape Name CUP SKYPHOS FRAGMENT', 'Shape Name CUP STEMLESS', 'Shape Name CUP STEMLESS FRAGMENT', 'Shape Name CUP STEMLESS FRAGMENTS', 'Shape Name DINOS', 'Shape Name DINOS FRAGMENT', 'Shape Name DISH', 'Shape Name EPICHYSIS', 'Shape Name EPINETRON FRAGMENT', 'Shape Name FEEDER', 'Shape Name FIGURE VASE', 'Shape Name FIGURE VASE ARYBALLOS', 'Shape Name FIGURE VASE ASKOS', 'Shape Name FIGURE VASE FRAGMENT', 'Shape Name FIGURE VASE KANTHAROS', 'Shape Name FIGURE VASE LEKYTHOS', 'Shape Name FIGURE VASE OINOCHOE', 'Shape Name FISH-PLATE', 'Shape Name FLASK', 'Shape Name FRAGMENT', 'Shape Name FRAGMENTS', 'Shape Name GUTTUS', 'Shape Name HYDRIA', 'Shape Name HYDRIA (?) FRAGMENT', 'Shape Name HYDRIA FRAGMENT', 'Shape Name HYDRIA FRAGMENTS', 'Shape Name INCENSE BURNER', 'Shape Name JAR', 'Shape Name JUG', 'Shape Name KALATHOS', 'Shape Name KANTHAROS', 'Shape Name KANTHAROS FRAGMENT', 'Shape Name KERNOS', 'Shape Name KOTYLE', 'Shape Name KOTYLE FRAGMENT', 'Shape Name KRATER', 'Shape Name KRATER (?) FRAGMENT', 'Shape Name KRATER FRAGMENT', 'Shape Name KRATER FRAGMENTS', 'Shape Name KYATHOS', 'Shape Name KYATHOS FRAGMENT', 'Shape Name LEBES', 'Shape Name LEBES FRAGMENT', 'Shape Name LEKANIS', 'Shape Name LEKANIS FRAGMENT', 'Shape Name LEKANIS FRAGMENTS', 'Shape Name LEKANIS LID', 'Shape Name LEKANIS LID FRAGMENT', 'Shape Name LEKYTHOS', 'Shape Name LEKYTHOS FRAGMENT', 'Shape Name LEKYTHOS FRAGMENTS', 'Shape Name LID', 'Shape Name LID FRAGMENT', 'Shape Name LOUTROPHOROS', 'Shape Name LOUTROPHOROS FRAGMENT', 'Shape Name LOUTROPHOROS FRAGMENTS', 'Shape Name LYDION', 'Shape Name MASTOID', 'Shape Name MINIATURE PANATHENAIC AMPHORA', 'Shape Name MUG', 'Shape Name MUG FRAGMENT', 'Shape Name NESTORIS', 'Shape Name OINOCHOE', 'Shape Name OINOCHOE (?) FRAGMENT', 'Shape Name OINOCHOE FRAGMENT', 'Shape Name OINOCHOE FRAGMENTS', 'Shape Name OLLA', 'Shape Name OLPE', 'Shape Name OLPE FRAGMENT', 'Shape Name OLPE FRAGMENTS', 'Shape Name PELIKE', 'Shape Name PELIKE FRAGMENT', 'Shape Name PELIKE FRAGMENTS', 'Shape Name PHIALE', 'Shape Name PHIALE FRAGMENT', 'Shape Name PITCHER', 'Shape Name PLAQUE FRAGMENT', 'Shape Name PLAQUE FRAGMENTS', 'Shape Name PLATE', 'Shape Name PLATE FRAGMENT', 'Shape Name PLATE FRAGMENTS', 'Shape Name PLEMOCHOE', 'Shape Name PSEUDO-PANATHENAIC AMPHORA', 'Shape Name PSEUDO-PANATHENAIC AMPHORA FRAGMENT', 'Shape Name PSYKTER', 'Shape Name PYXIS', 'Shape Name PYXIS FRAGMENT', 'Shape Name PYXIS FRAGMENTS', 'Shape Name PYXIS LID', 'Shape Name PYXIS LID FRAGMENT', 'Shape Name RHYTON', 'Shape Name SKYPHOS', 'Shape Name SKYPHOS (?) FRAGMENT', 'Shape Name SKYPHOS FRAGMENT', 'Shape Name SKYPHOS FRAGMENTS', 'Shape Name STAMNOS', 'Shape Name STAMNOS FRAGMENT', 'Shape Name STAMNOS FRAGMENTS', 'Shape Name STAND', 'Shape Name STAND FRAGMENT', 'Shape Name STEMLESS CUP FRAGMENT', 'Shape Name STIRRUP JAR', 'Shape Name TANKARD', 'Shape Name UNKNOWN', 'Shape Name URN', 'Shape Name VARIOUS', 'Shape Name VASE', 'Fabric ARGIVE GEOMETRIC', 'Fabric ARRETINE', 'Fabric ATHENIAN', 'Fabric ATHENIAN (?)', 'Fabric ATHENIAN GEOMETRIC', 'Fabric ATHENIAN PROTOGEOMETRIC', 'Fabric BOEOTIAN', 'Fabric BOEOTIAN GEOMETRIC', 'Fabric CALENIAN', 'Fabric CAMPANA', 'Fabric CANOSAN', 'Fabric CHALCIDIAN', 'Fabric CORINTHIAN', 'Fabric CRETAN', 'Fabric CYCLADIC', 'Fabric CYPRIOT', 'Fabric CYPRIOT, BRONZE AGE', 'Fabric CYPRIOT, IRON AGE', 'Fabric CYPRIOT, MYCENAEAN STYLE', 'Fabric DAUNIAN', 'Fabric EAST GREEK', 'Fabric EAST GREEK GEOMETRIC', 'Fabric EAST GREEK, CLAZOMENIAN', 'Fabric EAST GREEK, FIKELLURA', 'Fabric EAST GREEK, NAUCRATITE', 'Fabric EGYPTIAN', 'Fabric ETRUSCAN', 'Fabric ETRUSCO-CORINTHIAN', 'Fabric FALISCAN', 'Fabric GALLIC', 'Fabric GALLO-ROMAN', 'Fabric GREEK', 'Fabric HELLADIC', 'Fabric HELLENISTIC', 'Fabric IBERIAN', 'Fabric IONIAN', 'Fabric ITALIOTE', 'Fabric ITALO-CORINTHIAN', 'Fabric ITALO-GEOMETRIC', 'Fabric LACONIAN', 'Fabric LACONIAN GEOMETRIC', 'Fabric MESSAPIAN', 'Fabric MINOAN', 'Fabric MYCENEAN', 'Fabric PONTIC', 'Fabric PROTO-ELAMITE', 'Fabric PROTOATTIC', 'Fabric PROTOCORINTHIAN', 'Fabric PROTOCORINTHIAN, TRANSITIONAL', 'Fabric ROMAN', 'Fabric SOUTH ITALIAN', 'Fabric SOUTH ITALIAN, APULIAN', 'Fabric SOUTH ITALIAN, CAMPANIAN', 'Fabric SOUTH ITALIAN, GNATHIAN', 'Fabric SOUTH ITALIAN, LUCANIAN', 'Fabric SOUTH ITALIAN, PAESTAN', 'Fabric SOUTH ITALIAN, SICILIAN', 'Fabric TERRA SIGILLATA', 'Fabric UNCERTAIN', 'Fabric VEIAN', 'Fabric VILLA NOVA', 'Technique ADDED COLOUR', 'Technique BLACK GLAZE', 'Technique BLACK PATTERN', 'Technique BLACK-FIGURE', 'Technique BROWN GLAZE', 'Technique BUCCHERO', 'Technique IMPASTO', 'Technique OUTLINE', 'Technique PATTERN', 'Technique PLAIN', 'Technique PSEUDO RED-FIGURE', 'Technique RED POLISHED WARE', 'Technique RED-FIGURE', 'Technique RELIEF', 'Technique RESERVING', 'Technique SILHOUETTE', 'Technique WHITE PAINTED WARE']\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/ludovicaschaerf/anaconda3/envs/torch_arm/lib/python3.11/site-packages/torch/amp/autocast_mode.py:221: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
+      "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(318, 768)\n"
+     ]
+    }
+   ],
    "source": [
     "model_clip, _, preprocess = open_clip.create_model_and_transforms('ViT-L-14', pretrained='laion2b_s32b_b82k')\n",
     "tokenizer = open_clip.get_tokenizer('ViT-L-14')\n",
@@ -204,7 +257,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "f7858bbf",
    "metadata": {},
    "outputs": [],
@@ -214,7 +267,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "de6bd428",
    "metadata": {},
    "outputs": [],
@@ -224,7 +277,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "d19c8e4c",
    "metadata": {},
    "outputs": [],