init working on depth demo

multimodal_domain_adaptation_using_HD.ipynb

@@ -0,0 +1,319 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: matplotlib in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (3.5.1)\n",
+      "Requirement already satisfied: seaborn in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (0.11.2)\n",
+      "Requirement already satisfied: scikit-learn in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (1.0.2)\n",
+      "Requirement already satisfied: numpy in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (1.21.6)\n",
+      "Requirement already satisfied: pandas in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (1.3.5)\n",
+      "Requirement already satisfied: pillow in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (9.0.0)\n",
+      "Requirement already satisfied: transformers in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (4.24.0)\n",
+      "Requirement already satisfied: python-dateutil>=2.7 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from matplotlib) (2.8.2)\n",
+      "Requirement already satisfied: packaging>=20.0 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from matplotlib) (21.3)\n",
+      "Requirement already satisfied: cycler>=0.10 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from matplotlib) (0.11.0)\n",
+      "Requirement already satisfied: pyparsing>=2.2.1 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from matplotlib) (2.4.7)\n",
+      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from matplotlib) (1.3.2)\n",
+      "Requirement already satisfied: fonttools>=4.22.0 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from matplotlib) (4.28.5)\n",
+      "Requirement already satisfied: scipy>=1.0 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from seaborn) (1.7.1)\n",
+      "Requirement already satisfied: joblib>=0.11 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from scikit-learn) (1.1.0)\n",
+      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from scikit-learn) (3.1.0)\n",
+      "Requirement already satisfied: pytz>=2017.3 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from pandas) (2021.3)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from transformers) (6.0)\n",
+      "Requirement already satisfied: filelock in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from transformers) (3.8.0)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from transformers) (2022.10.31)\n",
+      "Requirement already satisfied: requests in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from transformers) (2.26.0)\n",
+      "Requirement already satisfied: tqdm>=4.27 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from transformers) (4.64.0)\n",
+      "Requirement already satisfied: importlib-metadata in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from transformers) (4.2.0)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from transformers) (0.16.4)\n",
+      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from transformers) (0.13.2)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.7.1)\n",
+      "Requirement already satisfied: fsspec in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (2022.1.0)\n",
+      "Requirement already satisfied: six>=1.5 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib) (1.15.0)\n",
+      "Requirement already satisfied: zipp>=0.5 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.6.0)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from requests->transformers) (3.2)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from requests->transformers) (2021.5.30)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from requests->transformers) (1.26.7)\n",
+      "Requirement already satisfied: charset-normalizer~=2.0.0 in /Users/e29154/.pyenv/versions/3.7-dev/lib/python3.7/site-packages (from requests->transformers) (2.0.6)\n",
+      "\u001b[33mWARNING: You are using pip version 21.3.1; however, version 23.3.2 is available.\n",
+      "You should consider upgrading via the '/Users/e29154/.pyenv/versions/3.7-dev/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install matplotlib seaborn scikit-learn numpy pandas pillow transformers "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-01-02 15:59:48.762816: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
+      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "import numpy as np\n",
+    "from numpy.random import MT19937\n",
+    "from numpy.random import RandomState, SeedSequence\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "sns.set_style('whitegrid')\n",
+    "rs = RandomState(MT19937(SeedSequence(123456789)))\n",
+    "import math\n",
+    "import pandas as pd\n",
+    "from turtle import title\n",
+    "import numpy as np\n",
+    "from PIL import Image\n",
+    "from transformers import CLIPProcessor, CLIPModel\n",
+    "import pandas as pd\n",
+    "from glob import glob\n",
+    "import random\n",
+    "from datetime import datetime\n",
+    "import numpy as np\n",
+    "from numpy.random import MT19937\n",
+    "from numpy.random import RandomState, SeedSequence"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "clip_model = CLIPModel.from_pretrained(\"openai/clip-vit-base-patch32\") \n",
+    "clip_processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-base-patch32\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def make_LUT(nvalues, dims):\n",
+    "    lut = np.zeros(shape=(nvalues, dims))\n",
+    "    lut[0, :] = rs.binomial(n=1, p=0.5, size=(dims))\n",
+    "    for row in range(1, nvalues):\n",
+    "        lut[row, :] = lut[row-1, :]\n",
+    "        # flip few randomly\n",
+    "        rand_idx = rs.choice(dims, size=dims//nvalues, replace=False)\n",
+    "        lut[row, rand_idx] = 1 - lut[row, rand_idx]\n",
+    "        assert np.abs(lut[row, :] - lut[row-1, :]).sum() ==dims//nvalues \n",
+    "    unique_rows = np.unique(lut, axis=0)\n",
+    "    assert len(unique_rows) == len(lut)\n",
+    "    return lut"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(256,) -1.0 1.0 val bins\n"
+     ]
+    }
+   ],
+   "source": [
+    "HYPERDIMS = 1024\n",
+    "VALUE_BITS = 8\n",
+    "POS_BITS = 9 # CLIP features are 512 dims\n",
+    "val_bins = np.linspace(start=-1., stop=1., num=2**VALUE_BITS)\n",
+    "print(val_bins.shape, val_bins.min(), val_bins.max(), 'val bins')\n",
+    "val_lut = make_LUT(2**VALUE_BITS, HYPERDIMS)\n",
+    "assert val_lut.shape[0] == val_bins.shape[0]\n",
+    "pos_lut = rs.binomial(n=1, p=0.5, size=(2**POS_BITS, HYPERDIMS))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def extract_features(image):\n",
+    "    PIL_image = Image.fromarray(np.uint8(image)).convert('RGB')\n",
+    "    inputs = clip_processor(text=[\"a photo of a cat\", \"a photo of a dog\"], images=PIL_image, return_tensors=\"pt\", padding=True)\n",
+    "    outputs = clip_model(**inputs)\n",
+    "    # print(outputs.image_embeds.shape)\n",
+    "    return outputs.image_embeds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "   \n",
+    "def quantize_embeds(embeds):\n",
+    "    assert np.all(embeds >= val_bins[0])\n",
+    "    assert np.all(embeds <= val_bins[-1])\n",
+    "    embeds_flat = embeds.flatten()\n",
+    "\n",
+    "    all_pairs_dist = np.abs(embeds_flat[:, np.newaxis] - val_bins[np.newaxis, :])\n",
+    "    closest_bin = np.argmin(all_pairs_dist, axis=-1)\n",
+    "    quantized_embeds_flat = val_bins[closest_bin]\n",
+    "    quantized_embeds = np.reshape(quantized_embeds_flat, embeds.shape)\n",
+    "    closest_bin = np.reshape(closest_bin, embeds.shape)\n",
+    "    print(closest_bin.shape, 'values are in bins', closest_bin.min(), 'to', closest_bin.max())\n",
+    "    print('abs quant error avg', np.abs(embeds - quantized_embeds).mean())\n",
+    "    return quantized_embeds, closest_bin\n",
+    "\n",
+    "def update_exemplars(df, rng, exemplars, lut):\n",
+    "    embeds = np.array(df['image_embed'].values.tolist()) # df[['image_embed']].to_numpy()\n",
+    "    labels = np.array(df['label'].values.tolist(), 'int')\n",
+    "    # print(labels, labels.shape)\n",
+    "    assert np.all(np.unique(labels) == [0, 1])\n",
+    "    labels_zero_idx = (labels == 0).nonzero()[0]\n",
+    "    labels_one_idx = (labels == 1).nonzero()[0]\n",
+    "    print(labels_zero_idx.shape, \" zeros and \", labels_one_idx.shape, \" ones\")\n",
+    "    # 70-30 split\n",
+    "    labels_zero_train_idx = rng[0].choice(labels_zero_idx, size=int(.7 * len(labels_zero_idx)), replace=False)\n",
+    "    labels_one_train_idx = rng[0].choice(labels_one_idx, size=int(.7 * len(labels_one_idx)), replace=False)\n",
+    "    embeds_train = np.concatenate([embeds[labels_zero_train_idx], embeds[labels_one_train_idx]], axis=0)\n",
+    "    labels_train = np.concatenate([labels[labels_zero_train_idx], labels[labels_one_train_idx]], axis=0)\n",
+    "    print('Training set ', embeds_train.shape, labels_train.shape)\n",
+    "    print(np.sum(labels_train == 0), \" zeros and \", np.sum(labels_train == 1).sum(), \" ones\")\n",
+    "    labels_zero_test_idx = np.setdiff1d(labels_zero_idx, labels_zero_train_idx)\n",
+    "    labels_one_test_idx = np.setdiff1d(labels_one_idx, labels_one_train_idx)\n",
+    "    embeds_test = np.concatenate([embeds[labels_zero_test_idx], embeds[labels_one_test_idx]], axis=0)\n",
+    "    labels_test = np.concatenate([labels[labels_zero_test_idx], labels[labels_one_test_idx]], axis=0)\n",
+    "    print('Test set ', embeds_test.shape, labels_test.shape)\n",
+    "\n",
+    "    quantized_embeds, closest_bin = quantize_embeds(embeds_train)\n",
+    "    # closest bin is nexample X 512\n",
+    "    # lut[0] is nvals X dims\n",
+    "    # hd_embeds in nexample x 512 x dims\n",
+    "    hd_embeds_per_pos = lut[0][closest_bin]\n",
+    "    # bundle along pos dimension 512\n",
+    "    # lut[1] is 512 x dims\n",
+    "    xor = lambda a,b: a*(1.-b) + b*(1.-a)\n",
+    "    hd_embeds = xor(lut[1][np.newaxis, ...], hd_embeds_per_pos)\n",
+    "    hd_embeds = np.sum(hd_embeds, axis=1) / embeds_train.shape[-1]\n",
+    "    hd_embeds[hd_embeds >= 0.5] = 1.\n",
+    "    hd_embeds[hd_embeds < 0.5] = 0.\n",
+    "    # hd_embeds_integer is nexample x dims\n",
+    "    \n",
+    "    exemplars_integer = [None, None]\n",
+    "    exemplars_integer[0] = np.sum(hd_embeds[labels_train == 0], axis=0)\n",
+    "    exemplars_integer[1] = np.sum(hd_embeds[labels_train == 1], axis=0)\n",
+    "    exemplars[0] = exemplars_integer[0] / np.sum(labels_train == 0)\n",
+    "    exemplars[1] = exemplars_integer[1] / np.sum(labels_train == 1)\n",
+    "    exemplars[0][exemplars[0] >= 0.5] = 1.\n",
+    "    exemplars[0][exemplars[0] < 0.5] = 0.\n",
+    "    exemplars[1][exemplars[1] >= 0.5] = 1.\n",
+    "    exemplars[1][exemplars[1] < 0.5] = 0.\n",
+    "    print(exemplars[0].shape, exemplars[1].shape, np.abs(exemplars[0] - exemplars[1]).sum())\n",
+    "    preds = np.zeros(hd_embeds.shape[0])\n",
+    "    dist_to_ex0 = np.abs(hd_embeds - exemplars[0][np.newaxis, ...]).sum(axis=-1)\n",
+    "    dist_to_ex1 = np.abs(hd_embeds - exemplars[1][np.newaxis, ...]).sum(axis=-1)\n",
+    "    preds[dist_to_ex1 < dist_to_ex0] = 1\n",
+    "    print(preds.shape, labels_train.shape, np.sum(preds == labels_train))\n",
+    "    train_acc = np.sum(preds == labels_train) / len(labels_train)\n",
+    "    rng, test_acc = score(embeds_test, labels_test, rng, exemplars, lut)\n",
+    "    return rng, exemplars, train_acc, test_acc\n",
+    "\n",
+    "def score(embeds, labels, rng, exemplars, lut):\n",
+    "    quantized_embeds, closest_bin = quantize_embeds(embeds)\n",
+    "    # closest bin is nexample X 512\n",
+    "    # lut[0] is nvals X dims\n",
+    "    # hd_embeds in nexample x 512 x dims\n",
+    "    hd_embeds_per_pos = lut[0][closest_bin]\n",
+    "    # bundle along pos dimension 512\n",
+    "    # lut[1] is 512 x dims\n",
+    "    xor = lambda a,b: a*(1.-b) + b*(1.-a)\n",
+    "    hd_embeds = xor(lut[1][np.newaxis, ...], hd_embeds_per_pos)\n",
+    "    hd_embeds = np.sum(hd_embeds, axis=1) / embeds.shape[-1]\n",
+    "    hd_embeds[hd_embeds >= 0.5] = 1.\n",
+    "    hd_embeds[hd_embeds < 0.5] = 0.\n",
+    "    # hd_embeds_integer is nexample x dims\n",
+    "    print(exemplars[0].shape, exemplars[1].shape, np.abs(exemplars[0] - exemplars[1]).sum())\n",
+    "    preds = np.zeros(hd_embeds.shape[0])\n",
+    "    dist_to_ex0 = np.abs(hd_embeds - exemplars[0][np.newaxis, ...]).sum(axis=-1)\n",
+    "    dist_to_ex1 = np.abs(hd_embeds - exemplars[1][np.newaxis, ...]).sum(axis=-1)\n",
+    "    preds[dist_to_ex1 < dist_to_ex0] = 1\n",
+    "    print(preds.shape, labels.shape, np.sum(preds == labels), len(labels))\n",
+    "    acc = np.sum(preds == labels) / len(labels)\n",
+    "    return rng, acc\n",
+    "\n",
+    "def predict(embeds, exemplars, lut):\n",
+    "    quantized_embeds, closest_bin = quantize_embeds(embeds)\n",
+    "    # closest bin is nexample X 512\n",
+    "    # lut[0] is nvals X dims\n",
+    "    # hd_embeds in nexample x 512 x dims\n",
+    "    hd_embeds_per_pos = lut[0][closest_bin]\n",
+    "    # bundle along pos dimension 512\n",
+    "    # lut[1] is 512 x dims\n",
+    "    xor = lambda a,b: a*(1.-b) + b*(1.-a)\n",
+    "    hd_embeds = xor(lut[1][np.newaxis, ...], hd_embeds_per_pos)\n",
+    "    hd_embeds = np.sum(hd_embeds, axis=1) / embeds.shape[-1]\n",
+    "    hd_embeds[hd_embeds >= 0.5] = 1.\n",
+    "    hd_embeds[hd_embeds < 0.5] = 0.\n",
+    "    # hd_embeds_integer is nexample x dims\n",
+    "    # print(exemplars[0].shape, exemplars[1].shape, np.abs(exemplars[0] - exemplars[1]).sum())\n",
+    "    dist_to_ex0 = np.abs(hd_embeds - exemplars[0][np.newaxis, ...]).sum(axis=-1)\n",
+    "    dist_to_ex1 = np.abs(hd_embeds - exemplars[1][np.newaxis, ...]).sum(axis=-1)\n",
+    "    print('dists', dist_to_ex0, dist_to_ex1)\n",
+    "    odds = np.abs(dist_to_ex0 - dist_to_ex1).item()\n",
+    "    if dist_to_ex1 < dist_to_ex0:\n",
+    "        preds = np.array([1., odds])\n",
+    "    else:\n",
+    "        preds = np.array([odds, 1.])\n",
+    "    print(preds)\n",
+    "    # preds = np.array([-1. * dist_to_ex0, -1. * dist_to_ex1])\n",
+    "    preds = preds / preds.sum()\n",
+    "    # print(preds.shape)\n",
+    "    print(preds)\n",
+    "    return {\"👍\": preds[1], \"👎\": preds[0]}"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "midas-py310",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12+"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}