{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "IsB9l3mBIGUN" }, "source": [ "## Analysis" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from PIL import Image\n", "from scipy.stats import pearsonr\n", "from utils.get_unique_values import get_unique_values\n", "from utils.remove_duplicates import unzip_fn\n", "from utils.show_tile_images import show_tile_images\n", "import zipfile\n", "import json\n", "from utils.visualize_bboxes_on_image import draw_text_on_image\n", "import numpy as np\n", "from sklearn.metrics.pairwise import cosine_similarity\n", "import matplotlib.pyplot as plt\n", "import tqdm as tqdm\n", "from functools import cache\n", "from utils.flatten import flatten" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5l6iv7ZrIGUP" }, "outputs": [], "source": [ "# !GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/spaces/mckabue/document-similarity-search-using-visual-layout-features --depth=1\n", "\n", "# !wget https://huggingface.co/spaces/mckabue/document-similarity-search-using-visual-layout-features/resolve/main/data/processed/RVL-CDIP-invoice/vectors.json.zip -P ./data/processed/RVL-CDIP-invoice/\n", "\n", "\n", "\n", "# import sys\n", "# sys.path.insert(0, './document-similarity-search-using-visual-layout-features')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "172P8Ey8ytD9" }, "outputs": [], "source": [ "# import os\n", "# vectors_chunks = os.listdir('/content/document-similarity-search-using-visual-layout-features/data/processed/RVL-CDIP-invoice/vectors.json.zip.chunks')\n", "# vectors_chunks.sort(key=lambda x: int(x.split('-')[0]))\n", "# vectors_chunks" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ZZD9JBaWa_T_" }, "outputs": [], "source": [ "vectors_df = pd.read_json('./data/local-data/processed/RVL-CDIP-invoice/vectors.json.zip')\n", "vectors_df" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# https://gemini.google.com/app/8cd4389df12d29e6\n", "\n", "# https://chat.openai.com/c/a345a9ec-9238-4089-a6c0-bb4d375148eb" ] }, { "cell_type": "markdown", "metadata": { "id": "X0n7rBnZIGUQ" }, "source": [ "### Correlation" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "unique_values = get_unique_values(start=0.17, end=1, count=10*1000)\n", "\n", "def get_stats(index: int):\n", " vectors = vectors_df.loc[index, 'vectors']\n", " weighted_vectors = vectors_df.loc[index, 'weighted_vectors']\n", " reduced_vectors = vectors_df.loc[index, 'reduced_vectors']\n", " reduced_weighted_vectors = vectors_df.loc[index, 'reduced_weighted_vectors']\n", " non_zero_vectors, non_zero_uniques = unzip_fn([(vector, unique) for vector, unique in zip(vectors, unique_values) if vector > 0]) if len([i for i in vectors if i > 0]) > 0 else ([], [])\n", "\n", " non_zero_vectors__uniques = pearsonr(non_zero_vectors, non_zero_uniques) if len(non_zero_vectors) > 0 else [0,1]\n", " vectors___unique_values = pearsonr(vectors, unique_values)\n", " vectors___weighted_vectors = pearsonr(vectors, weighted_vectors)\n", " vectors___reduced_vectors = pearsonr(vectors, reduced_vectors)\n", " vectors___reduced_weighted_vectors = pearsonr(vectors, reduced_weighted_vectors)\n", " weighted_vectors___reduced_vectors = pearsonr(weighted_vectors, reduced_vectors)\n", " weighted_vectors___reduced_weighted_vectors = pearsonr(weighted_vectors, reduced_weighted_vectors)\n", " reduced_vectors___reduced_weighted_vectors = pearsonr(weighted_vectors, reduced_weighted_vectors)\n", "\n", " return {\n", " 'non_zero_vectors__uniques': non_zero_vectors__uniques,\n", " 'vectors___unique_values': vectors___unique_values,\n", " 'vectors___weighted_vectors': vectors___weighted_vectors,\n", " 'vectors___reduced_vectors': vectors___reduced_vectors,\n", " 'vectors___reduced_weighted_vectors': vectors___reduced_weighted_vectors,\n", " 'weighted_vectors___reduced_vectors': weighted_vectors___reduced_vectors,\n", " 'weighted_vectors___reduced_weighted_vectors': weighted_vectors___reduced_weighted_vectors,\n", " 'reduced_vectors___reduced_weighted_vectors': reduced_vectors___reduced_weighted_vectors,\n", " }\n", "\n", "from matplotlib import pyplot as plt\n", "from scipy.signal import convolve\n", "kernel = np.array([0.25, 0.5, 0.25]) # Example kernel for simple averaging\n", "\n", "def smooth_vector(vector):\n", " # Perform convolution\n", " smoothed_vector = convolve(vector, kernel, mode='same') / sum(kernel)\n", " return smoothed_vector\n", "\n", "def get_modified_stats(image_1_index: int, image_2_index: int, vector_column: str = 'vectors', plot = False):\n", " image_1_values = vectors_df.loc[image_1_index, vector_column]\n", " image_2_values = vectors_df.loc[image_2_index, vector_column]\n", "\n", " image_1_matrix = np.array(image_1_values)\n", " image_2_matrix = np.array(image_2_values)\n", "\n", " vector_1_zero_indices = image_1_matrix == 0\n", " vector_2_zero_indices = image_2_matrix == 0\n", "\n", " image_1_matrix[vector_1_zero_indices] = unique_values[vector_1_zero_indices]\n", " image_2_matrix[vector_2_zero_indices] = unique_values[vector_2_zero_indices]\n", "\n", " _old_pearsonr = pearsonr(image_1_values, image_2_values)\n", " [[_old_cosine_similarity]] = cosine_similarity([image_1_values], [image_2_values])\n", " _pearsonr = pearsonr(image_1_matrix, image_2_matrix)\n", " [[_cosine_similarity]] = cosine_similarity([image_1_matrix], [image_2_matrix])\n", "\n", " image_1_matrix_smooth = smooth_vector(image_1_matrix)\n", " image_2_matrix_smooth = smooth_vector(image_2_matrix)\n", " _pearsonr_smooth = pearsonr(image_1_matrix_smooth, image_2_matrix)\n", " [[_cosine_similarity_smooth]] = cosine_similarity([image_1_matrix_smooth], [image_2_matrix])\n", "\n", " permuted_indices = np.random.permutation(len(image_1_matrix))\n", " _pearsonr_random = pearsonr(image_1_matrix[permuted_indices], image_2_matrix[permuted_indices])\n", " [[_cosine_similarity_random]] = cosine_similarity([image_1_matrix[permuted_indices]], [image_2_matrix[permuted_indices]])\n", "\n", " if plot:\n", " plt.figure(figsize=(12, 6))\n", " plt.plot(image_1_values, label='image_1_values', color = 'red')\n", " plt.plot(image_1_matrix_smooth, label='image_1_matrix_smooth', color = 'blue')\n", " # plt.plot(image_1_matrix, label='image_1_matrix', linestyle='--', color = 'blue')\n", " # plt.plot(image_1_matrix_smooth, label='image_1_matrix_smooth', linestyle='--', color = \"green\")\n", " plt.show()\n", "\n", " return {\n", " 'old_pearsonr' : f'{round(_old_pearsonr.statistic, 4)} - {_old_pearsonr.pvalue}',\n", " 'old_cosine_similarity' : round(_old_cosine_similarity, 4),\n", " 'pearsonr' : f'{round(_pearsonr.statistic, 4)} - {_pearsonr.pvalue}',\n", " 'cosine_similarity' : round(_cosine_similarity, 4),\n", " 'pearsonr_smooth' : f'{round(_pearsonr_smooth.statistic, 4)} - {_pearsonr_smooth.pvalue}',\n", " 'cosine_similarity_smooth' : round(_cosine_similarity_smooth, 4),\n", " 'pearsonr_random' : f'{round(_pearsonr_random.statistic, 4)} - {_pearsonr_random.pvalue}',\n", " 'cosine_similarity_random' : round(_cosine_similarity_random, 4),\n", " }\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "get_stats(19569)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "correlation_results = []\n", "for i in tqdm.tqdm(range(len(correlation_results), len(vectors_df))):\n", " correlation_results.append(get_stats(i))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "columns = list(correlation_results[0].keys())\n", "fig, axes = plt.subplots(4, 2, figsize=(12, 12))\n", "axes = axes.flatten()\n", "for i, column in enumerate(columns):\n", " ax = axes[i]\n", " ax.hist([j[column][0] for j in correlation_results], bins=100)\n", " ax.set_title(column)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def correlation_fn(index: int):\n", " vectors = vectors_df.loc[index, 'vectors']\n", " weighted_vectors = vectors_df.loc[index, 'weighted_vectors']\n", " reduced_vectors = vectors_df.loc[index, 'reduced_vectors']\n", " reduced_weighted_vectors = vectors_df.loc[index, 'reduced_weighted_vectors']\n", " return {\n", " 'vectors vs weighted_vectors': pearsonr(vectors, weighted_vectors),\n", " 'vectors vs reduced_vectors': pearsonr(vectors, reduced_vectors),\n", " 'vectors vs reduced_weighted_vectors': pearsonr(vectors, reduced_weighted_vectors),\n", " 'weighted_vectors vs reduced_vectors': pearsonr(weighted_vectors, reduced_vectors),\n", " 'weighted_vectors vs reduced_weighted_vectors': pearsonr(weighted_vectors, reduced_weighted_vectors),\n", " 'reduced_vectors vs reduced_weighted_vectors': pearsonr(reduced_vectors, reduced_weighted_vectors),\n", " }\n", "\n", "correlation_results_2 = [correlation_fn(i) for i in tqdm.tqdm(range(len(vectors_df)))]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "columns = list(correlation_results_2[0].keys())\n", "fig, axes = plt.subplots(6, 2, figsize=(24, 24))\n", "axes = axes.flatten()\n", "for i, column in enumerate(columns):\n", " ax = axes[i]\n", " corr = [j[column][0] for j in correlation_results_2]\n", " pvalues = [j[column][1] for j in correlation_results_2]\n", " # ax.hist([j[column][0] for j in correlation_results_2], bins=100)\n", " ax.plot(range(0, len(corr)), corr, label='Correlation', color='blue')\n", " # ax.plot(range(0, len(pvalues)), pvalues, label='pvalues', color='red')\n", " ax.set_title(column)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "columns = list(correlation_results_2[0].keys())\n", "fig, axes = plt.subplots(3, 2, figsize=(24, 24))\n", "axes = axes.flatten()\n", "for i, column in enumerate(columns):\n", " ax = axes[i]\n", " corr = [j[column][0] for j in correlation_results_2]\n", " pvalues = [j[column][1] for j in correlation_results_2]\n", " ax.plot(range(0, len(corr)), corr, label='correlation', color='blue')\n", " ax.plot(range(0, len(pvalues)), pvalues, label='p-value', color='red')\n", " ax.legend(bbox_to_anchor=(1, 0.1), loc='lower right')\n", " ax.set_ylabel('correlation & p-value')\n", " ax.set_xlabel(f'images - {column}')\n", " ax.set_title(column)\n", "\n", "fig.savefig('/Users/charleskabue/Downloads/vector-correlations.png')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "