{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os,sys,inspect\n", "sys.path.insert(0,\"..\")\n", "\n", "import matplotlib.pyplot as plt\n", "from matplotlib import rc\n", "import glob\n", "\n", "macos = False\n", "if macos == True:\n", " rc('font',**{'family':'sans-serif','sans-serif':['Computer Modern Roman']})\n", " rc('text', usetex=True)\n", "\n", "# Font Size\n", "import matplotlib\n", "font = {'family' : 'DejaVu Sans',\n", " 'weight' : 'bold',\n", " 'size' : 30}\n", "\n", "import cv2\n", "import numpy as np\n", "import string\n", "import random" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def visualize(idx, **images):\n", " \"\"\"Plot images in one row.\"\"\" \n", " n = len(images)\n", " fig = plt.figure(figsize=(60, 40))\n", " for i, (name, image) in enumerate(images.items()):\n", " plt.subplot(1, n, i + 1)\n", " plt.xticks([])\n", " plt.yticks([])\n", " #if idx==0:\n", " plt.title(' '.join(name.split('_')).lower(), fontsize=40)\n", " if i ==0:\n", " w,h = (1,25)\n", " fs = 1.0\n", " color = (0,0,0)\n", " #color = (255,255,255)\n", " font = cv2.FONT_HERSHEY_SIMPLEX #FONT_HERSHEY_DUPLEX #press tab for different operations\n", " cv2.putText(image, str(idx), (w,h), font, fs, color, 1, cv2.LINE_AA)\n", " if i !=0:\n", " #plt.imshow(image[:,:,0], cmap='magma')\n", " plt.imshow(image, cmap='gray')\n", " else:\n", " plt.imshow(image, cmap='gray')\n", " plt.axis(\"off\")\n", " #plt.tight_layout()\n", " plt.savefig(\"../outputs/visualizations/duts-te/compare-preds/{}.png\".format(idx), facecolor=\"white\", bbox_inches = 'tight')\n", " plt.show()\n", " \n", " \n", "def make_dataset(dir):\n", " images = []\n", " assert os.path.isdir(dir), '%s is not a valid directory' % dir\n", "\n", " f = dir.split('/')[-1].split('_')[-1]\n", " #print (dir, f)\n", " dirs= os.listdir(dir)\n", " for img in dirs:\n", "\n", " path = os.path.join(dir, img)\n", " #print(path)\n", " images.append(path)\n", " return images\n", "\n", "# def make_dataset(dir):\n", "# images = []\n", "# assert os.path.isdir(dir), '%s is not a valid directory' % dir\n", "\n", "# # f = dir.split('/')[-1].split('_')[-1]\n", "# # #print (dir, f)\n", "# # dirs= os.listdir(dir)\n", "# # for img in dirs:\n", "\n", "# # path = os.path.join(dir, img)\n", "# # #print(path)\n", "# # images.append(path)\n", "# images = natsorted(glob.glob(dir+ \"/\" + \"/*.png\"))\n", "# return images\n", "\n", "def read_image(path):\n", " image = cv2.imread(path, -1)\n", " image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n", " image = Image.fromarray(np.uint8(image)).convert('RGB')\n", " image = resize_center_crop(image)\n", " image = make_border(image)\n", " return image\n", "\n", "\n", "def make_border(im):\n", " row, col = im.shape[:2]\n", " bottom = im[row-2:row, 0:col]\n", " mean = cv2.mean(bottom)[0]\n", " bordersize = 5\n", " border = cv2.copyMakeBorder(\n", " im,\n", " top=bordersize,\n", " bottom=bordersize,\n", " left=bordersize,\n", " right=bordersize,\n", " borderType=cv2.BORDER_CONSTANT,\n", " value=[0, 0, 0]\n", " )\n", " return border\n", "\n", "from PIL import Image\n", "from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize\n", "\n", "try:\n", " from torchvision.transforms import InterpolationMode\n", "\n", " BICUBIC = InterpolationMode.BICUBIC\n", "except ImportError:\n", " BICUBIC = Image.BICUBIC\n", "\n", "\n", "def _convert_image_to_rgb(image):\n", " return image.convert(\"RGB\")\n", "\n", "\n", "def resize_center_crop(img):\n", " \"\"\" \n", " Load and resize an image to a desired size.\n", "\n", " Arguments:\n", " img (PIL image): Image to load and resize\n", "\n", " Returns:\n", " img (np.array): Resized and cropped image\n", "\n", " Examples:\n", " >>> img = resize_center_crop(img)\n", " \"\"\"\n", "\n", " if type(img) == str:\n", " img = Image.open(img)\n", "\n", " transform = Compose(\n", " [\n", " Resize(224, BICUBIC),\n", " CenterCrop(224),\n", " _convert_image_to_rgb,\n", " # ToTensor(),\n", " # Normalize(\n", " # (0.5, 0.5, 0.5),\n", " # (0.5, 0.5, 0.5),\n", " # ),\n", " ]\n", " )\n", " img = transform(img)\n", " img = np.array(img)\n", " return img\n", "\n", "def read_image_(path):\n", " image = cv2.imread(path, -1)\n", " image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n", " image = cv2.resize(image, (192, 256))\n", " return image" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# # ECSSD\n", "\n", "# # Images and GT\n", "\n", "# GT = \"../outputs/visualizations/ecssd/gts\"\n", "# IMG = \"../datasets_local/ECSSD/images/\"\n", "# GTS = [os.path.join(GT, x) for x in os.listdir(GT)]\n", "# IMGS = [os.path.join(IMG, x) for x in os.listdir(IMG)]\n", "\n", "# # Algo\n", "# algo1 = \"../outputs/visualizations/ecssd/found-MSL-DUTS-TR-vit_small8_ECSSD/\"\n", "# ours = \"../outputs/visualizations/ecssd/msl_a1.5_b1_g1_reg4-MSL-DUTS-TR-vit_small8_ECSSD/\"\n", "\n", "# algo1 = [os.path.join(algo1, x) for x in os.listdir(algo1)]\n", "# ours = [os.path.join(ours, x) for x in os.listdir(ours)]\n", "\n", "# print(len(GTS), len(IMGS))\n", "# print(ours[:3])\n", "\n", "# i = 0\n", "# for num in range(len(IMGS)):\n", "# visualize(i, \n", "# image=read_image(IMGS[num]),\n", "# found_method=read_image(algo1[num]),\n", "# our_method=read_image(ours[num]),\n", "# gt=read_image(GTS[num]))\n", "# i+=1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# # DUT_OMRON\n", "\n", "# # Images and GT\n", "\n", "# GT = \"../outputs/visualizations/dut-omron/gts\"\n", "# IMG = \"../datasets_local/DUT-OMRON/DUT-OMRON-image/\"\n", "# GTS = [os.path.join(GT, x) for x in os.listdir(GT)]\n", "# IMGS = [os.path.join(IMG, x) for x in os.listdir(IMG)]\n", "\n", "# # Algo\n", "# algo1 = \"../outputs/visualizations/dut-omron/found-MSL-DUTS-TR-vit_small8_DUT-OMRON/\"\n", "# ours = \"../outputs/visualizations/dut-omron/msl_a1.5_b1_g1_reg4-MSL-DUTS-TR-vit_small8_DUT-OMRON/\"\n", "\n", "# algo1 = [os.path.join(algo1, x) for x in os.listdir(algo1)]\n", "# ours = [os.path.join(ours, x) for x in os.listdir(ours)]\n", "\n", "# print(len(GTS), len(IMGS))\n", "# print(ours[:3])\n", "\n", "# i = 0\n", "# for num in range(len(IMGS)):\n", "# visualize(i, \n", "# image=read_image(IMGS[num]),\n", "# found_method=read_image(algo1[num]),\n", "# our_method=read_image(ours[num]),\n", "# gt=read_image(GTS[num]))\n", "# i+=1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# # DUT-TE\n", "\n", "# # Images and GT\n", "\n", "# GT = \"../outputs/visualizations/duts-te/gts\"\n", "# IMG = \"../datasets_local/DUTS-TE/DUTS-TE-Image/\"\n", "# GTS = [os.path.join(GT, x) for x in os.listdir(GT)]\n", "# IMGS = [os.path.join(IMG, x) for x in os.listdir(IMG)]\n", "\n", "# # Algo\n", "# algo1 = \"../outputs/visualizations/duts-te/found-MSL-DUTS-TR-vit_small8_DUTS-TE/\"\n", "# ours = \"../outputs/visualizations/duts-te/msl_a1.5_b1_g1_reg4-MSL-DUTS-TR-vit_small8_DUTS-TE/\"\n", "\n", "# algo1 = [os.path.join(algo1, x) for x in os.listdir(algo1)]\n", "# ours = [os.path.join(ours, x) for x in os.listdir(ours)]\n", "\n", "# print(len(GTS), len(IMGS))\n", "# print(ours[:3])\n", "\n", "# i = 0\n", "# for num in range(len(IMGS)):\n", "# visualize(i, \n", "# image=read_image(IMGS[num]),\n", "# found_method=read_image(algo1[num]),\n", "# our_method=read_image(ours[num]),\n", "# gt=read_image(GTS[num]))\n", "# i+=1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# GT\n", "ECSS_GT = \"../outputs/visualizations/ecssd/gts\"\n", "ECSS_IMG = \"../datasets_local/ECSSD/images/\"\n", "ECSS_GTS = [os.path.join(ECSS_GT, x) for x in os.listdir(ECSS_GT)]\n", "ECSS_IMGS = [os.path.join(ECSS_IMG, x) for x in os.listdir(ECSS_IMG)]\n", "# Pred\n", "ECSS_algo1 = \"../outputs/visualizations/ecssd/found-MSL-DUTS-TR-vit_small8_ECSSD/\"\n", "ECSS_ours = \"../outputs/visualizations/ecssd/msl_a1.5_b1_g1_reg4-MSL-DUTS-TR-vit_small8_ECSSD/\"\n", "ECSS_algo1 = [os.path.join(ECSS_algo1, x) for x in os.listdir(ECSS_algo1)]\n", "ECSS_ours = [os.path.join(ECSS_ours, x) for x in os.listdir(ECSS_ours)]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# GT\n", "DUT_OM_GT = \"../outputs/visualizations/dut-omron/gts\"\n", "DUT_OM_IMG = \"../datasets_local/DUT-OMRON/DUT-OMRON-image/\"\n", "DUT_OM_GTS = [os.path.join(DUT_OM_GT, x) for x in os.listdir(DUT_OM_GT)]\n", "DUT_OM_IMGS = [os.path.join(DUT_OM_IMG, x) for x in os.listdir(DUT_OM_IMG)]\n", "\n", "# Pred\n", "DUT_OM_algo1 = \"../outputs/visualizations/dut-omron/found-MSL-DUTS-TR-vit_small8_DUT-OMRON/\"\n", "DUT_OM_ours = \"../outputs/visualizations/dut-omron/msl_a1.5_b1_g1_reg4-MSL-DUTS-TR-vit_small8_DUT-OMRON/\"\n", "DUT_OM_algo1 = [os.path.join(DUT_OM_algo1, x) for x in os.listdir(DUT_OM_algo1)]\n", "DUT_OM_ours = [os.path.join(DUT_OM_ours, x) for x in os.listdir(DUT_OM_ours)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "DUT_GT = \"../outputs/visualizations/duts-te/gts\"\n", "DUT_IMG = \"../datasets_local/DUTS-TE/DUTS-TE-Image/\"\n", "DUT_GTS = [os.path.join(DUT_GT, x) for x in os.listdir(DUT_GT)]\n", "DUT_IMGS = [os.path.join(DUT_IMG, x) for x in os.listdir(DUT_IMG)]\n", "\n", "# Pred\n", "DUT_algo1 = \"../outputs/visualizations/duts-te/found-MSL-DUTS-TR-vit_small8_DUTS-TE/\"\n", "DUT_ours = \"../outputs/visualizations/duts-te/msl_a1.5_b1_g1_reg4-MSL-DUTS-TR-vit_small8_DUTS-TE/\"\n", "DUT_algo1 = [os.path.join(DUT_algo1, x) for x in os.listdir(DUT_algo1)]\n", "DUT_ours = [os.path.join(DUT_ours, x) for x in os.listdir(DUT_ours)]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "# ECSSD -\n", "# \t52, 132, 147 - over segmentation, fine details\n", "# \t353, 658, 780 - reflection of shiny surface and water\n", "# 432, 825, 835, 988 - noisy \n", "# 59 (bee) - complex background\n", "\n", "# DUT-OMRON\n", "# \t1, 14 - over segmentation\n", "# \t119, 365, 439, 440, 1238 - noisy\n", "# 1168, 1461 - segment other non-salient objects/parts\n", "# 1388 - fails in complex background\n", "# 1398 - small objects\n", "# 1973 - dark scenes\n", "\n", "# DUTS-TE\n", "# \t46, 698, 1712 - segment other non-salient objects/parts\n", "# \t260 - small objects \n", "# 776, 1255 - over segmentation\n", "# \t683, 830, 1465 - noisy\n", "# \t719, 1470 - reflection of water\n", "\n", "# 52, 132, 147, 353, 658, 780, - oversegment, reflection of shiny surface and water\n", "# 1388, 1398, 1972, 1168, 1461, 440 - fails in complex background, small objects, dark scenes, segment non-salient objects, noisy\n", "# 260, 719, 1470, 683, 830, 1465 - small objects, reflection of water, noisy predictions\n", "\n", "# idxs = [52, 59, 147, 353, 658, 780, 1388, 1398, 1973, 1168, 1461, 440, 260, 719, 1470, 683, 830, 1465]\n", "\n", "\n", "\n", "\n", "# ECSSD -\n", "# , 132, - over segmentation, fine details\n", "# ,, - reflection of shiny surface and water\n", "# 432, 825, 835, 988 - noisy \n", "# 59 (bee) - complex background\n", "\n", "# DUT-OMRON\n", "# \t1, 14 - over segmentation\n", "# \t119, 365, 439,, 1238 - noisy\n", "#, - segment other non-salient objects/parts\n", "# - fails in complex background\n", "# - small objects\n", "# - dark scenes\n", "\n", "# DUTS-TE\n", "# \t46, 698, 1712 - segment other non-salient objects/parts\n", "# - small objects \n", "# 776, 1255 - over segmentation\n", "# ,, - noisy\n", "# , - reflection of water\n", "\n", "idxs = [132,432,825,835,988,59,1,14,119,365,439,1238,46,698,1712,776,1255,4000]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "rows = int(len(idxs) / 3)\n", "rows, len(idxs)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "rows = int(len(idxs) / 3)\n", "cols = 12\n", "fig, axarr = plt.subplots(rows, cols, figsize=(30, 15), constrained_layout=True)\n", "\n", "\n", "alphabet_string = string.ascii_lowercase\n", "alphabet_list = list(alphabet_string)\n", "\n", "v = 0\n", "for r in range(rows):\n", " if r == 0 or r == 1:\n", " print(v, r)\n", " a=read_image(ECSS_IMGS[idxs[v+r]])\n", " b=read_image(ECSS_algo1[idxs[v+r]])\n", "\n", " c=read_image(ECSS_ours[idxs[v+r]])\n", " d=read_image(ECSS_GTS[idxs[v+r]])\n", "\n", " e=read_image(ECSS_IMGS[idxs[v+r+1]])\n", " f=read_image(ECSS_algo1[idxs[v+r+1]])\n", "\n", " g=read_image(ECSS_ours[idxs[v+r+1]])\n", " h=read_image(ECSS_GTS[idxs[v+r+1]])\n", "\n", " i=read_image(ECSS_IMGS[idxs[v+r+2]])\n", " j=read_image(ECSS_algo1[idxs[v+r+2]])\n", "\n", " k=read_image(ECSS_ours[idxs[v+r+2]])\n", " l=read_image(ECSS_GTS[idxs[v+r+2]])\n", "\n", " if r == 2 or r == 3:\n", " print(v, r)\n", " a=read_image(DUT_OM_IMGS[idxs[v+r]])\n", " b=read_image(DUT_OM_algo1[idxs[v+r]])\n", "\n", " c=read_image(DUT_OM_ours[idxs[v+r]])\n", " d=read_image(DUT_OM_GTS[idxs[v+r]])\n", "\n", " e=read_image(DUT_OM_IMGS[idxs[v+r+1]])\n", " f=read_image(DUT_OM_algo1[idxs[v+r+1]])\n", "\n", " g=read_image(DUT_OM_ours[idxs[v+r+1]])\n", " h=read_image(DUT_OM_GTS[idxs[v+r+1]])\n", "\n", " i=read_image(DUT_OM_IMGS[idxs[v+r+2]])\n", " j=read_image(DUT_OM_algo1[idxs[v+r+2]])\n", "\n", " k=read_image(DUT_OM_ours[idxs[v+r+2]])\n", " l=read_image(DUT_OM_algo1[idxs[v+r+2]])\n", "\n", " if r == 4 or r == 5:\n", " print(v, r)\n", " a=read_image(DUT_IMGS[idxs[v+r]])\n", " b=read_image(DUT_algo1[idxs[v+r]])\n", "\n", " c=read_image(DUT_ours[idxs[v+r]])\n", " d=read_image(DUT_GTS[idxs[v+r]])\n", "\n", " e=read_image(DUT_IMGS[idxs[v+r+1]])\n", " f=read_image(DUT_algo1[idxs[v+r+1]])\n", "\n", " g=read_image(DUT_ours[idxs[v+r+1]])\n", " h=read_image(DUT_GTS[idxs[v+r+1]])\n", "\n", " i=read_image(DUT_IMGS[idxs[v+r+2]])\n", " j=read_image(DUT_algo1[idxs[v+r+2]])\n", "\n", " k=read_image(DUT_ours[idxs[v+r+2]])\n", " l=read_image(DUT_GTS[idxs[v+r+2]])\n", "\n", " v+=2\n", " \n", " images = [a,b,c,d,e,f,g,h,i,j,k,l]\n", " \n", " captions = [\"Image\", \"FOUND\", \"Ours\", \"Ground Truth\", \n", " \"Image\", \"FOUND\", \"Ours\", \"Ground Truth\",\n", " \"Image\", \"FOUND\", \"Ours\", \"Ground Truth\"]\n", " \n", " for c in range(cols):\n", " axarr[r, c].imshow(images[c], cmap='gray')\n", " axarr[r, c].axis(\"off\")\n", " axarr[r, c].set_aspect('equal') \n", " if r==0:\n", " axarr[r, c].set_title(captions[c], fontsize=25)\n", "\n", "plt.savefig(\"../logs/compare_predictions_ext.pdf\", facecolor=\"white\", bbox_inches = 'tight', dpi=300)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stacks = np.hstack([a,b,c_])\n", "stacks.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "plt.imshow(stacks)\n", "plt.axis(\"off\")\n", "plt.savefig(\"../logs/failures.pdf\", facecolor=\"white\", bbox_inches = 'tight', dpi=300)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "a.shape, b.shape, c.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "uobjl", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.18" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }