{ "cells": [ { "cell_type": "code", "execution_count": 7, "id": "87d59ec1", "metadata": {}, "outputs": [], "source": [ "from duckduckgo_search import ddg_images\n", "from fastcore.all import *\n", "def search_images(term, max_images=30):\n", " print(f\"Searching for '{term}'\")\n", " return L(ddg_images(term, max_results=max_images)).itemgot('image')" ] }, { "cell_type": "code", "execution_count": 9, "id": "6ac770e8", "metadata": {}, "outputs": [], "source": [ "from fastdownload import download_url\n", "#from fastai.vision.all import *" ] }, { "cell_type": "code", "execution_count": 10, "id": "4e36b9c9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Searching for 'donut photo'\n", "Searching for 'donut sun photo'\n", "Searching for 'donut shade photo'\n", "Searching for 'scone photo'\n", "Searching for 'scone sun photo'\n", "Searching for 'scone shade photo'\n" ] } ], "source": [ "searches = 'donut','scone'\n", "path = Path('donut_or_not')\n", "from time import sleep\n", "\n", "for o in searches:\n", " dest = (path/o)\n", " dest.mkdir(exist_ok=True, parents=True)\n", " download_images(dest, urls=search_images(f'{o} photo'))\n", " sleep(10) # Pause between searches to avoid over-loading server\n", " download_images(dest, urls=search_images(f'{o} sun photo'))\n", " sleep(10)\n", " download_images(dest, urls=search_images(f'{o} shade photo'))\n", " sleep(10)\n", " resize_images(path/o, max_size=400, dest=path/o)" ] }, { "cell_type": "code", "execution_count": 11, "id": "d51b4218", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "3" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "failed = verify_images(get_image_files(path))\n", "failed\n", "failed.map(Path.unlink)\n", "len(failed)" ] }, { "cell_type": "code", "execution_count": 12, "id": "0cfa1c6d", "metadata": {}, "outputs": [], "source": [ "dls = DataBlock(\n", " blocks=(ImageBlock, CategoryBlock), \n", " get_items=get_image_files, \n", " splitter=RandomSplitter(valid_pct=0.2, seed=42),\n", " get_y=parent_label,\n", " item_tfms=[Resize(192, method='squish')]\n", ").dataloaders(path, bs=32)\n" ] }, { "cell_type": "code", "execution_count": 13, "id": "f6828f44", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.\n", " warnings.warn(\n", "/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n", " warnings.warn(msg)\n", "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /home/yigewu/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "0d149e2258a546869d7c7c9a06442933", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0.00/44.7M [00:00\n", " /* Turns off some styling */\n", " progress {\n", " /* gets rid of default border in Firefox and Opera. */\n", " border: none;\n", " /* Needs to be in here for Safari polyfill so background images work as expected. */\n", " background-size: auto;\n", " }\n", " progress:not([value]), progress:not([value])::-webkit-progress-bar {\n", " background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n", " }\n", " .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n", " background: #F44336;\n", " }\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_losserror_ratetime
01.1052290.3408210.17857100:23
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_losserror_ratetime
00.2916100.0794220.03571400:31
10.1995870.0404300.01785700:34
20.1410430.0374860.00000000:32
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "learn = vision_learner(dls, resnet18, metrics=error_rate)\n", "learn.fine_tune(3)" ] }, { "cell_type": "code", "execution_count": 14, "id": "b80d9503", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "16920438763447d7babcf088ec67ce8e", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(Dropdown(options=('donut', 'scone'), value='donut'), Dropdown(options=('Train', 'Valid'), value…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from fastai.vision.widgets import *\n", "cleaner = ImageClassifierCleaner(learn)\n", "cleaner" ] }, { "cell_type": "code", "execution_count": 15, "id": "f4f497c3", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_losserror_ratetime
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "ename": "FileNotFoundError", "evalue": "Caught FileNotFoundError in DataLoader worker process 0.\nOriginal Traceback (most recent call last):\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py\", line 302, in _worker_loop\n data = fetcher.fetch(index)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py\", line 39, in fetch\n data = next(self.dataset_iter)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/load.py\", line 140, in create_batches\n yield from map(self.do_batch, self.chunkify(res))\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/basics.py\", line 230, in chunked\n res = list(itertools.islice(it, chunk_sz))\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/load.py\", line 155, in do_item\n try: return self.after_item(self.create_item(s))\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/load.py\", line 162, in create_item\n if self.indexed: return self.dataset[s or 0]\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/core.py\", line 455, in __getitem__\n res = tuple([tl[it] for tl in self.tls])\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/core.py\", line 455, in \n res = tuple([tl[it] for tl in self.tls])\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/core.py\", line 414, in __getitem__\n return self._after_item(res) if is_indexer(idx) else res.map(self._after_item)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/core.py\", line 374, in _after_item\n def _after_item(self, o): return self.tfms(o)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 208, in __call__\n def __call__(self, o): return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 158, in compose_tfms\n x = f(x, **kwargs)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 81, in __call__\n def __call__(self, x, **kwargs): return self._call('encodes', x, **kwargs)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 91, in _call\n return self._do_call(getattr(self, fn), x, **kwargs)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 97, in _do_call\n return retain_type(f(x, **kwargs), x, ret)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/dispatch.py\", line 120, in __call__\n return f(*args, **kwargs)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/vision/core.py\", line 123, in create\n return cls(load_image(fn, **merge(cls._open_args, kwargs)))\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/vision/core.py\", line 98, in load_image\n im = Image.open(fn)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/PIL/Image.py\", line 3092, in open\n fp = builtins.open(filename, \"rb\")\nFileNotFoundError: [Errno 2] No such file or directory: '/diskmnt/Projects/Users/yigewu/Deep_Learning2022/pet_test/donut_or_not/scone/24f75cc7-6e8a-44e3-80bc-8c5132d2ec30.jpg'\n", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn [15], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m cleaner\u001b[38;5;241m.\u001b[39mdelete(): cleaner\u001b[38;5;241m.\u001b[39mfns[idx]\u001b[38;5;241m.\u001b[39munlink()\n\u001b[0;32m----> 2\u001b[0m \u001b[43mlearn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfine_tune\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/callback/schedule.py:165\u001b[0m, in \u001b[0;36mfine_tune\u001b[0;34m(self, epochs, base_lr, freeze_epochs, lr_mult, pct_start, div, **kwargs)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFine tune with `Learner.freeze` for `freeze_epochs`, then with `Learner.unfreeze` for `epochs`, using discriminative LR.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfreeze()\n\u001b[0;32m--> 165\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_one_cycle\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfreeze_epochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mslice\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mbase_lr\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpct_start\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.99\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 166\u001b[0m base_lr \u001b[38;5;241m/\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m2\u001b[39m\n\u001b[1;32m 167\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39munfreeze()\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/callback/schedule.py:119\u001b[0m, in \u001b[0;36mfit_one_cycle\u001b[0;34m(self, n_epoch, lr_max, div, div_final, pct_start, wd, moms, cbs, reset_opt, start_epoch)\u001b[0m\n\u001b[1;32m 116\u001b[0m lr_max \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marray([h[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlr\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m h \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mopt\u001b[38;5;241m.\u001b[39mhypers])\n\u001b[1;32m 117\u001b[0m scheds \u001b[38;5;241m=\u001b[39m {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlr\u001b[39m\u001b[38;5;124m'\u001b[39m: combined_cos(pct_start, lr_max\u001b[38;5;241m/\u001b[39mdiv, lr_max, lr_max\u001b[38;5;241m/\u001b[39mdiv_final),\n\u001b[1;32m 118\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmom\u001b[39m\u001b[38;5;124m'\u001b[39m: combined_cos(pct_start, \u001b[38;5;241m*\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmoms \u001b[38;5;28;01mif\u001b[39;00m moms \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m moms))}\n\u001b[0;32m--> 119\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn_epoch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcbs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mParamScheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscheds\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43mL\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcbs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreset_opt\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreset_opt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwd\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mwd\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstart_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstart_epoch\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/learner.py:256\u001b[0m, in \u001b[0;36mLearner.fit\u001b[0;34m(self, n_epoch, lr, wd, cbs, reset_opt, start_epoch)\u001b[0m\n\u001b[1;32m 254\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mopt\u001b[38;5;241m.\u001b[39mset_hypers(lr\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlr \u001b[38;5;28;01mif\u001b[39;00m lr \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m lr)\n\u001b[1;32m 255\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_epoch \u001b[38;5;241m=\u001b[39m n_epoch\n\u001b[0;32m--> 256\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_with_events\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_do_fit\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mfit\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mCancelFitException\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_end_cleanup\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/learner.py:193\u001b[0m, in \u001b[0;36mLearner._with_events\u001b[0;34m(self, f, event_type, ex, final)\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_with_events\u001b[39m(\u001b[38;5;28mself\u001b[39m, f, event_type, ex, final\u001b[38;5;241m=\u001b[39mnoop):\n\u001b[0;32m--> 193\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m: \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbefore_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m); \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ex: \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_cancel_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 195\u001b[0m \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m); final()\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/learner.py:245\u001b[0m, in \u001b[0;36mLearner._do_fit\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 243\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_epoch):\n\u001b[1;32m 244\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepoch\u001b[38;5;241m=\u001b[39mepoch\n\u001b[0;32m--> 245\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_with_events\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_do_epoch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mepoch\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mCancelEpochException\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/learner.py:193\u001b[0m, in \u001b[0;36mLearner._with_events\u001b[0;34m(self, f, event_type, ex, final)\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_with_events\u001b[39m(\u001b[38;5;28mself\u001b[39m, f, event_type, ex, final\u001b[38;5;241m=\u001b[39mnoop):\n\u001b[0;32m--> 193\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m: \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbefore_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m); \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ex: \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_cancel_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 195\u001b[0m \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m); final()\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/learner.py:240\u001b[0m, in \u001b[0;36mLearner._do_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 238\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_do_epoch\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 239\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_do_epoch_train()\n\u001b[0;32m--> 240\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_do_epoch_validate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/learner.py:236\u001b[0m, in \u001b[0;36mLearner._do_epoch_validate\u001b[0;34m(self, ds_idx, dl)\u001b[0m\n\u001b[1;32m 234\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m dl \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m: dl \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdls[ds_idx]\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdl \u001b[38;5;241m=\u001b[39m dl\n\u001b[0;32m--> 236\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad(): \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_with_events\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mall_batches\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mvalidate\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mCancelValidException\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/learner.py:193\u001b[0m, in \u001b[0;36mLearner._with_events\u001b[0;34m(self, f, event_type, ex, final)\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_with_events\u001b[39m(\u001b[38;5;28mself\u001b[39m, f, event_type, ex, final\u001b[38;5;241m=\u001b[39mnoop):\n\u001b[0;32m--> 193\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m: \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbefore_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m); \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ex: \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_cancel_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 195\u001b[0m \u001b[38;5;28mself\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevent_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m); final()\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/learner.py:199\u001b[0m, in \u001b[0;36mLearner.all_batches\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mall_batches\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 198\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_iter \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdl)\n\u001b[0;32m--> 199\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m o \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdl): \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mone_batch(\u001b[38;5;241m*\u001b[39mo)\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/load.py:129\u001b[0m, in \u001b[0;36mDataLoader.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbefore_iter()\n\u001b[1;32m 128\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__idxs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_idxs() \u001b[38;5;66;03m# called in context of main process (not workers/subprocesses)\u001b[39;00m\n\u001b[0;32m--> 129\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m b \u001b[38;5;129;01min\u001b[39;00m _loaders[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfake_l\u001b[38;5;241m.\u001b[39mnum_workers\u001b[38;5;241m==\u001b[39m\u001b[38;5;241m0\u001b[39m](\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfake_l):\n\u001b[1;32m 130\u001b[0m \u001b[38;5;66;03m# pin_memory causes tuples to be converted to lists, so convert them back to tuples\u001b[39;00m\n\u001b[1;32m 131\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpin_memory \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(b) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mlist\u001b[39m: b \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtuple\u001b[39m(b)\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m: b \u001b[38;5;241m=\u001b[39m to_device(b, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torch/utils/data/dataloader.py:681\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 678\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 679\u001b[0m \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m 680\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset() \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 681\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 682\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 683\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m 684\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m 685\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called:\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torch/utils/data/dataloader.py:1376\u001b[0m, in \u001b[0;36m_MultiProcessingDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1374\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1375\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_task_info[idx]\n\u001b[0;32m-> 1376\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torch/utils/data/dataloader.py:1402\u001b[0m, in \u001b[0;36m_MultiProcessingDataLoaderIter._process_data\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 1400\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_try_put_index()\n\u001b[1;32m 1401\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(data, ExceptionWrapper):\n\u001b[0;32m-> 1402\u001b[0m \u001b[43mdata\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreraise\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1403\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m data\n", "File \u001b[0;32m/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torch/_utils.py:461\u001b[0m, in \u001b[0;36mExceptionWrapper.reraise\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 457\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[1;32m 458\u001b[0m \u001b[38;5;66;03m# If the exception takes multiple arguments, don't try to\u001b[39;00m\n\u001b[1;32m 459\u001b[0m \u001b[38;5;66;03m# instantiate since we don't know how to\u001b[39;00m\n\u001b[1;32m 460\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(msg) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[0;32m--> 461\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exception\n", "\u001b[0;31mFileNotFoundError\u001b[0m: Caught FileNotFoundError in DataLoader worker process 0.\nOriginal Traceback (most recent call last):\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py\", line 302, in _worker_loop\n data = fetcher.fetch(index)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py\", line 39, in fetch\n data = next(self.dataset_iter)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/load.py\", line 140, in create_batches\n yield from map(self.do_batch, self.chunkify(res))\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/basics.py\", line 230, in chunked\n res = list(itertools.islice(it, chunk_sz))\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/load.py\", line 155, in do_item\n try: return self.after_item(self.create_item(s))\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/load.py\", line 162, in create_item\n if self.indexed: return self.dataset[s or 0]\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/core.py\", line 455, in __getitem__\n res = tuple([tl[it] for tl in self.tls])\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/core.py\", line 455, in \n res = tuple([tl[it] for tl in self.tls])\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/core.py\", line 414, in __getitem__\n return self._after_item(res) if is_indexer(idx) else res.map(self._after_item)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/data/core.py\", line 374, in _after_item\n def _after_item(self, o): return self.tfms(o)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 208, in __call__\n def __call__(self, o): return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 158, in compose_tfms\n x = f(x, **kwargs)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 81, in __call__\n def __call__(self, x, **kwargs): return self._call('encodes', x, **kwargs)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 91, in _call\n return self._do_call(getattr(self, fn), x, **kwargs)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/transform.py\", line 97, in _do_call\n return retain_type(f(x, **kwargs), x, ret)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastcore/dispatch.py\", line 120, in __call__\n return f(*args, **kwargs)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/vision/core.py\", line 123, in create\n return cls(load_image(fn, **merge(cls._open_args, kwargs)))\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/fastai/vision/core.py\", line 98, in load_image\n im = Image.open(fn)\n File \"/diskmnt/Projects/Users/yigewu/tools/miniconda3/envs/deep_learning/lib/python3.10/site-packages/PIL/Image.py\", line 3092, in open\n fp = builtins.open(filename, \"rb\")\nFileNotFoundError: [Errno 2] No such file or directory: '/diskmnt/Projects/Users/yigewu/Deep_Learning2022/pet_test/donut_or_not/scone/24f75cc7-6e8a-44e3-80bc-8c5132d2ec30.jpg'\n" ] } ], "source": [ "for idx in cleaner.delete(): cleaner.fns[idx].unlink()" ] }, { "cell_type": "code", "execution_count": 16, "id": "6cf656c1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This is a: donut.\n", "Probability it's a donut: 0.9999\n" ] } ], "source": [ "is_donut,_,probs = learn.predict(PILImage.create('donut.jpg'))\n", "print(f\"This is a: {is_donut}.\")\n", "print(f\"Probability it's a donut: {probs[0]:.4f}\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "c6e7eb32", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "This is a: scone.\n", "Probability it's a donut: 0.0000\n" ] } ], "source": [ "is_donut,_,probs = learn.predict(PILImage.create('scone.jpg'))\n", "print(f\"This is a: {is_donut}.\")\n", "print(f\"Probability it's a donut: {probs[0]:.4f}\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "f6e944f8", "metadata": {}, "outputs": [], "source": [ "learn.export('donut_model2.pkl')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6" } }, "nbformat": 4, "nbformat_minor": 5 }