{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/home/shrey/Desktop/Kidney-Disease-Classifcation/research'" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%pwd" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "os.chdir(\"../\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/home/shrey/Desktop/Kidney-Disease-Classifcation'" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%pwd" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "from dataclasses import dataclass\n", "from pathlib import Path\n", "\n", "\n", "@dataclass(frozen=True)\n", "class TrainingConfig:\n", " root_dir: Path\n", " trained_model_path: Path\n", " updated_base_model_path: Path\n", " training_data: Path\n", " params_epochs: int\n", " params_batch_size: int\n", " params_image_size: list" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "os.chdir(\"./src\")" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2024-01-03 15:58:59.001906: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", "2024-01-03 15:58:59.003769: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n", "2024-01-03 15:58:59.032703: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "2024-01-03 15:58:59.032753: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "2024-01-03 15:58:59.033922: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "2024-01-03 15:58:59.038705: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n", "2024-01-03 15:58:59.039146: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", "2024-01-03 15:58:59.655806: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], "source": [ "from kidney_classification.constants import *\n", "from kidney_classification.utils.common import read_yaml, create_directories\n", "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "class ConfigurationManager:\n", " def __init__(\n", " self,\n", " config_filepath = CONFIG_FILE_PATH,\n", " params_filepath = PARAMS_FILE_PATH):\n", "\n", " self.config = read_yaml(config_filepath)\n", " self.params = read_yaml(params_filepath)\n", "\n", " create_directories([self.config.artifacts_root])\n", "\n", "\n", " \n", " def get_training_config(self) -> TrainingConfig:\n", " training = self.config.training\n", " prepare_base_model = self.config.prepare_base_model\n", " params = self.params\n", " training_data = os.path.join(self.config.data_ingestion.unzip_dir, \"CT-KIDNEY-DATASET-Normal-Cyst-Tumor-Stone\")\n", " create_directories([\n", " Path(training.root_dir)\n", " ])\n", "\n", " training_config = TrainingConfig(\n", " root_dir=Path(training.root_dir),\n", " trained_model_path=Path(training.trained_model_path),\n", " updated_base_model_path=Path(prepare_base_model.updated_base_model_path),\n", " training_data=Path(training_data),\n", " params_epochs=params.EPOCHS,\n", " params_batch_size=params.BATCH_SIZE,\n", " params_image_size=params.IMAGE_SIZE\n", " )\n", "\n", " return training_config" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "class Training:\n", " def __init__(self, config: TrainingConfig):\n", " self.config = config\n", "\n", " \n", " def get_base_model(self):\n", " self.model = tf.keras.models.load_model(\n", " self.config.updated_base_model_path\n", " )\n", "\n", " def train_valid_generator(self):\n", " img_height, img_width = self.config.params_image_size[:-1]\n", "\n", " train = tf.keras.utils.image_dataset_from_directory(\n", " self.config.training_data,\n", " image_size=(img_height, img_width),\n", " validation_split=0.1,\n", " subset='training',\n", " seed=123\n", " )\n", " \n", " val = tf.keras.utils.image_dataset_from_directory(\n", " self.config.training_data,\n", " image_size=(img_height, img_width),\n", " validation_split=0.2,\n", " subset='validation',\n", " seed=123\n", " )\n", " train = train.map(lambda x, y: (x / 255, y))\n", " val = val.map(lambda x, y: (x / 255, y))\n", " AUTOTUNE = tf.data.AUTOTUNE\n", "\n", " self.train_dataset = train.cache().prefetch(buffer_size=AUTOTUNE)\n", " self.val_dataset = val.cache().prefetch(buffer_size=AUTOTUNE)\n", "\n", " \n", " @staticmethod\n", " def save_model(path: Path, model: tf.keras.Model):\n", " model.save(path)\n", "\n", " \n", " def define_and_train_model(self):\n", "\n", " self.model.fit(\n", " self.train_dataset,\n", " validation_data=self.val_dataset,\n", " epochs=self.config.params_epochs,\n", " )\n", "\n", " self.save_model(\n", " path=self.config.trained_model_path,\n", " model=self.model\n", " )" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "os.chdir(\"../\")" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2024-01-03 15:59:11,299: INFO: common yaml file: config/config.yaml loaded successfully]\n", "[2024-01-03 15:59:11,301: INFO: common yaml file: params.yaml loaded successfully]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2024-01-03 15:59:11.602362: E external/local_xla/xla/stream_executor/cuda/cuda_driver.cc:274] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[2024-01-03 15:59:12,070: WARNING: optimizer Skipping variable loading for optimizer 'Adam', because it has 13 variables whereas the saved optimizer has 1 variables. ]\n", "Found 12446 files belonging to 4 classes.\n", "Using 11202 files for training.\n", "Found 12446 files belonging to 4 classes.\n", "Using 2489 files for validation.\n", "Epoch 1/15\n", "351/351 [==============================] - 561s 2s/step - loss: 0.5099 - accuracy: 0.8103 - val_loss: 0.2366 - val_accuracy: 0.9152\n", "Epoch 2/15\n", "351/351 [==============================] - 566s 2s/step - loss: 0.2120 - accuracy: 0.9229 - val_loss: 0.2941 - val_accuracy: 0.8803\n", "Epoch 3/15\n", "351/351 [==============================] - 571s 2s/step - loss: 0.1573 - accuracy: 0.9466 - val_loss: 0.0994 - val_accuracy: 0.9667\n", "Epoch 4/15\n", "351/351 [==============================] - 547s 2s/step - loss: 0.1370 - accuracy: 0.9528 - val_loss: 0.0742 - val_accuracy: 0.9787\n", "Epoch 5/15\n", "351/351 [==============================] - 552s 2s/step - loss: 0.1217 - accuracy: 0.9565 - val_loss: 0.0564 - val_accuracy: 0.9835\n", "Epoch 6/15\n", "351/351 [==============================] - 550s 2s/step - loss: 0.1064 - accuracy: 0.9621 - val_loss: 0.0626 - val_accuracy: 0.9791\n", "Epoch 7/15\n", "351/351 [==============================] - 550s 2s/step - loss: 0.1028 - accuracy: 0.9622 - val_loss: 0.0845 - val_accuracy: 0.9658\n", "Epoch 8/15\n", "351/351 [==============================] - 551s 2s/step - loss: 0.0953 - accuracy: 0.9655 - val_loss: 0.0527 - val_accuracy: 0.9811\n", "Epoch 9/15\n", "351/351 [==============================] - 526s 2s/step - loss: 0.0900 - accuracy: 0.9692 - val_loss: 0.0698 - val_accuracy: 0.9731\n", "Epoch 10/15\n", "351/351 [==============================] - 526s 1s/step - loss: 0.0806 - accuracy: 0.9708 - val_loss: 0.0454 - val_accuracy: 0.9867\n", "Epoch 11/15\n", "351/351 [==============================] - 529s 2s/step - loss: 0.0756 - accuracy: 0.9723 - val_loss: 0.0404 - val_accuracy: 0.9871\n", "Epoch 12/15\n", "351/351 [==============================] - 533s 2s/step - loss: 0.0762 - accuracy: 0.9723 - val_loss: 0.0458 - val_accuracy: 0.9867\n", "Epoch 13/15\n", "351/351 [==============================] - 538s 2s/step - loss: 0.0651 - accuracy: 0.9760 - val_loss: 0.0331 - val_accuracy: 0.9912\n", "Epoch 14/15\n", "351/351 [==============================] - 547s 2s/step - loss: 0.0740 - accuracy: 0.9746 - val_loss: 0.0726 - val_accuracy: 0.9771\n", "Epoch 15/15\n", "351/351 [==============================] - 562s 2s/step - loss: 0.0742 - accuracy: 0.9744 - val_loss: 0.0441 - val_accuracy: 0.9859\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/shrey/Desktop/Kidney-Disease-Classifcation/env/lib/python3.11/site-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n", " saving_api.save_model(\n" ] } ], "source": [ "try:\n", " config = ConfigurationManager()\n", " training_config = config.get_training_config()\n", " training = Training(config=training_config)\n", " training.get_base_model()\n", " training.train_valid_generator()\n", " training.define_and_train_model()\n", " \n", "except Exception as e:\n", " raise e" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.6" } }, "nbformat": 4, "nbformat_minor": 2 }