import pydicom 
import os 
import pandas as pd 
import numpy as np 
​
import cv2
import matplotlib.pyplot as plt 
​
add Codeadd Markdown
class_data = pd.read_csv("/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv")
add Codeadd Markdown
train_labels = pd.read_csv("/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv")
add Codeadd Markdown
data_dir = "/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images"
add Codeadd Markdown
num_dir = os.listdir(data_dir)
print(len(num_dir))
26684
add Codeadd Markdown
dataset = pd.concat([class_data.drop(columns = 'patientId'), train_labels], axis = 1)
add Codeadd Markdown
dataset.head()
class	patientId	x	y	width	height	Target
0	No Lung Opacity / Not Normal	0004cfab-14fd-4e49-80ba-63a80b6bddd6	NaN	NaN	NaN	NaN	0
1	No Lung Opacity / Not Normal	00313ee0-9eaa-42f4-b0ab-c148ed3241cd	NaN	NaN	NaN	NaN	0
2	No Lung Opacity / Not Normal	00322d4d-1c29-4943-afc9-b6754be640eb	NaN	NaN	NaN	NaN	0
3	Normal	003d8fa0-6bf1-40ed-b54c-ac657f8495c5	NaN	NaN	NaN	NaN	0
4	Lung Opacity	00436515-870c-4b36-a041-de91049b9ab4	264.0	152.0	213.0	379.0	1
add Codeadd Markdown
dataset['bbox'] = dataset[['x', 'y', 'height', 'width']].apply(lambda x: '-'.join(str(i) for i in x), axis=1)
add Codeadd Markdown
dataset.drop(columns = ["x", "y", "width", "height"])
class	patientId	Target	bbox
0	No Lung Opacity / Not Normal	0004cfab-14fd-4e49-80ba-63a80b6bddd6	0	nan-nan-nan-nan
1	No Lung Opacity / Not Normal	00313ee0-9eaa-42f4-b0ab-c148ed3241cd	0	nan-nan-nan-nan
2	No Lung Opacity / Not Normal	00322d4d-1c29-4943-afc9-b6754be640eb	0	nan-nan-nan-nan
3	Normal	003d8fa0-6bf1-40ed-b54c-ac657f8495c5	0	nan-nan-nan-nan
4	Lung Opacity	00436515-870c-4b36-a041-de91049b9ab4	1	264.0-152.0-379.0-213.0
...	...	...	...	...
30222	Lung Opacity	c1ec14ff-f6d7-4b38-b0cb-fe07041cbdc8	1	185.0-298.0-379.0-228.0
30223	Normal	c1edf42b-5958-47ff-a1e7-4f23d99583ba	0	nan-nan-nan-nan
30224	Normal	c1f6b555-2eb1-4231-98f6-50a963976431	0	nan-nan-nan-nan
30225	Lung Opacity	c1f7889a-9ea9-4acb-b64c-b737c929599a	1	570.0-393.0-345.0-261.0
30226	Lung Opacity	c1f7889a-9ea9-4acb-b64c-b737c929599a	1	233.0-424.0-356.0-201.0
30227 rows × 4 columns

add Codeadd Markdown
print(len(dataset[dataset["Target"] == 1]))
9555
add Codeadd Markdown
Image View
add Codeadd Markdown
image_directory = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'
​
add Codeadd Markdown
dataset['image_path'] = image_directory + dataset['patientId'] + '.dcm'
​
add Codeadd Markdown
dataset.head()
class	patientId	x	y	width	height	Target	bbox	image_path
0	No Lung Opacity / Not Normal	0004cfab-14fd-4e49-80ba-63a80b6bddd6	NaN	NaN	NaN	NaN	0	nan-nan-nan-nan	/kaggle/input/rsna-pneumonia-detection-challen...
1	No Lung Opacity / Not Normal	00313ee0-9eaa-42f4-b0ab-c148ed3241cd	NaN	NaN	NaN	NaN	0	nan-nan-nan-nan	/kaggle/input/rsna-pneumonia-detection-challen...
2	No Lung Opacity / Not Normal	00322d4d-1c29-4943-afc9-b6754be640eb	NaN	NaN	NaN	NaN	0	nan-nan-nan-nan	/kaggle/input/rsna-pneumonia-detection-challen...
3	Normal	003d8fa0-6bf1-40ed-b54c-ac657f8495c5	NaN	NaN	NaN	NaN	0	nan-nan-nan-nan	/kaggle/input/rsna-pneumonia-detection-challen...
4	Lung Opacity	00436515-870c-4b36-a041-de91049b9ab4	264.0	152.0	213.0	379.0	1	264.0-152.0-379.0-213.0	/kaggle/input/rsna-pneumonia-detection-challen...
add Codeadd Markdown
def read_and_resize_images(pneumonia, num_samples=None):
    resized_images = []
    boxes = []
    
    if num_samples:
        pneumonia = pneumonia[:num_samples]
    
    for _, row in pneumonia.iterrows():
        image_path = row['image_path']
        target = row['Target']
        
        dicom_data = pydicom.read_file(image_path)
        img = dicom_data.pixel_array
        
        img = cv2.resize(img, (224, 224))
        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
        
        resized_images.append(img)
        boxes.append(np.array(target, dtype=np.float32))
    
    return np.array(resized_images), np.array(boxes)
add Codeadd Markdown
import random
add Codeadd Markdown
random_patient_id = random.choice(dataset['patientId'])
​
image_path = dataset[dataset['patientId'] == random_patient_id]['image_path'].values[0]
​
add Codeadd Markdown
dicom_data = pydicom.read_file(image_path)
​
add Codeadd Markdown
def visualize_dicom_image(dicom_data):
   
    img = dicom_data.pixel_array
​
    plt.figure(figsize=(6, 6))
    plt.imshow(img, cmap='gray')
    plt.title('DICOM Image')
    plt.axis('off') 
    plt.show()
    
visualize_dicom_image(dicom_data)

add Codeadd Markdown
CNN - Simple CNN
add Codeadd Markdown
CNN Model - transfer learning with resnet101
add Codeadd Markdown
from sklearn.model_selection import train_test_split
from torchvision import transforms
add Codeadd Markdown
X, y = read_and_resize_images(dataset[:1000])
add Codeadd Markdown
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
add Codeadd Markdown
img_size = 224
add Codeadd Markdown
transform = transforms.Compose([
    transforms.ToPILImage(), 
    transforms.Resize((int(img_size), int(img_size))), 
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
​
add Codeadd Markdown
import torch
from torchvision.transforms import ToTensor
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
import torchvision.models as models
import torch.nn as nn
​
add Codeadd Markdown
class PneumoniaDataset(Dataset):
    def __init__(self, images, labels, transform=None):
        self.images = images
        self.labels = labels.astype(int)
        self.transform = transform
​
    def __len__(self):
        return len(self.images)
​
    def __getitem__(self, idx):
        image = self.images[idx]
        label = self.labels[idx]
​
        if self.transform:
            image = self.transform(image)
​
        label = torch.tensor(label, dtype=torch.long)
​
        return image, label
​
add Codeadd Markdown
train_dataset = PneumoniaDataset(X_train, y_train, transform=transform)
test_dataset = PneumoniaDataset(X_test, y_test, transform=transform)
​
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)
​
add Codeadd Markdown
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
resnet101 = models.resnet101(pretrained=True)
num_classes = 2  
resnet101.fc = nn.Linear(resnet101.fc.in_features, num_classes)
resnet101 = resnet101.to(device)
​
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(resnet101.parameters(), lr=0.001)
/opt/conda/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/opt/conda/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet101_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet101_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
add Codeadd Markdown
num_epochs = 10
​
train_losses = []
train_accuracies = []
​
for epoch in range(num_epochs):
    resnet101.train()
    running_loss = 0.0
    correct = 0
    total = 0
​
    for images, labels in train_loader:
        images = images.to(device)
        labels = labels.to(device)
​
        optimizer.zero_grad()
​
        outputs = resnet101(images)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
​
        running_loss += loss.item()
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
​
    epoch_loss = running_loss / len(train_loader)
    epoch_accuracy = 100 * correct / total
    train_losses.append(epoch_loss)
    train_accuracies.append(epoch_accuracy)
​
    print(f"Epoch {epoch+1}/{num_epochs}, Loss: {epoch_loss}, Accuracy: {epoch_accuracy}%")
​
Epoch 1/10, Loss: 0.6249697494506836, Accuracy: 70.375%
Epoch 2/10, Loss: 0.43603590846061707, Accuracy: 81.5%
Epoch 3/10, Loss: 0.4294892305135727, Accuracy: 81.625%
Epoch 4/10, Loss: 0.3221330940723419, Accuracy: 85.5%
Epoch 5/10, Loss: 0.22102454751729966, Accuracy: 92.0%
Epoch 6/10, Loss: 0.3135212221741676, Accuracy: 87.25%
Epoch 7/10, Loss: 0.22327293410897256, Accuracy: 92.25%
Epoch 8/10, Loss: 0.11748717650771141, Accuracy: 95.75%
Epoch 9/10, Loss: 0.09963407680392265, Accuracy: 95.875%
Epoch 10/10, Loss: 0.0964412534609437, Accuracy: 96.5%
add Codeadd Markdown
plt.figure(figsize=(12, 5))
​
plt.subplot(1, 2, 1)
plt.plot(train_losses, label='Training Loss')
plt.title('Training Loss over Epochs')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
​
plt.subplot(1, 2, 2)
plt.plot(train_accuracies, label='Training Accuracy')
plt.title('Training Accuracy over Epochs')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
​
plt.show()
​

add Codeadd Markdown
resnet101 = resnet101.to(device)
resnet101.eval()  
​
correct = 0
total = 0
with torch.no_grad():  
    for images, labels in test_loader:
        images = images.to(device)
        labels = labels.to(device)
​
        outputs = resnet101(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
​
accuracy = 100 * correct / total
print(f'Accuracy on the test set: {accuracy:.2f}%')
Accuracy on the test set: 79.00%
add Codeadd Markdown
Custom CNN
add Codeadd Markdown
import torch.nn.functional as F
​
add Codeadd Markdown
class SimpleCNN(nn.Module):
    def __init__(self):
        super(SimpleCNN, self).__init__()
        
        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, padding=1)
        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)
        self.fc1 = nn.Linear(64 * 56 * 56, 128)
        self.fc2 = nn.Linear(128, 2)
​
    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2)
        x = x.view(x.size(0), -1)  
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return x
​
add Codeadd Markdown
model = SimpleCNN()
model = model.to(device)
add Codeadd Markdown
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
add Codeadd Markdown
def train_model(model, train_loader, criterion, optimizer, num_epochs=10):
    model.train()
    train_losses = []
    train_accuracies = []
​
    for epoch in range(num_epochs):
        running_loss = 0.0
        correct = 0
        total = 0
​
        for images, labels in train_loader:
            images, labels = images.to(device), labels.to(device)
​
            optimizer.zero_grad()
            outputs = model(images)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
​
            running_loss += loss.item()
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
​
        epoch_loss = running_loss / len(train_loader)
        epoch_accuracy = 100 * correct / total
        train_losses.append(epoch_loss)
        train_accuracies.append(epoch_accuracy)
​
        print(f"Epoch {epoch+1}/{num_epochs}, Loss: {epoch_loss:.4f}, Accuracy: {epoch_accuracy:.2f}%")
​
    return train_losses, train_accuracies
​
add Codeadd Markdown
num_epochs = 10 
add Codeadd Markdown
train_losses, train_accuracies = train_model(model, train_loader, criterion, optimizer, num_epochs)
​
Epoch 1/10, Loss: 3.0904, Accuracy: 56.75%
Epoch 2/10, Loss: 0.6478, Accuracy: 58.25%
Epoch 3/10, Loss: 0.5368, Accuracy: 73.88%
Epoch 4/10, Loss: 0.4931, Accuracy: 78.12%
Epoch 5/10, Loss: 0.4391, Accuracy: 80.25%
Epoch 6/10, Loss: 0.4099, Accuracy: 80.88%
Epoch 7/10, Loss: 0.3807, Accuracy: 83.38%
Epoch 8/10, Loss: 0.3467, Accuracy: 84.75%
Epoch 9/10, Loss: 0.2982, Accuracy: 88.25%
Epoch 10/10, Loss: 0.2672, Accuracy: 89.38%
add Codeadd Markdown
plt.figure(figsize=(12, 5))
​
plt.subplot(1, 2, 1)
plt.plot(train_losses, label='Training Loss')
plt.title('Training Loss over Epochs')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
​
plt.subplot(1, 2, 2)
plt.plot(train_accuracies, label='Training Accuracy')
plt.title('Training Accuracy over Epochs')
plt.xlabel('Epoch')
plt.ylabel('Accuracy (%)')
plt.legend()
​
plt.show()

add Codeadd Markdown
from sklearn.metrics import confusion_matrix
import seaborn as sns
add Codeadd Markdown
def evaluate_model(model, test_loader):
    model.eval()
    y_true = []
    y_pred = []
​
    with torch.no_grad():
        for images, labels in test_loader:
            images, labels = images.to(device), labels.to(device)
            outputs = model(images)
            _, predicted = torch.max(outputs, 1)
            y_true.extend(labels.cpu().numpy())
            y_pred.extend(predicted.cpu().numpy())
​
    accuracy = 100 * sum([1 for true, pred in zip(y_true, y_pred) if true == pred]) / len(y_true)
    print(f'Accuracy on test set: {accuracy:.2f}%')
​
    
evaluate_model(model, test_loader)
Accuracy on test set: 80.50%