Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
%matplotlib inline
py.init_notebook_mode(connected=True)
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 PassengerId = test['PassengerId']
train.head(3)
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(
 pretrained=False,
pretrained_backbone=False,
)<find_best_params>
original_train = train.copy()
full_data = [train, test]
train['Has_Cabin'] = train["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 test['Has_Cabin'] = test["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
for dataset in full_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 for dataset in full_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
 for dataset in full_data:
age_avg = dataset['Age'].mean()
age_std = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset.loc[np.isnan(dataset['Age']), 'Age'] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 num_classes = 15
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 model.load_state_dict(torch.load(WEIGHTS_FILE, map_location=device))
model.eval()
x = model.to(device )<create_dataframe>
def get_title(name):
title_search = re.search('([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ""
for dataset in full_data:
dataset['Title'] = dataset['Name'].apply(get_title)
 for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_mapping = {"Mr": 1, "Master": 2, "Mrs": 3, "Miss": 4, "Rare": 5}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] 						 = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset.loc[ dataset['Age'] <= 16, 'Age'] 					 = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] ;
def collate_fn(batch):
return tuple(zip(*batch))
test_dataset = VinBigTestDataset(test_df, DIR_TEST, get_test_transform())
 test_data_loader = DataLoader(
 test_dataset,
batch_size=6,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<define_variables>
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']
train = train.drop(drop_elements, axis = 1)
 test = test.drop(drop_elements, axis = 1 )
def format_prediction_string(labels, boxes, scores):
pred_strings = []
for j in zip(labels, scores, boxes):
pred_strings.append("{0} {1:.4f} {2} {3} {4} {5}".format(
 j[0], j[1], j[2][0], j[2][1], j[2][2], j[2][3]))
return " ".join(pred_strings )<categorify>
train[['Title', 'Survived']].groupby(['Title'], as_index=False ).agg(['mean', 'count', 'sum'])
detection_threshold = 0.2
results = []
with torch.no_grad() :
for images, image_ids in test_data_loader:
images = list(image.to(device)for image in images)
 outputs = model(images)
 for i, image in enumerate(images):
image_id = image_ids[i]
result = {
'image_id': image_id,
'PredictionString': '14 1.0 0 0 1 1'
}
boxes = outputs[i]['boxes'].data.cpu().numpy()
labels = outputs[i]['labels'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
if len(boxes)> 0:
labels = labels - 1
labels[labels == -1] = 14
selected = scores >= detection_threshold
boxes = boxes[selected].astype(np.int32)
 scores = scores[selected]
labels = labels[selected]
if len(boxes)> 0:
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(labels, boxes, scores)
 }
results.append(result)
<prepare_output>
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).agg(['mean', 'count', 'sum'])
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.head()<save_to_csv>
title_and_sex = original_train.copy() [['Name', 'Sex']]
title_and_sex['Title'] = title_and_sex['Name'].apply(get_title)
 title_and_sex['Sex'] = title_and_sex['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_and_sex[['Title', 'Sex']].groupby(['Title'], as_index=False ).agg(['mean', 'count', 'sum'])
test_df.to_csv('submission.csv', index=False )<install_modules>
def get_gini_impurity(survived_count, total_count):
survival_prob = survived_count/total_count
not_survival_prob =(1- survival_prob)
 random_observation_survived_prob = survival_prob
random_observation_not_survived_prob =(1 - random_observation_survived_prob)
 mislabelling_survided_prob = not_survival_prob * random_observation_survived_prob
mislabelling_not_survided_prob = survival_prob * random_observation_not_survived_prob
gini_impurity = mislabelling_survided_prob + mislabelling_not_survided_prob
return gini_impurity
!pip install timm==0.1.26<set_options>
gini_impurity_starting_node = get_gini_impurity(342,891)
 gini_impurity_starting_node
sys.path.insert(0, ".. /input/efficientdet-torch")
 sys.path.insert(0, ".. /input/omegaconf")
 sys.path.insert(0, ".. /input/weightedboxesfusion")
 warnings.filterwarnings('ignore' )<categorify>
gini_impurity_men = get_gini_impurity(109, 577)
 print(gini_impurity_men)
 gini_impurity_women = get_gini_impurity(233, 314)
 print(gini_impurity_women )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<normalization>
gini_impurity_title_Mr = get_gini_impurity(81, 517)
 print(gini_impurity_title_Mr)
 gini_impurity_title_others = get_gini_impurity(261, 374)
 print(gini_impurity_title_others )
image_dir = ".. /input/vinbigdata-chest-xray-abnormalities-detection/test"
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
dicom = pydicom.dcmread(f"{image_dir}/{image_id}.dicom")
 image = dicom.pixel_array
if "PhotometricInterpretation" in dicom:
if dicom.PhotometricInterpretation == "MONOCHROME1":
image = np.amax(image)- image
image = np.stack([image, image, image])
 image = image.astype('float32')
 image = image - image.min()
image = image / image.max()
image = image.transpose(1, 2, 0)
 if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<define_variables>
men_weight = 577/891
women_weight = 314/891
weighted_gini_impurity_sex_split =(gini_impurity_men * men_weight)+(gini_impurity_women * women_weight)
 print(weighted_gini_impurity_sex_split)
 title_1_weight = 517/891
title_others_weight = 374/891
weighted_gini_impurity_title_split =(gini_impurity_title_Mr * title_1_weight)+(gini_impurity_title_others * title_others_weight)
 print(weighted_gini_impurity_title_split )
image_ids = glob(os.path.join('.. /input/vinbigdata-chest-xray-abnormalities-detection/test', "*.dicom"))
image_ids = [image_id.split('/')[-1].split('.')[0] for image_id in image_ids]
dataset = DatasetRetriever(image_ids = np.array(image_ids), transforms = get_valid_transforms())
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(dataset, batch_size=8, shuffle=False, num_workers=2, drop_last=False, collate_fn=collate_fn )<choose_model_class>
sex_gini_decrease = weighted_gini_impurity_sex_split - gini_impurity_starting_node
print(sex_gini_decrease)
 title_gini_decrease = weighted_gini_impurity_title_split - gini_impurity_starting_node
print(title_gini_decrease)
device = torch.device('cuda:0')if torch.cuda.is_available() else torch.device('cpu')
 def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d4')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 14
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 net = DetBenchEval(net, config)
 net.eval()
return net.to(device)
 net = load_net('.. /input/effdet-weights/last-checkpoint.bin' )<predict_on_test>
cv = KFold(n_splits = 10)
 accuracies = list()
max_attributes = len(list(test))
depth_range = range(1, max_attributes + 1)
 for depth in depth_range:
fold_accuracy = []
tree_model = tree.DecisionTreeClassifier(max_depth = depth)
 for train_fold, valid_fold in cv.split(train):
f_train = train.loc[train_fold]
f_valid = train.loc[valid_fold]
model = tree_model.fit(X = f_train.drop(['Survived'], axis=1), y = f_train["Survived"])
 valid_acc = model.score(X = f_valid.drop(['Survived'], axis=1), y = f_valid["Survived"])
 fold_accuracy.append(valid_acc)
 avg = sum(fold_accuracy)/len(fold_accuracy)
 accuracies.append(avg)
 df = pd.DataFrame({"Max Depth": depth_range, "Average Accuracy": accuracies})
 df = df[["Max Depth", "Average Accuracy"]]
print(df.to_string(index=False))
def make_predictions(images, score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
det = net(images, image_scales=torch.tensor([1] * images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:, :4]
scores = det[i].detach().cpu().numpy() [:, 4]
labels = det[i].detach().cpu().numpy() [:, 5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
'labels': labels[indexes]
})
 return [predictions]
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.5, skip_box_thr=0.4):
boxes = [(prediction[image_index]['boxes'] /(image_size - 1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [prediction[image_index]['labels'].tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr,
skip_box_thr=skip_box_thr)
 boxes = boxes *(image_size - 1)
 return boxes, scores, labels<define_search_space>
y_train = train['Survived']
x_train = train.drop(['Survived'], axis=1 ).values
x_test = test.values
decision_tree = tree.DecisionTreeClassifier(max_depth = 3)
 decision_tree.fit(x_train, y_train)
 y_pred = decision_tree.predict(x_test)
 submission = pd.DataFrame({"PassengerId": PassengerId,"Survived": y_pred})
 submission.to_csv('submission.csv', index=False)
 print(submission.head(4))
mapping = {0: 'Aortic enlargement', 1: 'Atelectasis', 2: 'Calcification', 3: 'Cardiomegaly', 4: 'Consolidation', 5: 'ILD',6: 'Infiltration', 7: 'Lung Opacity',
8: 'Nodule/Mass', 9: 'Other lesion', 10: 'Pleural effusion', 11: 'Pleural thickening',12: 'Pneumothorax', 13: 'Pulmonary fibrosis'}
font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 1
thickness = 3
color =(1, 0, 0)
 for j,(images, image_ids)in enumerate(data_loader):
if j >= 20:
break
predictions = make_predictions(images=images, score_threshold=0.5)
 i = 1
sample = images[i].permute(1, 2, 0 ).cpu().numpy()
boxes, scores, labels = run_wbf(predictions, image_index=i)
 if len(labels)> 0:
boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 fig, ax = plt.subplots(1, 1, figsize=(8, 8))
for score, box, label in zip(scores, boxes, labels):
cv2.rectangle(sample,(box[0], box[1]),(box[2], box[3]),(0, 1, 1), thickness)
 cv2.putText(sample, mapping[label] + "---" + str(round(score, 2)) ,(box[0], box[1]),
font, fontScale, color, thickness, cv2.LINE_AA)
 ax.set_axis_off()
ax.imshow(sample)
 plt.show()<categorify>
with open("tree1.dot", 'w')as f:
f = tree.export_graphviz(decision_tree,
out_file=f,
max_depth = 3,
impurity = True,
feature_names = list(train.drop(['Survived'], axis=1)) ,
class_names = ['Died', 'Survived'],
rounded = True,
filled= True)
 check_call(['dot','-Tpng','tree1.dot','-o','tree1.png'])
 img = Image.open("tree1.png")
 draw = ImageDraw.Draw(img)
 font = ImageFont.truetype('/usr/share/fonts/truetype/liberation/LiberationSerif-Bold.ttf', 26)
 draw.text(( 10, 0),
'"Title <= 1.5" corresponds to "Mr." title', 
(0,0,255),
font=font)
 img.save('sample-out.png')
 PImage("sample-out.png" )
<predict_on_test><EOS> 
acc_decision_tree = round(decision_tree.score(x_train, y_train)* 100, 2)
 acc_decision_tree