Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<drop_column>
import os
import time
import copy
import pathlib
from collections import defaultdict
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from PIL import Image
test_passenger_ids = test.PassengerId<drop_column>
import torch
import torchvision
import torch.nn as nn
import torch.optim as optmim
from torch.optim import lr_scheduler
from torchvision import models, datasets, transforms
from torch.utils.data import Dataset
from torch.utils.data.sampler import SubsetRandomSampler
use_columns = ['Pclass','Sex','Title','CabinLetter','Age','TicketStub','FamilySize_Code','Fare','Embarked','PctLived','Survived']
drop_columns = set(test.columns)- set(use_columns)
 test.drop(drop_columns, axis=1,inplace=True)
 test.columns<feature_engineering>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )
non_convertible_columns = \
test.columns[(test.dtypes == 'float64')|(test.dtypes == 'category')|(test.columns=='Survived')]
convertible_columns = set(test.columns)- set(non_convertible_columns)
 for column in convertible_columns:
test[column] = pd.Categorical(test[column] )<categorify>
base_path = pathlib.Path(".. /input/" )
test_with_dummies=pd.get_dummies(test,drop_first=True )<feature_engineering>
df_data = pd.read_csv(base_path/'train.csv' )
missing_cols = set(train_with_dummies.columns)- set(test_with_dummies.columns)
 for c in missing_cols:
test_with_dummies[c] = 0
test_with_dummies = test_with_dummies[train_with_dummies.columns]<prepare_x_and_y>
class CactusDataset(Dataset):

 def __init__(self, df_data, image_path, image_size, transform=None):
self.data = df_data
self.image_path = image_path
self.transform = transform
def __len__(self):

 return len(self.data)
 def __getitem__(self, index):

 image_name = self.data.iloc[index, 0]
image = Image.open(str(self.image_path)+ '/' +image_name)
 image = image.convert('RGB')
 image = image.resize(image_size, Image.ANTIALIAS)
 if self.transform is not None:
image = self.transform(image)
 label = self.data.iloc[index, 1]
label = torch.from_numpy(np.asarray(label))
return image, label
X_full = np.array(train_with_dummies.loc[:,(train_with_dummies.columns !='Survived')])
 test_array = np.array(test_with_dummies.loc[:,(test_with_dummies.columns !='Survived')] )<normalization>
train_transform = transforms.Compose([
transforms.RandomHorizontalFlip() ,
transforms.ColorJitter(brightness=0.2, contrast=0.1, saturation=0.1),
transforms.RandomAffine(0.1),
transforms.RandomGrayscale() ,
transforms.RandomRotation(10),
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
 ] )
full_scaler = preprocessing.StandardScaler().fit(X_full)
 X_full_standardized = full_scaler.transform(X_full)
 test_array = full_scaler.transform(test_array )<predict_on_test>
valid_transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
 ] )
_=logitmodel.fit(X_full_standardized, y_full)
 logit_predictions = logitmodel.predict(test_array)
 _=adaboost.fit(X_full_standardized, y_full)
 adaboost_predictions = adaboost.predict(test_array)
 _=randomforest.fit(X_full_standardized, y_full)
 randomforest_predictions = randomforest.predict(test_array)
 deep_model = make_deep_model(num_features=X_full_standardized.shape[1])
 _=deep_model.fit(X_full_standardized, y_full, epochs=40, verbose=0)
 deep_predictions = deep_model.predict(test_array)[:, 1]
deep_predictions[deep_predictions<.5] = 0
deep_predictions[deep_predictions>=.5] = 1
deep_predictions=deep_predictions.astype('int')
 all_model_predictions = [logit_predictions, adaboost_predictions, randomforest_predictions, deep_predictions]
ada_rf_deep_predictions = [adaboost_predictions, randomforest_predictions, deep_predictions]
all_no_deep_predictions = [adaboost_predictions, logit_predictions, randomforest_predictions]
voted_allmodels_predictions = create_voted_predictions(all_model_predictions)
 voted_ada_rf_deep_predictions = create_voted_predictions(ada_rf_deep_predictions)
 voted_all_no_deep_predictions = create_voted_predictions(all_no_deep_predictions)
 voted_allmodels_tilted_predictions = create_voted_predictions(all_model_predictions, survived_tilt=1)
 voted_ada_rf_deep_tilted_predictions = create_voted_predictions(ada_rf_deep_predictions, survived_tilt=1)
 voted_all_no_deep_tilted_predictions = create_voted_predictions(all_no_deep_predictions, survived_tilt=1 )<define_variables>
image_path = base_path/'train'/'train'
image_size =(224, 224)
 bs = 64
potential_submissions = dict([('LogisticRegression',logit_predictions),\ 
('AdaBoost',adaboost_predictions),\ 
('RandomForest',randomforest_predictions),\ 
('DeepModel',deep_predictions),\ 
('Voted_AllModels',voted_allmodels_predictions),\ 
('Voted_Ada_Rf_Deep',voted_ada_rf_deep_predictions),\ 
('Voted_All_No_Deep',voted_all_no_deep_predictions),\ 
('Voted_AllModels_Tilted',voted_allmodels_tilted_predictions),\ 
('Voted_Ada_Rf_Deep_Tilted',voted_ada_rf_deep_tilted_predictions),\ 
('Voted_All_No_Deep_Tilted',voted_all_no_deep_tilted_predictions)] )<save_to_csv>
cac_dataset = CactusDataset(df_data, image_path, image_size, transform=train_transform )
timestamp = datetime.today().strftime('%y%m%d_%H%M')
 submission_df = pd.DataFrame()
submission_df['PassengerId'] = test_passenger_ids
for i in range(5):
model = model_results_initial_features_with_pct_lived.data.index[i]
submission_df['Survived'] = potential_submissions[model]
file_name = model + '_' + timestamp + '.csv'
submission_df.to_csv(file_name,index=False )<import_modules>
sample_loader = torch.utils.data.DataLoader(cac_dataset, batch_size=8, shuffle=True)
 images, labels = next(iter(sample_loader))
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 5000)
 pd.set_option('display.max_rows', 5000)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
X = df_data['id']
y = df_data['has_cactus']
train=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 sample_sub=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv' )<filter>
X_train, X_val, y_train, y_val = train_test_split(X, y,
stratify=y,
test_size=0.20, shuffle=True )
train[train.SalePrice>300000]<sort_values>
df_train = pd.DataFrame({'id': X_train.values, 'has_cactus':y_train.values})
 df_val = pd.DataFrame({'id':X_val.values, 'has_cactus':y_val.values} )
corr_dict=correlation_train['SalePrice'].sort_values(ascending=False ).to_dict()
important_columns=[]
for key,value in corr_dict.items() :
if(( value>0.1)&(value<0.8)) |(value<=-0.1):
important_columns.append(key)
 important_columns<concatenate>
cac_dataset_train = CactusDataset(df_train, image_path, image_size, transform=train_transform)
 cac_dataset_valid = CactusDataset(df_val, image_path, image_size, transform=valid_transform )
train_test=pd.concat([train,test],axis=0,sort=False)
 train_test.head()<create_dataframe>
train_loader = torch.utils.data.DataLoader(cac_dataset_train, batch_size=32, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(cac_dataset_valid, batch_size=32, shuffle=True )
pd.set_option('display.max_rows', 5000)
 train_test_null_info=pd.DataFrame(train_test.isnull().sum() ,columns=['Count of NaN'])
 train_test_dtype_info=pd.DataFrame(train_test.dtypes,columns=['DataTypes'])
 train_tes_info=pd.concat([train_test_null_info,train_test_dtype_info],axis=1)
 train_tes_info<data_type_conversions>
def get_base_model(model_name, num_classes, pretrained=True, unfreeze=True, to_gpu=True):
if model_name == "resnet":
model = models.resnet152(pretrained=pretrained)
 input_features = model.fc.in_features
fc_custom = nn.Linear(input_features, num_classes)
 model.fc = fc_custom
elif model_name == "densenet":
model = models.densenet121(pretrained=pretrained)
 input_features = model.classifier.in_features
fc_custom = nn.Linear(input_features, num_classes)
 model.classifier = fc_custom
if unfreeze:
for param in model.parameters() :
param.requires_grad = True
if to_gpu:
model = model.to(device)
 return model
train_test.loc[train_test['Fireplaces']==0,'FireplaceQu']='Nothing'
train_test['LotFrontage'] = train_test['LotFrontage'].fillna(train_test.groupby('1stFlrSF')['LotFrontage'].transform('mean'))
train_test['LotFrontage'].interpolate(method='linear',inplace=True)
 train_test['LotFrontage']=train_test['LotFrontage'].astype(int)
 train_test['MasVnrArea'] = train_test['MasVnrArea'].fillna(train_test.groupby('MasVnrType')['MasVnrArea'].transform('mean'))
train_test['MasVnrArea'].interpolate(method='linear',inplace=True)
 train_test['MasVnrArea']=train_test['MasVnrArea'].astype(int)
 train_test["Fence"] = train_test["Fence"].fillna("None")
 train_test["FireplaceQu"] = train_test["FireplaceQu"].fillna("None")
 train_test["Alley"] = train_test["Alley"].fillna("None")
 train_test["PoolQC"] = train_test["PoolQC"].fillna("None")
 train_test["MiscFeature"] = train_test["MiscFeature"].fillna("None")
 train_test.loc[train_test['BsmtFinSF1']==0,'BsmtFinType1']='Unf'
train_test.loc[train_test['BsmtFinSF2']==0,'BsmtQual']='TA'
train_test['YrBltRmd']=train_test['YearBuilt']+train_test['YearRemodAdd']
train_test['Total_Square_Feet'] =(train_test['BsmtFinSF1'] + train_test['BsmtFinSF2'] + train_test['1stFlrSF'] + train_test['2ndFlrSF'] + train_test['TotalBsmtSF'])
 train_test['Total_Bath'] =(train_test['FullBath'] +(0.5 * train_test['HalfBath'])+ train_test['BsmtFullBath'] +(0.5 * train_test['BsmtHalfBath']))
train_test['Total_Porch_Area'] =(train_test['OpenPorchSF'] + train_test['3SsnPorch'] + train_test['EnclosedPorch'] + train_test['ScreenPorch'] + train_test['WoodDeckSF'])
 train_test['exists_pool'] = train_test['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_garage'] = train_test['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_fireplace'] = train_test['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_bsmt'] = train_test['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 train_test['old_house'] = train_test['YearBuilt'].apply(lambda x: 1 if x <1990 else 0)
 for i in train_test.columns:
if 'SalePrice' not in i:
if 'object' in str(train_test[str(i)].dtype):
train_test[str(i)]=train_test[str(i)].fillna(method='ffill')
<categorify>
model = get_base_model("resnet", 2 )
columns =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', 'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1', 'YrSold', 'MoSold', 'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope', 'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond')
 for col in columns:
lbl_enc = LabelEncoder()
lbl_enc.fit(list(train_test[col].values))
train_test[col] = lbl_enc.transform(list(train_test[col].values))<sort_values>
criterion = nn.CrossEntropyLoss()
optimizer = optmim.SGD([
{'params': model.layer1.parameters() , 'lr': 1e-6},
{'params': model.layer2.parameters() , 'lr': 1e-5},
{'params': model.layer3.parameters() , 'lr': 1e-4},
{'params': model.layer4.parameters() , 'lr': 1e-4},
{'params': model.fc.parameters() , 'lr': 1e-3}
], lr=1e-3)
numeric_features = train_test.dtypes[train_test.dtypes != "object"].index
skewed_features = train_test[numeric_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 print(skewed_features )<filter>
def get_auc_score(y_true, y_pred):
y_true = np.array([item for sublist in y_true for item in sublist])
 y_pred = np.array([item for sublist in y_pred for item in sublist])
 return roc_auc_score(y_true, y_pred )
high_skewness = skewed_features[abs(skewed_features)> 0.9]
skewed_features = high_skewness.index<train_model>
n_epochs = 50
print(high_skewness)
 print('
Variables with high skewness:
',skewed_features )<feature_engineering>
def train_model(model, dataloaders, criterion, optimizer, epochs=50):
start_time = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0
best_loss = np.Inf
metrics = defaultdict(list)
 losses = defaultdict(list)
 for epoch_no in range(epochs):
print("*" * 100)
 print(f"Starting epoch no {epoch_no+1}")
 for phase in ['train','valid']:
y_true = list()
y_pred = list()
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
 idxs, preds = torch.max(outputs, 1)
 loss = criterion(outputs, labels)
 if phase == 'train':
loss.backward()
optimizer.step()
probs = torch.nn.functional.softmax(outputs, dim=1)[:, 1]
running_loss += loss.item() * inputs.size(0)
 running_corrects += torch.sum(preds == labels.data)
 y_true.append(list(labels.data.cpu().numpy()))
y_pred.append(list(probs.detach().cpu().numpy()))
auc_score = get_auc_score(y_true, y_pred)
 epoch_loss = running_loss / len(dataloaders[phase].dataset)
 epoch_acc = float(running_corrects)/ len(dataloaders[phase].dataset)
 if phase == 'valid' and epoch_loss < best_loss:
print(f"Validation loss decreased from {best_loss} to {epoch_loss}.Saving Model ")
 best_loss = epoch_loss
checkpoint = {'model': model,
'state_dict': model.state_dict() ,
'optimizer' : optimizer.state_dict()
}
torch.save(checkpoint, 'model_resnet_50_v2.pth')
 best_model_wts = copy.deepcopy(model.state_dict())
 print(f"Ending epoch no {epoch_no+1} with below stats")
 print(f"-------------Stats for {phase}-------------")
 print(f"Loss: {epoch_loss}...Accuracy: {epoch_acc}")
 print(f"AUC for {phase} is {auc_score}")
 metrics[phase].append(epoch_acc)
 losses[phase].append(epoch_loss)
 time_elapsed = time.time() - start_time
print(f"Total time taken in training: {time_elapsed / 60} minutes")
 model.load_state_dict(best_model_wts)
 return model, metrics, losses
for feature in skewed_features:
train_test[feature] = boxcox1p(train_test[feature], boxcox_normmax(train_test[feature] + 1)) 
<categorify>
df_submission = pd.read_csv(base_path/'sample_submission.csv' )
train_test=pd.get_dummies(train_test,dtype='int8' )<create_dataframe>
def load_model(checkpoint_path):
checkpoint = torch.load(checkpoint_path)
 model = checkpoint['model']
model.load_state_dict(checkpoint['state_dict'])
 model.cpu()
for parameter in model.parameters() :
parameter.requires_grad = False
model.eval()
return model
train_test_null_info=pd.DataFrame(train_test.isnull().sum() ,columns=['Count of NaN'])
 train_test_dtype_info=pd.DataFrame(train_test.dtypes,columns=['DataTypes'])
 train_test_info=pd.concat([train_test_null_info,train_test_dtype_info],axis=1)
 train_test_info<split>
model = load_model('model_resnet_50_v2.pth' )
train=train_test[0:1460]
test=train_test[1460:2919]<sort_values>
test_image_path = base_path / 'test' / 'test'
corr_dict2=corr_new_train['SalePrice'].sort_values(ascending=False ).to_dict()
corr_dict2<define_variables>
def add_full_path(file_name):
full_path = str(test_image_path)+ '/' + file_name
return full_path
best_columns=[]
for key,value in corr_dict2.items() :
if(( value>=0.3175)&(value<0.9)) |(value<=-0.315):
best_columns.append(key)
 best_columns<feature_engineering>
df_submission['id'] = df_submission['id'].apply(add_full_path )
train['SalePrice_Log1p'] = np.log1p(train.SalePrice )<filter>
def read_image(image_path):
image = Image.open(image_path)
 image = image.convert('RGB')
 image = image.resize(image_size, Image.ANTIALIAS)
 return image
print(len(train[(train.KitchenQual==3)&(train.SalePrice>360000)].index))<train_on_grid>
test_transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
 ] )
dbscan = DBSCAN(eps=1400, min_samples=20 ).fit(train_pca)
 core_samples_mask = np.zeros_like(dbscan.labels_, dtype=bool)
 core_samples_mask[dbscan.core_sample_indices_] = True
labels=dbscan.labels_
n_clusters_ = len(set(labels)) -(1 if -1 in labels else 0 )<create_dataframe>
def get_predictions(image_path):
image = read_image(image_path)
 image = test_transform(image)
 image = torch.unsqueeze(image, 0)
 pred = model(image)
 p_cactus = torch.nn.functional.softmax(pred, dim=1)[0][1].item()
return p_cactus
labels=pd.DataFrame(labels,columns=['Classes'])
 print(labels[labels['Classes']==-1] )<concatenate>
df_submission['has_cactus'] = df_submission['id'].apply(get_predictions )
train=pd.concat([train,labels],axis=1 )<filter>
def remove_full_path(image_path):
paths = image_path.split("/")
 path = paths[4]
return path
train[train.Classes==-1]<drop_column>
df_submission['id'] = df_submission['id'].apply(remove_full_path )
<drop_column><EOS> 
df_submission.to_csv('submission_resnet50_unfreeze_1.csv', index=None )