Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<prepare_output>
start = time.time()
np.random.seed(10)
 %matplotlib inline
%reload_ext autoreload
%autoreload 2
prediction_test_x = prediction_test[['ElasticNet','BRidge','GBR','SVR']]
prediction_test_x.head()<predict_on_test>
data_folder = Path(".. /input")
 train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )
stacked_prediction_test = reg.predict(prediction_test_x)
 df_stacked_prediction_test = pd.DataFrame(data = {'Id': df_test['Id'], 'SalePrice': stacked_prediction_test})
 df_stacked_prediction_test.head()<save_to_csv>
df1 = train_df[train_df.has_cactus==0].copy()
df2 = df1.copy()
train_df = train_df.append([df1, df2], ignore_index=True )
df_stacked_prediction_test.to_csv('mysubmission.csv', index = False )<set_options>
test_img = ImageList.from_df(test_df, path=data_folder/'test', folder='test')
 trfm = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1,
max_lighting=0.2, max_warp=0.2,
p_affine=0.75, p_lighting=0.75 ,xtra_tfms=[contrast(scale=(0.5, 1), p=0.75)])
 imagesize = 64
batchsize = 64
def train_data(imagesize,batchsize):
train_img =(ImageList.from_df(train_df, path=data_folder/'train', folder='train')
.split_by_rand_pct(0.2,seed =344)
.label_from_df() 
.add_test(test_img)
.transform(trfm, size=imagesize)
.databunch(path='.', bs=batchsize, device= torch.device('cuda:0')) 
.normalize(imagenet_stats)
)
 return train_img
%matplotlib notebook
sns.set_style('darkgrid')
<load_from_csv>
data = train_data(imagesize,batchsize )
original_data = pd.read_csv('.. /input/train.csv')
 print('\tFeature\t\t\tdtype\tunique')
 for col in original_data.columns:
print('%15s\t\t%10s\t%d'%(col,original_data[col].dtype,original_data[col].nunique()))
original_data = original_data.drop('Id',axis=1)
 quantitative_features = original_data.select_dtypes('number' ).columns
quantitative_features = quantitative_features[:-1]
qualitative_features = original_data.select_dtypes('object' ).columns
print('Houses: %d , columns: %d
'%(original_data.shape[0],original_data.shape[1]))
print('%d numerical features, %d qualitative features'%(len(quantitative_features),len(qualitative_features)) )<create_dataframe>
train_img1 = train_data(64,64)
def qualitative_to_quantitative(df):
transformed_data = df.copy()
for col in qualitative_features:
ordering = pd.DataFrame()
ordering['vals'] = original_data[col].unique()
ordering.index = ordering.vals
ordering['mean_price'] = original_data[[col,'SalePrice']].groupby(col ).mean() ['SalePrice']
ordering['rank'] = original_data[[col,'SalePrice']].groupby(col ).mean() ['SalePrice'].rank()
ordering_dict = ordering.to_dict() ['rank']
transformed_data[col] = transformed_data[col].replace(to_replace=ordering_dict)
 return transformed_data
massaged_data = qualitative_to_quantitative(original_data )<split>
train_img1.show_batch(rows=5, figsize=(10,10))
y = massaged_data.SalePrice
all_data = massaged_data.drop('SalePrice',axis=1)
 X_train, X_test, y_train, y_test = train_test_split(all_data,y,random_state=2)
 imputer = SimpleImputer()
X_train = imputer.fit_transform(X_train)
 X_test = imputer.fit_transform(X_test)
 plot_different_fitting_routines(X_train,y_train,X_test,y_test )<split>
nmodels = 3
y = massaged_data.SalePrice
numerical_data = massaged_data.drop('SalePrice',axis=1)
 numerical_data = numerical_data[quantitative_features]
X_train, X_test, y_train, y_test = train_test_split(numerical_data,y,random_state=1)
 imputer = SimpleImputer()
X_train = imputer.fit_transform(X_train)
 X_test = imputer.fit_transform(X_test)
 plot_different_fitting_routines(X_train,y_train,X_test,y_test )<split>
def get_ensemble(nmodels):
ens_model = []
learning_rate =[2.95e-02,3e-02,3e-02]
model_list = [models.densenet161,models.densenet161,models.densenet201]
for i in range(nmodels):
print(f'-----Training model: {i+1}--------')
 data = train_data(64,64)
 learn_resnet = cnn_learner(data, model_list[i], metrics=[error_rate, accuracy,AUROC() ],
model_dir="/tmp/model/")
 print('training for 128x128')
 learn_resnet.set_data = train_data(128,64)
 learn_resnet.lr_find()
learn_resnet.recorder.plot(suggestion=True)
 learn_resnet.fit_one_cycle(3,slice(learning_rate[i]))
print('training for 150x150')
 learn_resnet.set_data = train_data(150,96)
 learn_resnet.fit_one_cycle(3,slice(learning_rate[i]))
learn_resnet.save(f'ensem_model_{i}.weights')
 ens_model.append(learn_resnet)
 print(f'-----Training of model {i+1} complete----')
 return ens_model
mse = []
thresholds = np.arange(0.0,0.8,0.1)
 for thr in thresholds:
y = massaged_data.SalePrice
some_data = massaged_data.drop('SalePrice',axis=1)
 some_data = some_data[select_features(thr)]
X_train, X_test, y_train, y_test = train_test_split(some_data,y,random_state=1)
 imputer = SimpleImputer()
X_train = imputer.fit_transform(X_train)
 X_test = imputer.fit_transform(X_test)
 mse.append(mse_different_fitting_routines(X_train,y_train,X_test,y_test))
mse = np.array(mse ).T
plt.figure(figsize=(14,5))
plt.plot(thresholds,mse[0], label = 'regression')
 plt.plot(thresholds,mse[1], label = 'ridge')
 plt.plot(thresholds,mse[2], label = 'lasso')
 plt.plot(thresholds,mse[3], label = 'lassolars')
 plt.plot(thresholds,mse[4], label = 'bayes')
 plt.xlabel('Correlation threshold')
 plt.ylabel('MSE')
 plt.yscale('log')
 plt.gca().invert_xaxis()
plt.legend()<split>
ens = get_ensemble(3 )
y = massaged_data.SalePrice
all_data = massaged_data.drop('SalePrice',axis=1)
 X_train, X_test, y_train, y_test = train_test_split(all_data,y,random_state=2)
 imputer = SimpleImputer()
X_train = imputer.fit_transform(X_train)
 X_test = imputer.fit_transform(X_test)
 plot_different_fitting_routines(X_train,y_train,X_test,y_test )<drop_column>
end = time.time()
print(end - start )
prunned_data = massaged_data.drop('PoolQC',axis=1)
 prunned_data = prunned_data.drop('GarageYrBlt',axis=1)
<split>
ens_test_preds = []
for mdl in ens:
preds,_ = mdl.TTA(ds_type=DatasetType.Test)
 print(np.array(preds ).shape)
 ens_test_preds.append(np.array(preds))
y = prunned_data.SalePrice
all_data = prunned_data.drop('SalePrice',axis=1)
 X_train, X_test, y_train, y_test = train_test_split(all_data,y,random_state=2)
 imputer = SimpleImputer()
X_train = imputer.fit_transform(X_train)
 X_test = imputer.fit_transform(X_test)
 plot_different_fitting_routines(X_train,y_train,X_test,y_test )<split>
ens_preds = np.mean(ens_test_preds, axis =0 )
logy = np.log(y)
 all_data = prunned_data.drop('SalePrice',axis=1)
 X_train, X_test, logy_train, logy_test = train_test_split(all_data,logy,random_state=2)
 imputer = SimpleImputer()
X_train = imputer.fit_transform(X_train)
 X_test = imputer.fit_transform(X_test)
 plot_different_fitting_routines(X_train,logy_train,X_test,logy_test,log=True )<split>
test_df.has_cactus = ens_preds[:, 0]
test_df.head()
logy = np.log(y)
 all_data = prunned_data.drop('SalePrice',axis=1)
 all_data['GrLivArea'] = np.log(all_data['GrLivArea'])
 all_data['1stFlrSF'] = np.log(all_data['1stFlrSF'])
 all_data['ExterQual'] = np.log(all_data['ExterQual'])
 X_train, X_test, logy_train, logy_test = train_test_split(all_data,logy,random_state=2)
 imputer = SimpleImputer()
X_train = imputer.fit_transform(X_train)
 X_test = imputer.fit_transform(X_test)
 plot_different_fitting_routines(X_train,logy_train,X_test,logy_test,log=True )<feature_engineering>
test_df.to_csv('submission.csv', index=False )
logy = np.log(y)
 all_data = prunned_data.drop('SalePrice',axis=1)
 all_data['GrLivArea'] = np.log(all_data['GrLivArea'])
 all_data['1stFlrSF'] = np.log(all_data['1stFlrSF'])
 all_data['ExterQual'] = np.log(all_data['ExterQual'])
 all_data['GarageCars'] = np.power(all_data['GarageCars'],2)
 all_data['BsmtQual'] = np.power(all_data['BsmtQual'],2)
 all_data['OverallQual'] = np.power(all_data['OverallQual'],2)
 X_train, X_test, logy_train, logy_test = train_test_split(all_data,logy,random_state=2)
 imputer = SimpleImputer()
X_train = imputer.fit_transform(X_train)
 X_test = imputer.fit_transform(X_test)
 plot_different_fitting_routines(X_train,logy_train,X_test,logy_test,log=True )<prepare_x_and_y>
interp = ClassificationInterpretation.from_learner(ens[0])
 interp1 = ClassificationInterpretation.from_learner(ens[1])
 interp2 = ClassificationInterpretation.from_learner(ens[2])