Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
%matplotlib inline
np.random.seed(42 )
callbacks = [
EarlyStopping(monitor='val_loss', patience=10),
ModelCheckpoint(filepath='model.h5', monitor='val_loss', save_best_only=True)
 ]
history = model.fit_generator(
 train_gen,
validation_data=valid_gen,
validation_steps=len(train_gen),
steps_per_epoch=len(train_gen_df)/ batch_size,
epochs=100,
verbose=True,
shuffle=True,
callbacks=callbacks
)<load_pretrained>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 data = pd.concat([train, test], axis=0, ignore_index=True)
model.load_weights('model.h5')
 test_datagen = ImageDataGenerator(
 rescale=1.0 / 255.0 
)
 test_gen = valid_datagen.flow_from_dataframe(
 dataframe=test_df,
directory=test_dir,
x_col='id',
class_mode=None,
batch_size=1,
target_size=(32, 32),
shuffle=False
)<predict_on_test>
data.isnull().sum()
predictions = model.predict_generator(test_gen, steps=len(test_gen), verbose=True)
<save_to_csv>
train[['Fare','Name','Ticket','SibSp','Parch']].iloc[train.index[(train['Pclass']==3)&(train['Fare']>60)]]
test_df['has_cactus'] = predictions
test_df.to_csv('submission.csv', index=False)
 print('Done!' )<import_modules>
train['FareCorr'] = train['Fare'].copy()
m=0
for grp, grp_df in train[['Ticket', 'Name', 'Pclass', 'Fare', 'PassengerId']].groupby(['Ticket']):
if(len(grp_df)!= 1):
if m==0:
print(grp_df)
 m=1
for ind, row in grp_df.iterrows() :
passID = row['PassengerId']
train.loc[train['PassengerId'] == passID, 'FareCorr'] = train['Fare'][train['PassengerId'] == passID]/len(grp_df)
import numpy as np
import pandas as pd<import_modules>
train['FareCat']=pd.cut(train['FareCorr'], bins=[0,15,65,max(train["FareCorr"]+1)], labels=['low','mid','high'])
 train['FareCat'].value_counts()
from pathlib import Path
from fastai import *
from fastai.vision import *
import torch<define_variables>
for name_string in data['Name']:
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 data['Title']=data['Title'].replace({'Ms':'Miss','Mlle':'Miss','Mme':'Mrs'})
 data['Title'].value_counts()
data_folder = Path(".. /input")
<load_from_csv>
data['Title']=data['Title'].replace(['Sir','Don','Dona','Jonkheer','Lady','Countess'], 'Noble')
 data['Title']=data['Title'].replace(['Dr', 'Rev','Col','Major','Capt'], 'Others')
 data['Title'].value_counts()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )<choose_model_class>
data['Parch'][(data['Age']<15)&(data['Title']=='Miss')].value_counts()
learn = cnn_learner(train_img, models.densenet201, metrics=[error_rate, accuracy] )<train_model>
data['Parch'][(data['Age']>=15)&(data['Age']<25)&(data['Title']=='Miss')].value_counts()
lr = 3e-02
learn.fit_one_cycle(5, slice(lr))<predict_on_test>
title_list=data.groupby('Title')['Age'].median().index.to_list()
for title in title_list:
if title=='Miss':
data.loc[(data['Age'].isnull())&(data['Title'] == title)&(data['Parch'] == 0), 'Age'] \
= data['Age'][(data['Title']== title)&(data['Age']>=15)].median()
data.loc[(data['Age'].isnull())&(data['Title'] == title)&(data['Parch'] > 0), 'Age'] \
= data['Age'][(data['Title']== title)&(data['Age']<15)].median()
data.loc[(data['Age']<15)&(data['Title'] == title), 'Title'] \
= 'youngMiss'
else:
data.loc[(data['Age'].isnull())&(data['Title'] == title), 'Age'] \
= data['Age'][(data['Title']== title)].median()
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
data['Title'].value_counts()
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
data.loc[data['SibSp'] + data['Parch'] + 1 == 1, 'FamilySize'] = 'Single'
data.loc[data['SibSp'] + data['Parch'] + 1 > 1 , 'FamilySize'] = 'Small'
data.loc[data['SibSp'] + data['Parch'] + 1 > 4 , 'FamilySize'] = 'Big'
test_df.to_csv('submission.csv', index=False )<import_modules>
data.FamilySize.value_counts()
from pathlib import Path
from fastai import *
from fastai.vision import *
import torch<load_from_csv>
data['Last_Name'] = data['Name'].str.extract('([A-Za-z]+),', expand=True)
 data['Last_Name'].value_counts()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )<define_variables>
data['Last_Name'] = data['Name'].apply(lambda x: str.split(x, ",")[0])
 default_value = 0.5
data['FamilySurvival'] = default_value
for grp, grp_df in data[['Survived', 'Last_Name', 'Ticket', 'PassengerId']].groupby(['Last_Name']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'FamilySurvival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == passID, 'FamilySurvival'] = 0
for _, grp_df in data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['FamilySurvival'] == 0)|(row['FamilySurvival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'FamilySurvival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == passID, 'FamilySurvival'] = 0
data['FamilySurvival'].value_counts()
test_img = ImageList.from_df(test_df, path=data_folder/'test', folder='test' )<categorify>
data['FareCorr'] = data['Fare'].copy()
for grp, grp_df in data[['Ticket','Name', 'Pclass', 'Fare', 'PassengerId']].groupby(['Ticket']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
passID = row['PassengerId']
data.loc[data['PassengerId'] == passID, 'FareCorr'] = data['Fare'][data['PassengerId'] == passID]/len(grp_df )
train_img =(ImageList.from_df(train_df, path=data_folder/'train', folder='train')
.split_by_rand_pct(0.01)
.label_from_df() 
.add_test(test_img)
.transform(trfm, size=128)
.databunch(path='.', bs=64, device= torch.device('cuda:0')) 
.normalize(imagenet_stats)
 )<choose_model_class>
data['FareCat']=pd.qcut(data['FareCorr'], 7, labels=['1','2','3','4','5','6','7'])
 data['FareCat'].value_counts()
learn = cnn_learner(train_img, models.resnet18, metrics=[error_rate, accuracy] )<train_model>
data[data['Embarked'].isnull() ]
lr = 3e-02
learn.fit_one_cycle(5, slice(lr))<predict_on_test>
data['Embarked'].fillna('C', inplace = True )
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
data_pre = data.drop(['Embarked','Survived','PassengerId','Name','Age','Parch','SibSp','Ticket','Fare','Cabin','Last_Name','FareCorr'],axis=1 )
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
num_attribs = []
cat_attribs = list(data_pre.drop(labels=num_attribs, axis=1 ).columns )
test_df.to_csv('submission_resnet_18.csv', index=False )<load_from_csv>
num_pipeline = Pipeline([ 
('imputer', SimpleImputer(strategy="median")) , 
('std_scaler', StandardScaler()),
] )
def load_data(dataframe=None, batch_size=1024, mode='categorical'):
if dataframe is None:
dataframe = pd.read_csv('.. /input/aerial-cactus-identification/train.csv')
 dataframe['has_cactus'] = dataframe['has_cactus'].apply(str)
 gen = ImageDataGenerator(rescale=1./255.,
validation_split=0.1, horizontal_flip=True, vertical_flip=True,rotation_range = 3,
width_shift_range = 0.1,height_shift_range = 0.1,shear_range = 0.1)
 trainGen = gen.flow_from_dataframe(dataframe, directory='.. /input/aerial-cactus-identification/train/train', x_col='id', y_col='has_cactus', has_ext=True, target_size=(32, 32),
class_mode=mode, batch_size=batch_size, shuffle=True, subset='training')
 testGen = gen.flow_from_dataframe(dataframe, directory='.. /input/aerial-cactus-identification/train/train', x_col='id', y_col='has_cactus', has_ext=True, target_size=(32, 32),
class_mode=mode, batch_size=batch_size, shuffle=True, subset='validation')
 return trainGen, testGen<choose_model_class>
cat_pipeline = Pipeline([ 
('imputer', SimpleImputer(strategy="most_frequent")) , 
('cat', OneHotEncoder()),
] )
def baseline_model() :
model = Sequential()
model.add(Conv2D(32,(3, 3), input_shape=(32, 32, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(GlobalAveragePooling2D())
 model.add(Dense(2, activation='softmax'))
return model<train_model>
full_pipeline = ColumnTransformer([ 
('cat', cat_pipeline, cat_attribs),
])
 data_post = full_pipeline.fit_transform(data_pre )
def train_baseline() :
batch_size = 1024
trainGen, valGen = load_data(batch_size=batch_size)
 model = baseline_model()
model.load_weights('.. /input/kernel658e346ac9/baseline.h5')
 opt = Adam(1e-4)
 model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
 cbs = [ReduceLROnPlateau(monitor='loss', factor=0.1, patience=1, min_lr=1e-8, verbose=1)]
model.fit_generator(trainGen, steps_per_epoch=1000, epochs=10, validation_data=valGen,
validation_steps=100, shuffle=True, callbacks=cbs)
 return model
def predict_baseline(model):
testdf = pd.read_csv('.. /input/aerial-cactus-identification/sample_submission.csv')
 pred = np.empty(( testdf.shape[0],))
for n in tqdm(range(testdf.shape[0])) :
data = np.array(Image.open('.. /input/aerial-cactus-identification/test/test/'+testdf.id[n]))
data = data.astype(np.float32)/ 255.
 pred[n] = model.predict(data.reshape(( 1, 32, 32, 3)))[0][1]
testdf['has_cactus'] = pred
testdf.to_csv('sample_submission.csv', index=False )<predict_on_test>
oneHot=OneHotEncoder()
data_post_alt=oneHot.fit_transform(data_pre[cat_attribs] )
model = train_baseline()
model.save('baseline.h5')
 predict_baseline(model )<set_options>
X_train = data_post[:len(train['Survived']),:].toarray()
y_train = train['Survived'].copy().to_numpy()
X_test = data_post[len(train['Survived']):,:].toarray()
warnings.filterwarnings('ignore')
 %matplotlib inline
print(os.listdir(".. /input")) 
<load_from_csv>
def hyperparameter_analysis(searcher, top_values=5):
tested_hyperparameters=pd.DataFrame()
for i in range(len(searcher.cv_results_['params'])) :
tested_hyperparameters = tested_hyperparameters.append(searcher.cv_results_['params'][i], ignore_index=True)
 tested_hyperparameters['train score in %']=(searcher.cv_results_['mean_train_score']*100 ).round(3)
 tested_hyperparameters['test score in %']=(searcher.cv_results_['mean_test_score']*100 ).round(3)
 tested_hyperparameters['test std in %']=(searcher.cv_results_['std_test_score']*100 ).round(3)
 tested_hyperparameters.index=searcher.cv_results_['rank_test_score']
return(tested_hyperparameters.sort_index().head(top_values))
def feature_importance(searcher_best_estimator, list_feature_names, top_values=5):
fi_norm = 1/abs(searcher_best_estimator.feature_importances_ ).sum() *abs(searcher_best_estimator.feature_importances_)
 fi_df= pd.DataFrame(data=(fi_norm*100 ).round(2), index=list_feature_names, columns=['feature importance in %'])
 return(fi_df.sort_values(['feature importance in %'], ascending=[False] ).head(top_values))
def feature_importance_ann(perm, list_feature_names, top_values=5, neg_values=False):
if neg_values==True:
fi_df= pd.DataFrame(data=(perm.feature_importances_*100 ).round(2), index=list_feature_names, columns=['score reduction in % points'])
 return(fi_df[fi_df['score reduction in % points']<0] ).sort_values(['score reduction in % points'], ascending=[True] ).head(top_values)
 else:
fi_df= pd.DataFrame(data=(perm.feature_importances_*100 ).round(2), index=list_feature_names, columns=['score reduction in % points'])
 return(fi_df.sort_values(['score reduction in % points'], ascending=[False] ).head(top_values))
def metric_scores(y_reference, y_prediction):
precision = precision_score(y_reference, y_prediction)
 recall = recall_score(y_reference, y_prediction)
 F1_score = f1_score(y_reference, y_prediction)
 print('precision: '+ str(precision.round(3)))
 print('recall: '+ str(recall.round(3)))
 print('F1_score: '+ str(F1_score.round(3)))
 return(precision, recall, F1_score)
 def plot_confusion_matrix(cm, classes, title='Confusion matrix', cmap=plt.cm.Blues):
plt.imshow(cm, interpolation='nearest', cmap=cmap)
 plt.title(title)
 tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
 plt.yticks(tick_marks, classes)
 fmt = 'd'
thresh = cm.max() / 2.
 for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])) :
plt.text(j, i, format(cm[i, j], fmt),
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
 plt.ylabel('True label')
 plt.xlabel('Predicted label')
 plt.tight_layout()
def pred_to_df(y_predicted):
survived = pd.DataFrame(y_predicted,index=test.index, columns=['Survived'])
 survived['PassengerId'] = test.PassengerId
columns_titles = ["PassengerId","Survived"]
survived=survived.reindex(columns=columns_titles)
 return(survived)
 def save_to_csv(survived, filename,index=False):
survived.to_csv(filename + '.csv',index=False, header=survived.columns)
 def plot_roc_curve(fpr, tpr, label=None):
plt.plot(fpr, tpr, linewidth=2, label=label)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.axis([0, 1, 0, 1])
 plt.xlabel('False Positive Rate', fontsize=16)
 plt.ylabel('True Positive Rate', fontsize=16)
device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 print('device: {}'.format(device))
train_folder='.. /input/train/train/'
test_folder='.. /input/test//test/'
labels=pd.read_csv('.. /input/train.csv')
 submission=pd.read_csv('.. /input/sample_submission.csv')
 print('train dataset size: {}'.format(len(listdir(train_folder))))
print('test dataset size: {}'.format(len(listdir(test_folder))))<categorify>
def create_model(input_shape=X_train.shape[1:],
number_hidden=2,
neurons_per_hidden=10,
hidden_drop_rate= 0.2,
hidden_activation = 'selu',
hidden_initializer="lecun_normal",
output_activation ='sigmoid',
loss='binary_crossentropy',
optimizer = Nadam(lr=0.0005), 
):
model = Sequential()
model.add(Input(shape=input_shape)) ,
for layer in range(number_hidden):
model.add(Dense(neurons_per_hidden, activation = hidden_activation ,kernel_initializer=hidden_initializer))
model.add(Dense(1, activation = output_activation))
model.compile(loss=loss,
optimizer = Nadam(lr=0.0005),
metrics = ['accuracy'])
 return model
class Cactus(Dataset):
def __init__(self, folder, labels, transform=None):
self.transform=transform
self.folder=folder
self.labels=labels
def __len__(self):
return self.labels.shape[0]
def __getitem__(self,index):
img_path=os.path.join(self.folder, self.labels['id'].iloc[index])
 img=Image.open(img_path)
 img_label=self.labels['has_cactus'].iloc[index]
if self.transform:
img=self.transform(img)
 return img, img_label<create_dataframe>
keras.backend.clear_session()
np.random.seed(42)
 tf.random.set_seed(42)
 dnn_clf=create_model()
history = dnn_clf.fit(X_train, y_train, epochs=30, batch_size=30, verbose=0 )
transformed_dataset=Cactus(train_folder, labels,
transform=transforms.Compose([
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.RandomAffine(10),
transforms.RandomRotation(( -10,10))
]))
visualize_samples(transformed_dataset, indices )<create_dataframe>
print('Training score: ' + str(( pd.DataFrame(history.history)['accuracy'].max() *100)) + '%' )
tfs_train=transforms.Compose([
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.RandomRotation(( -10,10)) ,
transforms.ToTensor() ,
transforms.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225))
])
 tfs_test=transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225))
])
 train_dataset=Cactus(train_folder, labels, transform=tfs_train)
 test_dataset=Cactus(test_folder, submission, transform=tfs_test)
 bs=128
data_size=len(train_dataset)
 split=int(np.floor(0.1*data_size))
data_indices=list(range(data_size))
np.random.shuffle(data_indices)
 train_indices, val_indices=data_indices[split:], data_indices[:split]
train_sampler=SubsetRandomSampler(train_indices)
 val_sampler=SubsetRandomSampler(val_indices)
 train_loader=DataLoader(train_dataset, batch_size=bs, sampler=train_sampler)
 val_loader=DataLoader(train_dataset, batch_size=bs//2, sampler=val_sampler)
 test_loader=DataLoader(test_dataset, batch_size=bs//2 )<train_model>
y_train_pred_dnn = dnn_clf.predict_classes(X_train )
def train_model(model, train_loader, val_loader, loss, optimizer, num_epoch=10):
print('Training...')
 loss_history=[]
train_history=[]
val_history=[]
val_loss_history=[]
for epoch in range(num_epoch):
model.train()
loss_accum=0
correct_samples=0
total_samples=0
for i_step,(x,y)in enumerate(train_loader):
x,y=x.to(device),y.to(device)
 prediction=model(x)
 loss_value=criterion(prediction,y)
 optimizer.zero_grad()
loss_value.backward()
optimizer.step()
_, indices=torch.max(prediction, 1)
 correct_samples+=torch.sum(indices==y ).item()
total_samples+=y.shape[0]
loss_accum+=loss_value
scheduler.step()
train_loss=loss_accum/i_step
train_accuracy=float(correct_samples)/total_samples
val_accuracy, val_loss=compute_accuracy(model, val_loader)
 loss_history.append(float(train_loss))
train_history.append(train_accuracy)
 val_history.append(val_accuracy)
 val_loss_history.append(val_loss)
 print('Epoch: {}/{}, Train_loss: {}, Train_accuracy: {}, Val_loss: {}, Val_accuracy: {}'.format(epoch+1,
num_epoch,train_loss,train_accuracy,val_loss,val_accuracy))
return loss_history, train_history, val_history, val_loss_history
def compute_accuracy(model,loader):
model.eval()
correct_samples=0
total_samples=0
loss_accum=0
with torch.no_grad() :
for i_step,(x,y)in enumerate(loader):
x,y=x.to(device),y.to(device)
 prediction=model(x)
 loss_value=criterion(prediction,y)
 _,indices=torch.max(prediction,1)
 correct_samples+=torch.sum(indices==y ).item()
total_samples+=y.shape[0]
loss_accum+=loss_value
ave_loss=loss_accum/i_step
accuracy=float(correct_samples)/total_samples
return accuracy, ave_loss<choose_model_class>
metric_scores(y_train, y_train_pred_dnn )
model=models.resnet18(pretrained=True)
 num_ftrs=model.fc.in_features
model.fc=nn.Linear(num_ftrs,2)
 model=model.to(device)
 criterion=nn.CrossEntropyLoss()
optimizer=optim.Adamax(model.parameters() , lr=0.003, weight_decay=0.001)
 scheduler=StepLR(optimizer, step_size=10, gamma=0.1)
 loss_history, train_history, val_history, val_loss_history = train_model(model, train_loader,
val_loader, criterion, optimizer, 30 )<save_to_csv>
y_train_proba_dnn = dnn_clf.predict(X_train )
model.eval()
predictions=[]
for i,(x,y)in enumerate(test_loader):
x,y=x.to(device), y.to(device)
 prediction=model(x)
 pred=prediction[:,1].detach().cpu().numpy()
for i in pred:
predictions.append(i)
 submission['has_cactus']=predictions
submission.to_csv('submission.csv',index=False )<set_options>
proba_df=pd.DataFrame(y_train_proba_dnn, columns=['probabilities'])*100
proba_df['false_predictions']=(pd.DataFrame(y_train_pred_dnn)-pd.DataFrame(y_train)) [0]
proba_df['probabilities'][proba_df['false_predictions']==0].hist()
start = time.time()
np.random.seed(10)
 %matplotlib inline
%reload_ext autoreload
%autoreload 2<load_from_csv>
check_df=data[:891].copy()
check_df['false_predictions']=proba_df['false_predictions']
check_df['probabilities']=proba_df['probabilities']
check_df[check_df['PassengerId']==18]
data_folder = Path(".. /input")
 train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )<concatenate>
np.random.seed(42)
 tf.random.set_seed(42)
 dnn_clf_fi=KerasClassifier(build_fn = create_model)
 dnn_clf_fi.fit(X_train, y_train, epochs=30, batch_size=30)
 perm = PermutationImportance(dnn_clf_fi, random_state=42 ).fit(X_train,y_train)
df1 = train_df[train_df.has_cactus==0].copy()
df2 = df1.copy()
train_df = train_df.append([df1, df2], ignore_index=True )<train_model>
feature_importance_ann(perm, feature_names, top_values=10, neg_values=False )
test_img = ImageList.from_df(test_df, path=data_folder/'test', folder='test')
 trfm = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1,
max_lighting=0.2, max_warp=0.2,
p_affine=0.75, p_lighting=0.75 ,xtra_tfms=[contrast(scale=(0.5, 1), p=0.75)])
 imagesize = 64
batchsize = 64
def train_data(imagesize,batchsize):
train_img =(ImageList.from_df(train_df, path=data_folder/'train', folder='train')
.split_by_rand_pct(0.2,seed =344)
.label_from_df() 
.add_test(test_img)
.transform(trfm, size=imagesize)
.databunch(path='.', bs=batchsize, device= torch.device('cuda:0')) 
.normalize(imagenet_stats)
)
 return train_img<define_variables>
feature_importance_ann(perm, feature_names, top_values=20, neg_values=True )
data = train_data(imagesize,batchsize )<train_model>
k = 17
sel = SelectFromModel(perm, threshold=-np.inf, max_features=k, prefit=True)
 best_festures_index=np.where(sel.get_support() ==True)
 X_train_bf = data_post[:len(train['Survived']),best_festures_index[0]].copy()
X_test_bf = data_post[len(train['Survived']):,best_festures_index[0]].copy()
train_img1 = train_data(64,64)
<define_variables>
y_pred_dnn = dnn_clf.predict_classes(X_test )
train_img1.show_batch(rows=5, figsize=(10,10)) 
<choose_model_class>
my_submission=pred_to_df(y_pred_dnn)
 my_submission.head()
nmodels = 3<choose_model_class>
save_to_csv(my_submission,'submission' )
def get_ensemble(nmodels):
ens_model = []
learning_rate =[2.95e-02,3e-02,3e-02]
model_list = [models.densenet161,models.densenet161,models.densenet201]
for i in range(nmodels):
print(f'-----Training model: {i+1}--------')
 data = train_data(64,64)
 learn_resnet = cnn_learner(data, model_list[i], metrics=[error_rate, accuracy,AUROC() ],
model_dir="/tmp/model/")
 print('training for 128x128')
 learn_resnet.set_data = train_data(128,64)
 learn_resnet.lr_find()
learn_resnet.recorder.plot(suggestion=True)
 learn_resnet.fit_one_cycle(3,slice(learning_rate[i]))
print('training for 150x150')
 learn_resnet.set_data = train_data(150,96)
 learn_resnet.fit_one_cycle(3,slice(learning_rate[i]))
learn_resnet.save(f'ensem_model_{i}.weights')
 ens_model.append(learn_resnet)
 print(f'-----Training of model {i+1} complete----')
 return ens_model<choose_model_class>
X_train_gs=X_train.copy()
X_test_gs=X_test.copy()
ens = get_ensemble(3 )<train_model>
keras.backend.clear_session()
np.random.seed(42)
 tf.random.set_seed(42)
 dnn_clf_gs = KerasClassifier(build_fn = create_model, verbose = 0)
 param_grid = {
'input_shape': X_train_gs.shape[1:]
}
grid_search_dnn = GridSearchCV(dnn_clf_gs, param_grid, cv=5, n_jobs=-1, verbose=0, return_train_score=True)
 training_gs = grid_search_dnn.fit(X_train_gs, y_train,
epochs = 30,
batch_size = 30,
validation_split=0.2, shuffle=True, 
)
end = time.time()
print(end - start )<categorify>
print('Mean training score: ' + str(( grid_search_dnn.cv_results_['mean_train_score'][grid_search_dnn.best_index_]*100 ).round(2)) +'%(' + str(( grid_search_dnn.cv_results_['std_train_score'][grid_search_dnn.best_index_]*100 ).round(2)) + '%)')
 print('Mean validation score: ' + str(( grid_search_dnn.cv_results_['mean_test_score'][grid_search_dnn.best_index_]*100 ).round(2)) +'%(' + str(( grid_search_dnn.cv_results_['std_test_score'][grid_search_dnn.best_index_]*100 ).round(2)) + '%)' )
ens_test_preds = []
for mdl in ens:
preds,_ = mdl.TTA(ds_type=DatasetType.Test)
 print(np.array(preds ).shape)
 ens_test_preds.append(np.array(preds))<predict_on_test>
hyperparameter_analysis(grid_search_dnn, 10 )
ens_preds = np.mean(ens_test_preds, axis =0 )<predict_on_test>
y_pred_dnn_opti = grid_search_dnn.best_estimator_.predict(X_test_gs )
test_df.has_cactus = ens_preds[:, 0]
test_df.head() 
<save_to_csv>
survived_dnn_opti=pred_to_df(y_pred_dnn_opti)
 survived_dnn_opti.head()
test_df.to_csv('submission.csv', index=False )<categorify>
save_to_csv(survived_dnn_opti,'survived_dnn_opti' )
interp = ClassificationInterpretation.from_learner(ens[0])
 interp1 = ClassificationInterpretation.from_learner(ens[1])
 interp2 = ClassificationInterpretation.from_learner(ens[2])
<define_variables>
param_distribs = {
'n_estimators': randint(low=50, high=150),
'max_features': randint(low=5, high=15),
'min_samples_split': randint(low=10, high=30),
}
forest_clf = RandomForestClassifier(random_state=42)
 rnd_search_rf = RandomizedSearchCV(forest_clf, param_distributions=param_distribs, n_jobs=-1,
n_iter=50, cv=5, scoring="accuracy", random_state=42, return_train_score=True)
 rnd_search_rf.fit(X_train, y_train )
train_imgs_path = '.. /input/train/train'
test_imgs_path = '.. /input/test/test'
labels_path = '.. /input/train.csv'
in_path = '.. /input/'<load_from_csv>
print('Best score: ' + str(( rnd_search_rf.best_score_*100 ).round(2)) + '%' )
df = pd.read_csv(labels_path)
 df['id'] = 'train/train/' + df['id']
df.head()<categorify>
print('Mean training score: ' + str(( rnd_search_rf.cv_results_['mean_train_score'][rnd_search_rf.best_index_]*100 ).round(2)) +'%(' + str(( rnd_search_rf.cv_results_['std_train_score'][0]*100 ).round(2)) + '%)')
 print('Mean validation score: ' + str(( rnd_search_rf.cv_results_['mean_test_score'][rnd_search_rf.best_index_]*100 ).round(2)) +'%(' + str(( rnd_search_rf.cv_results_['std_test_score'][0]*100 ).round(2)) + '%)' )
data = vision.ImageDataBunch.from_df(in_path, df, ds_tfms=vision.get_transforms() , size=224)
 data = data.normalize(vision.imagenet_stats )<define_variables>
pd.DataFrame(rnd_search_rf.best_params_.items() , columns=['hyperparameter','value'] )
data.show_batch(rows=3, figsize=(10, 8))<train_on_grid>
hyperparameter_analysis(rnd_search_rf )
learn = vision.cnn_learner(data, vision.models.resnet34, metrics=metrics.accuracy)
 learn.fit(2 )<find_best_params>
feature_importance(rnd_search_rf.best_estimator_, feature_names, 10 )
interp = vision.ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()<load_from_csv>
final_rf_clf = rnd_search_rf.best_estimator_
final_rf_clf
submission_df = pd.read_csv('.. /input/sample_submission.csv')
 files = submission_df['id'].values
img_paths =('.. /input/test/test/' + submission_df['id'] ).values<predict_on_test>
y_train_pred_rf = final_rf_clf.predict(X_train )
preds = []
for p in tqdm(img_paths):
pred = learn.predict(vision.open_image(p)) [-1].numpy()
preds.append(pred )<prepare_output>
metric_scores(y_train, y_train_pred_rf )
submission_df['has_cactus'] = np.array(preds)[:, 1]
submission_df.head()<save_to_csv>
y_pred_rf = final_rf_clf.predict(X_test )
submission_df.to_csv('submission.csv', index=False )<import_modules>
survived_rf=pred_to_df(y_pred_rf)
 survived_rf.head()
resnet_weights_path = '.. /input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'
print(os.listdir(".. /input"))<categorify>
save_to_csv(survived_rf,'survived_rf' )
def prep_cnn_data(df, n_x, n_c, path):

 tensors = np.zeros(( df.shape[0], n_x, n_x, n_c))
for i in range(df.shape[0]):
pic = load_img(path+df.iloc[i]['id'])
 pic_array = img_to_array(pic)
 tensors[i,:] = pic_array
tensors = tensors / 255.
 return tensors<load_from_csv>
param_distribs = {
'max_depth': randint(low=1, high=5),
'n_estimators': randint(low=5, high=120),
'max_features': randint(low=5, high=15),
}
gb_clf = GradientBoostingClassifier()
rnd_search_gb = RandomizedSearchCV(gb_clf, param_distributions=param_distribs,
n_iter=40, cv=5, scoring="accuracy", random_state=42, return_train_score=True)
 rnd_search_gb.fit(X_train, y_train )
train_df = pd.read_csv('.. /input/aerial-cactus-identification/train.csv')
 test_df = pd.read_csv('.. /input/aerial-cactus-identification/sample_submission.csv')
 train = prep_cnn_data(train_df, 32, 3, path='.. /input/aerial-cactus-identification/train/train/')
 train_y = train_df['has_cactus'].values
test = prep_cnn_data(test_df, 32, 3, path='.. /input/aerial-cactus-identification/test/test/' )<choose_model_class>
print('Mean training score: ' + str(( rnd_search_gb.cv_results_['mean_train_score'][rnd_search_gb.best_index_]*100 ).round(2)) +'%(' + str(( rnd_search_gb.cv_results_['std_train_score'][0]*100 ).round(2)) + '%)')
 print('Mean validation score: ' + str(( rnd_search_gb.cv_results_['mean_test_score'][rnd_search_gb.best_index_]*100 ).round(2)) +'%(' + str(( rnd_search_gb.cv_results_['std_test_score'][0]*100 ).round(2)) + '%)' )
datagen = ImageDataGenerator(
 zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=True)
<drop_column>
rnd_search_gb.cv_results_['mean_train_score']
del model<choose_model_class>
hyperparameter_analysis(rnd_search_gb, 5 )
model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(32,32,3)))
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.5))
model.add(Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.5))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = "sigmoid"))<choose_model_class>
feature_importance(rnd_search_gb.best_estimator_, feature_names, 10 )
model.compile(optimizer = 'adam', loss = "binary_crossentropy", metrics=["accuracy"] )<train_model>
final_gb_clf = rnd_search_gb.best_estimator_
final_gb_clf
batch_size=256
epochs=200
x_train=train[3500:]
y_train=train_y[3500:]
x_val=train[:3500]
y_val=train_y[:3500]
red_lr= ReduceLROnPlateau(monitor='val_acc',patience=3,verbose=1,factor=0.7)
 History = model.fit_generator(datagen.flow(x_train,y_train,batch_size=batch_size),
epochs= epochs, steps_per_epoch=x_train.shape[0]//batch_size,
validation_data=(x_val,y_val),callbacks=[red_lr] )<save_to_csv>
y_train_pred_gb = final_gb_clf.predict(X_train )
pred=model.predict(test)
 test_csv = pd.read_csv('.. /input/aerial-cactus-identification/sample_submission.csv')
 df=pd.DataFrame({'id':test_csv['id'] })
 df['has_cactus']=pred
df.to_csv("submission.csv",index=False )<set_options>
metric_scores(y_train, y_train_pred_gb )
%matplotlib inline<set_options>
y_pred_gb = final_gb_clf.predict(X_test )
%matplotlib inline<set_options>
survived_gb=pred_to_df(y_pred_gb)
 survived_gb.head()
<load_from_csv><EOS> 
save_to_csv(survived_gb,'survived_gb' )