Kaggle competition: Titanic - Machine Learning from Disaster
validation_split = 0.8
idxs = np.random.permutation(range(len(labels)))< validation_split*len(labels)
 train_labels = labels[idxs]
val_labels = labels[~idxs]
print(len(train_labels), len(val_labels))<define_variables>
def warn(*args, **kwargs):
pass
warnings.warn = warn
train_datagen = keras.preprocessing.image.ImageDataGenerator(rescale=1/255, horizontal_flip=True, vertical_flip=True)
 batch_size = 128
train_generator = train_datagen.flow_from_dataframe(train_labels,directory=train_dir,x_col='id',
y_col='has_cactus',class_mode='binary',batch_size=batch_size,
target_size=(32,32))
val_generator = train_datagen.flow_from_dataframe(val_labels,directory=train_dir,x_col='id',
y_col='has_cactus',class_mode='binary',batch_size=batch_size,
target_size=(32,32))<choose_model_class>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )
input_shape =(32, 32, 3)
 model = keras.models.Sequential()
model.add(Conv2D(32,(3, 3), padding='same', activation='relu', input_shape=input_shape))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Conv2D(64,(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Conv2D(128,(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Conv2D(128,(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Flatten())
 model.add(Dense(512, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.summary() 
<choose_model_class>
cols = ['Survived', 'Sex', 'Pclass', 'SibSp', 'Parch', 'Embarked']
model.compile(loss = keras.losses.binary_crossentropy,
optimizer = 'adam',
metrics = ['acc'])
 callbacks = [EarlyStopping(monitor='val_loss', patience=20, verbose=1, restore_best_weights=True),
ReduceLROnPlateau(patience=10, verbose=1),
]<train_model>
cm_surv = ["darkgrey" , "lightgreen"]
epochs = 100
history = model.fit(train_generator,
epochs = epochs,
verbose = 1,
callbacks = callbacks,
validation_data = val_generator,
)<find_best_params>
for df in [df_train, df_test] :
df['FamilySize'] = df['SibSp'] + df['Parch'] +1
df['Alone']=0
df.loc[(df.FamilySize==1),'Alone'] = 1
df['NameLen'] = df.Name.apply(lambda x : len(x))
df['NameLenBin']=np.nan
for i in range(20,0,-1):
df.loc[ df['NameLen'] <= i*5, 'NameLenBin'] = i
df['Title']=0
df['Title']=df.Name.str.extract(r'([A-Za-z]+)\.')
 df['Title'].replace(['Mlle','Mme','Ms','Dr','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don'],
['Miss','Miss','Miss','Mr','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr'],inplace=True )
idx = np.argmax(history.history['val_acc'])
 print(history.history['val_loss'][idx], history.history['val_acc'][idx])
 idx = np.argmin(history.history['val_loss'])
 print(history.history['val_loss'][idx], history.history['val_acc'][idx] )<predict_on_test>
grps_namelenbin_survrate = df_train.groupby(['NameLenBin'])['Survived'].mean().to_frame()
grps_namelenbin_survrate
test_datagen = keras.preprocessing.image.ImageDataGenerator(rescale = 1/255)
 test_generator = test_datagen.flow_from_directory(
 directory = test_dir,
target_size =(32, 32),
batch_size = 1,
class_mode = None,
shuffle = False)
 probabilities = model.predict(test_generator )<save_to_csv>
grps_title_survrate = df_train.groupby(['Title'])['Survived'].mean().to_frame()
grps_title_survrate
sample_submission = pd.read_csv('.. /input/aerial-cactus-identification/sample_submission.csv')
 df = pd.DataFrame({'id': sample_submission['id']})
 df['has_cactus'] = probabilities
df.to_csv("submission.csv", index=False )<load_from_disk>
for df in [df_train, df_test]:
df['Title'] = df['Title'].fillna(df['Title'].mode().iloc[0])
 df.loc[(df.Age.isnull())&(df.Title=='Mr'),'Age']= df.Age[df.Title=="Mr"].mean()
df.loc[(df.Age.isnull())&(df.Title=='Mrs'),'Age']= df.Age[df.Title=="Mrs"].mean()
df.loc[(df.Age.isnull())&(df.Title=='Master'),'Age']= df.Age[df.Title=="Master"].mean()
df.loc[(df.Age.isnull())&(df.Title=='Miss'),'Age']= df.Age[df.Title=="Miss"].mean()
df.loc[(df.Age.isnull())&(df.Title=='Other'),'Age']= df.Age[df.Title=="Other"].mean()
df = df.drop('Name', axis=1)
!unzip /kaggle/input/aerial-cactus-identification/train.zip
!unzip /kaggle/input/aerial-cactus-identification/test.zip<set_options>
df_train['Embarked'] = df_train['Embarked'].fillna(df_train['Embarked'].mode().iloc[0])
 df_test['Embarked'] = df_test['Embarked'].fillna(df_test['Embarked'].mode().iloc[0])
 df_train['Fare'] = df_train['Fare'].fillna(df_train['Fare'].mean())
 df_test['Fare'] = df_test['Fare'].fillna(df_test['Fare'].mean() )
%matplotlib inline 
<import_modules>
for df in [df_train, df_test]:
df['Age_bin']=np.nan
for i in range(8,0,-1):
df.loc[ df['Age'] <= i*10, 'Age_bin'] = i
df['Fare_bin']=np.nan
for i in range(12,0,-1):
df.loc[ df['Fare'] <= i*50, 'Fare_bin'] = i
df['Title'] = df['Title'].map({'Other':0, 'Mr': 1, 'Master':2, 'Miss': 3, 'Mrs': 4 })
 df['Title'] = df['Title'].fillna(df['Title'].mode().iloc[0])
 df['Title'] = df['Title'].astype(int )
print(sys.version)
 print('tensorflow -> ', tf.__version__ )<define_variables>
df_train_ml = df_train.copy()
df_test_ml = df_test.copy()
passenger_id = df_test_ml['PassengerId']
np.random.seed(12)
 tf.random.set_seed(12 )<load_from_csv>
df_train_ml = pd.get_dummies(df_train_ml, columns=['Sex', 'Embarked', 'Pclass'], drop_first=True)
 df_test_ml = pd.get_dummies(df_test_ml, columns=['Sex', 'Embarked', 'Pclass'], drop_first=True)
 df_train_ml.drop(['PassengerId','Name','Ticket', 'Cabin', 'Age', 'Fare_bin'],axis=1,inplace=True)
 df_test_ml.drop(['PassengerId','Name','Ticket', 'Cabin', 'Age', 'Fare_bin'],axis=1,inplace=True)
main_df = pd.read_csv('/kaggle/input/aerial-cactus-identification/train.csv')
 sub_df = pd.read_csv('/kaggle/input/aerial-cactus-identification/sample_submission.csv')
 train_dir = '/kaggle/working/train/'
test_dir = '/kaggle/working/test/'<count_values>
df_train_ml.dropna(inplace=True )
print('shape: ', main_df.shape)
 print('===================================')
 print(main_df['has_cactus'].value_counts() )<split>
for df in [df_train_ml, df_test_ml]:
df.drop(['NameLen'], axis=1, inplace=True)
 df.drop(['SibSp'], axis=1, inplace=True)
 df.drop(['Parch'], axis=1, inplace=True)
 df.drop(['Alone'], axis=1, inplace=True )
train_df, val_df = train_test_split(main_df, test_size=0.25, stratify=main_df['has_cactus'], shuffle=True, random_state=12)
 train_df = train_df.reset_index()
val_df = val_df.reset_index()
total_train = train_df.shape[0]
total_val = val_df.shape[0]
print('total_train: {}, total_val: {}'.format(total_train, total_val))<categorify>
df_test_ml.fillna(df_test_ml.mean() , inplace=True)
 df_test_ml.head()
img_width, img_height = 32, 32
target_size =(img_width, img_height)
 train_datagen = ImageDataGenerator(rescale=1./255)
 val_datagen = ImageDataGenerator(rescale=1./255)
 test_datagen = ImageDataGenerator(rescale=1./255)
 train_df['has_cactus'] = train_df['has_cactus'].astype(str)
 val_df['has_cactus'] = val_df['has_cactus'].astype(str )<create_dataframe>
scaler = StandardScaler()
scaler.fit(df_train_ml.drop(['Survived'],axis=1))
scaled_features = scaler.transform(df_train_ml.drop(['Survived'],axis=1))
df_train_ml_sc = pd.DataFrame(scaled_features)
 df_test_ml.fillna(df_test_ml.mean() , inplace=True)
 scaled_features = scaler.transform(df_test_ml)
 df_test_ml_sc = pd.DataFrame(scaled_features )
batch_size = 32
x_col, y_col = 'id', 'has_cactus'
class_mode = 'binary'
train_gen = train_datagen.flow_from_dataframe(train_df,
train_dir,
x_col=x_col,
y_col=y_col,
class_mode=class_mode,
target_size=target_size,
batch_size=batch_size, 
)
 val_gen = val_datagen.flow_from_dataframe(val_df,
train_dir,
x_col=x_col,
y_col=y_col,
class_mode=class_mode,
target_size=target_size,
batch_size=batch_size,
)<choose_model_class>
X = df_train_ml.drop('Survived', axis=1)
 y = df_train_ml['Survived']
X_test = df_test_ml
X_sc = df_train_ml_sc
y_sc = df_train_ml['Survived']
X_test_sc = df_test_ml_sc
input_shape =(img_width, img_height, 3)
 optimizer = optimizers.Adam(lr=1e-3 )<choose_model_class>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn import tree
from sklearn.metrics import accuracy_score
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(3,3), padding='same', activation='relu', input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64, kernel_size=(3,3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size=(3,3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(GlobalMaxPooling2D())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', metrics=['acc'], optimizer=optimizer)
 model.summary()<normalization>
from sklearn.model_selection import cross_val_score
def step_decay(epoch):
initial_rate = 0.001
drop = 0.5
epochs_drop = 10.0
lrate = initial_rate * math.pow(drop, math.floor(( epoch)/ epochs_drop))
return lrate<choose_model_class>
svc = SVC(gamma = 0.01, C = 100)
 scores_svc = cross_val_score(svc, X, y, cv=10, scoring='accuracy')
 print(scores_svc)
 print(scores_svc.mean() )
lrate = LearningRateScheduler(step_decay)
 es = EarlyStopping(monitor='val_loss', min_delta=0, patience=5)
 callbacks = [lrate, es]<train_model>
svc = SVC(gamma = 0.01, C = 100)
 scores_svc_sc = cross_val_score(svc, X_sc, y_sc, cv=10, scoring='accuracy')
 print(scores_svc_sc)
 print(scores_svc_sc.mean() )
epochs = 30
history = model.fit(
 train_gen,
epochs=epochs,
steps_per_epoch=total_train//batch_size,
validation_data=val_gen,
validation_steps=total_val//batch_size,
callbacks=callbacks,
)<predict_on_test>
rfc = RandomForestClassifier(max_depth=5, max_features=6)
 scores_rfc = cross_val_score(rfc, X, y, cv=10, scoring='accuracy')
 print(scores_rfc)
 print(scores_rfc.mean() )
def predict(model, sub_df):
pred = np.empty(( sub_df.shape[0],))
for n in tqdm(range(sub_df.shape[0])) :
image = np.array(Image.open(test_dir + sub_df.id[n]))
pred[n] = model.predict(image.reshape(( 1, 32, 32, 3)) /255.0)[0]
sub_df['has_cactus'] = pred
return sub_df<predict_on_test>
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
from scipy.stats import uniform
predictions = predict(model, sub_df )<install_modules>
model = SVC()
param_grid = {'C':uniform(0.1, 5000), 'gamma':uniform(0.0001, 1)}
rand_SVC = RandomizedSearchCV(model, param_distributions=param_grid, n_iter=100)
 rand_SVC.fit(X_sc,y_sc)
 score_rand_SVC = get_best_score(rand_SVC )
!rm -r *<save_to_csv>
param_grid = {'C': [0.1,10, 100, 1000,5000], 'gamma': [1,0.1,0.01,0.001,0.0001], 'kernel': ['rbf']}
svc_grid = GridSearchCV(SVC() , param_grid, cv=10, refit=True, verbose=1)
 svc_grid.fit(X_sc,y_sc)
 sc_svc = get_best_score(svc_grid )
predictions.to_csv('submission.csv', header=True, index=False )<set_options>
pred_all_svc = svc_grid.predict(X_test_sc)
 sub_svc = pd.DataFrame()
sub_svc['PassengerId'] = df_test['PassengerId']
sub_svc['Survived'] = pred_all_svc
sub_svc.to_csv('svc.csv',index=False )
%matplotlib inline
random.seed(0)
<import_modules>
knn = KNeighborsClassifier()
leaf_range = list(range(3, 15, 1))
k_range = list(range(1, 15, 1))
weight_options = ['uniform', 'distance']
param_grid = dict(leaf_size=leaf_range, n_neighbors=k_range, weights=weight_options)
 print(param_grid)
 knn_grid = GridSearchCV(knn, param_grid, cv=10, verbose=1, scoring='accuracy')
 knn_grid.fit(X, y)
 sc_knn = get_best_score(knn_grid )
print("python", sys.version)
 for module in np, pd, tf, keras:
print(module.__name__, module.__version__ )<import_modules>
pred_all_knn = knn_grid.predict(X_test)
 sub_knn = pd.DataFrame()
sub_knn['PassengerId'] = df_test['PassengerId']
sub_knn['Survived'] = pred_all_knn
sub_knn.to_csv('knn.csv',index=False )
assert sys.version_info >=(3, 5)
 assert tf.__version__ >= "2.0"<load_from_csv>
dtree = DecisionTreeClassifier()
param_grid = {'min_samples_split': [4,7,10,12]}
dtree_grid = GridSearchCV(dtree, param_grid, cv=10, refit=True, verbose=1)
 dtree_grid.fit(X_sc,y_sc)
 print(dtree_grid.best_score_)
 print(dtree_grid.best_params_)
 print(dtree_grid.best_estimator_ )
train_dir = ".. /input/cactus-dataset/cactus/train.csv"
train_data = pd.read_csv(train_dir)
 train_data.has_cactus = train_data.has_cactus.astype(str )<count_values>
pred_all_dtree = dtree_grid.predict(X_test_sc)
 sub_dtree = pd.DataFrame()
sub_dtree['PassengerId'] = df_test['PassengerId']
sub_dtree['Survived'] = pred_all_dtree
sub_dtree.to_csv('dtree.csv',index=False )
train_data.has_cactus.value_counts()<load_pretrained>
rfc = RandomForestClassifier()
param_grid = {'max_depth': [3, 5, 6, 7, 8], 'max_features': [6,7,8,9,10],
'min_samples_split': [5, 6, 7, 8]}
rf_grid = GridSearchCV(rfc, param_grid, cv=10, refit=True, verbose=1)
 rf_grid.fit(X_sc,y_sc)
 sc_rf = get_best_score(rf_grid )
img = mpimg.imread(".. /input/cactus-dataset/cactus/train/000c8a36845c0208e833c79c1bffedd1.jpg")
 plt.axis("off")
 imgplot = mlp.imshow(img )<create_dataframe>
pred_all_rf = rf_grid.predict(X_test_sc)
 sub_rf = pd.DataFrame()
sub_rf['PassengerId'] = df_test['PassengerId']
sub_rf['Survived'] = pred_all_rf
sub_rf.to_csv('rf.csv',index=False )
def generator(train_data, directory, batch_size, target_size, class_mode):
x_col = 'id'
y_col = 'has_cactus'
train_datagen = ImageDataGenerator(
 rescale = 1./255,
horizontal_flip = True,
vertical_flip = True,
validation_split = 0.2)
 train_generator = train_datagen.flow_from_dataframe(
 train_data,
directory = directory,
x_col = x_col,
y_col = y_col,
target_size = target_size,
class_mode = class_mode,
batch_size = batch_size,
shuffle = True,
subset = 'training')
 valid_generator = train_datagen.flow_from_dataframe(
 train_data,
directory = directory,
x_col = x_col,
y_col = y_col,
target_size = target_size,
class_mode = class_mode,
batch_size = batch_size,
shuffle = True,
subset = 'validation')
 return train_generator, valid_generator<define_variables>
extr = ExtraTreesClassifier()
param_grid = {'max_depth': [6,7,8,9], 'max_features': [7,8,9,10],
'n_estimators': [50, 100, 200]}
extr_grid = GridSearchCV(extr, param_grid, cv=10, refit=True, verbose=1)
 extr_grid.fit(X_sc,y_sc)
 sc_extr = get_best_score(extr_grid )
directory = ".. /input/cactus-dataset/cactus/train"
batch_size = 64
target_size =(32,32)
 class_mode = 'binary'
train_generator, valid_generator = generator(train_data, directory, batch_size, target_size, class_mode )<import_modules>
pred_all_extr = extr_grid.predict(X_test_sc)
 sub_extr = pd.DataFrame()
sub_extr['PassengerId'] = df_test['PassengerId']
sub_extr['Survived'] = pred_all_extr
sub_extr.to_csv('extr.csv',index=False )
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, Dense, Flatten, Dropout, Activation
from tensorflow.keras.layers import BatchNormalization, MaxPooling2D, GlobalAveragePooling2D<choose_model_class>
gbc = GradientBoostingClassifier()
param_grid = {'n_estimators': [50, 100],
'min_samples_split': [3, 4, 5, 6, 7],
'max_depth': [3, 4, 5, 6]}
gbc_grid = GridSearchCV(gbc, param_grid, cv=10, refit=True, verbose=1)
 gbc_grid.fit(X_sc,y_sc)
 sc_gbc = get_best_score(gbc_grid )
model = Sequential([
Conv2D(32,(3, 3), padding = 'same', activation = 'relu', input_shape =(32,32,3)) ,
BatchNormalization() ,
Conv2D(32,(3, 3), padding = 'same', activation = 'relu', input_shape =(32,32,3)) ,
BatchNormalization() ,
MaxPooling2D() ,
Conv2D(64,(3, 3), padding = 'same', activation = 'relu'),
BatchNormalization() ,
MaxPooling2D() ,
Conv2D(128,(3, 3), padding = 'same', activation = 'relu'),
BatchNormalization() ,
MaxPooling2D() ,
Conv2D(256,(3, 3), padding = 'same', activation = 'relu'),
BatchNormalization() ,
MaxPooling2D() ,
GlobalAveragePooling2D() ,
Dense(256, activation = 'relu'),
Dropout(0.5),
Dense(1, activation = 'sigmoid')
 ])
 model.compile(loss = 'binary_crossentropy',
optimizer = 'adam',
metrics = ['accuracy'])
 model.summary()<choose_model_class>
pred_all_gbc = gbc_grid.predict(X_test_sc)
 sub_gbc = pd.DataFrame()
sub_gbc['PassengerId'] = df_test['PassengerId']
sub_gbc['Survived'] = pred_all_gbc
sub_gbc.to_csv('gbc.csv',index=False )
<train_model>
xgb = XGBClassifier()
param_grid = {'max_depth': [5,6,7,8], 'gamma': [1, 2, 4], 'learning_rate': [0.1, 0.2, 0.3, 0.5]}
with ignore_warnings(category=DeprecationWarning):
xgb_grid = GridSearchCV(xgb, param_grid, cv=10, refit=True, verbose=1)
 xgb_grid.fit(X_sc,y_sc)
 sc_xgb = get_best_score(xgb_grid )
class_weights = class_weight.compute_class_weight('balanced', np.unique(train_generator.classes), train_generator.classes)
 callbacks = [EarlyStopping(monitor = 'val_loss', patience = 20),
ReduceLROnPlateau(patience = 10, verbose = 1),
ModelCheckpoint(filepath = 'best_model.h5', monitor = 'val_loss', verbose = 0, save_best_only = True)]
history = model.fit_generator(train_generator,
validation_data = valid_generator,
epochs = 100,
verbose = 1,
shuffle = True,
callbacks = callbacks,
class_weight = class_weights )<load_pretrained>
with ignore_warnings(category=DeprecationWarning):
pred_all_xgb = xgb_grid.predict(X_test_sc)
 sub_xgb = pd.DataFrame()
sub_xgb['PassengerId'] = df_test['PassengerId']
sub_xgb['Survived'] = pred_all_xgb
sub_xgb.to_csv('xgb.csv',index=False )
model.load_weights("best_model.h5" )<define_variables>
ada = AdaBoostClassifier()
param_grid = {'n_estimators': [30, 50, 100], 'learning_rate': [0.08, 0.1, 0.2]}
ada_grid = GridSearchCV(ada, param_grid, cv=10, refit=True, verbose=1)
 ada_grid.fit(X_sc,y_sc)
 sc_ada = get_best_score(ada_grid)
 pred_all_ada = ada_grid.predict(X_test_sc )
def test_gen(test_dir, target_size, batch_size, class_mode):
test_datagen = ImageDataGenerator(
 rescale = 1./255)
 test_generator = test_datagen.flow_from_directory(
 directory = test_dir,
target_size = target_size,
batch_size = batch_size,
class_mode = class_mode,
shuffle = False)
 return test_generator<define_variables>
sub_ada = pd.DataFrame()
sub_ada['PassengerId'] = df_test['PassengerId']
sub_ada['Survived'] = pred_all_ada
sub_ada.to_csv('ada.csv',index=False )
test_dir = ".. /input//cactus-dataset/cactus/test/"
target_size =(32,32)
 batch_size = 1
class_mode = None
test_generator = test_gen(test_dir, target_size, batch_size, class_mode )<save_to_csv>
cat=CatBoostClassifier()
param_grid = {'iterations': [100, 150], 'learning_rate': [0.3, 0.4, 0.5], 'loss_function' : ['Logloss']}
cat_grid = GridSearchCV(cat, param_grid, cv=10, refit=True, verbose=1)
 cat_grid.fit(X_sc,y_sc, verbose=False)
 sc_cat = get_best_score(cat_grid)
 pred_all_cat = cat_grid.predict(X_test_sc )
def submission() :
sample_submission = pd.read_csv(".. /input/cactus-dataset/cactus/sample_submission.csv")
 filenames = [path.split('/')[-1] for path in test_generator.filenames]
proba = list(model.predict_generator(test_generator)[:,0])
 sample_submission.id = filenames
sample_submission.has_cactus = proba
sample_submission.to_csv('submission.csv', index=False)
 return sample_submission
sample_submission = submission()
sample_submission.head()<import_modules>
sub_cat = pd.DataFrame()
sub_cat['PassengerId'] = df_test['PassengerId']
sub_cat['Survived'] = pred_all_cat
sub_cat['Survived'] = sub_cat['Survived'].astype(int)
 sub_cat.to_csv('cat.csv',index=False )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import keras
import os
import cv2
from PIL import Image
from IPython.display import FileLink<load_from_csv>
lgbm = lgb.LGBMClassifier(silent=False)
 param_grid = {"max_depth": [8,10,15], "learning_rate" : [0.008,0.01,0.012],
"num_leaves": [80,100,120], "n_estimators": [200,250] }
lgbm_grid = GridSearchCV(lgbm, param_grid, cv=10, refit=True, verbose=1)
 lgbm_grid.fit(X_sc,y_sc, verbose=True)
 sc_lgbm = get_best_score(lgbm_grid)
 pred_all_lgbm = lgbm_grid.predict(X_test_sc )
dataset = pd.read_csv(".. /input/train.csv")
 dataset.head()<groupby>
sub_lgbm = pd.DataFrame()
sub_lgbm['PassengerId'] = df_test['PassengerId']
sub_lgbm['Survived'] = pred_all_lgbm
sub_lgbm.to_csv('lgbm.csv',index=False )
grouped_dataset = dataset.groupby("has_cactus")
 grouped_dataset.count()<count_values>
from sklearn.ensemble import VotingClassifier
dataset.count()<prepare_x_and_y>
clf1 = LogisticRegression(random_state=1)
 clf2 = RandomForestClassifier(random_state=1)
 clf3 = GaussianNB()
eclf = VotingClassifier(estimators=[('lr', clf1),('rf', clf2),('gnb', clf3)], voting='soft')
 params = {'lr__C': [1.0, 100.0], 'rf__n_estimators': [20, 200],}
with ignore_warnings(category=DeprecationWarning):
votingclf_grid = GridSearchCV(estimator=eclf, param_grid=params, cv=10)
 votingclf_grid.fit(X_sc,y_sc)
 sc_vot1 = get_best_score(votingclf_grid )
def datagen(dataset=dataset, path=".. /input/train/train/"):
x = np.ones(( 17500, 224, 224, 3), dtype=np.uint8)
 y = np.ones(17500)
 counter = 0
for rec in dataset.values:
img = cv2.imread(path + rec[0])
 img = cv2.resize(img,(32, 32))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = cv2.copyMakeBorder(img, 96, 96, 96, 96, cv2.BORDER_REFLECT)
 x[counter] = img
y[counter] = rec[1]
counter += 1
permutation = np.random.permutation(x.shape[0])
 x = x[permutation]
y = y[permutation]
return x, y<choose_model_class>
clf4 = GradientBoostingClassifier()
clf5 = SVC()
clf6 = RandomForestClassifier()
eclf_2 = VotingClassifier(estimators=[('gbdt', clf4), 
('svc', clf5), 
('rf', clf6)], voting='soft')
 params = {'gbdt__n_estimators': [50], 'gbdt__min_samples_split': [3],
'svc__C': [10, 100] , 'svc__gamma': [0.1,0.01,0.001] , 'svc__kernel': ['rbf'] , 'svc__probability': [True],
'rf__max_depth': [7], 'rf__max_features': [2,3], 'rf__min_samples_split': [3] }
with ignore_warnings(category=DeprecationWarning):
votingclf_grid_2 = GridSearchCV(estimator=eclf_2, param_grid=params, cv=10)
 votingclf_grid_2.fit(X_sc,y_sc)
 sc_vot2_cv = get_best_score(votingclf_grid_2 )
densenet = keras.applications.densenet.DenseNet169(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000 )<choose_model_class>
with ignore_warnings(category=DeprecationWarning):
pred_all_vot2 = votingclf_grid_2.predict(X_test_sc)
 sub_vot2 = pd.DataFrame()
sub_vot2['PassengerId'] = df_test['PassengerId']
sub_vot2['Survived'] = pred_all_vot2
sub_vot2.to_csv('vot2.csv',index=False )
base_model = densenet.layers[-2].output
prediction = keras.layers.Dense(1, activation="sigmoid" )(base_model)
 densenet_model = keras.models.Model(inputs=densenet.input, outputs=prediction)
 densenet_model.compile(loss="binary_crossentropy", optimizer=keras.optimizers.Adam(lr=0.00001), metrics=["accuracy"])
 densenet_model.summary()<train_model>
from mlxtend.classifier import StackingClassifier
densenet_model.fit(X, Y, batch_size=16, epochs=5, verbose=1, validation_split=0.2 )<train_model>
clf1 = xgb_grid.best_estimator_
clf2 = gbc_grid.best_estimator_
clf3 = rf_grid.best_estimator_
clf4 = svc_grid.best_estimator_
lr = LogisticRegression()
st_clf = StackingClassifier(classifiers=[clf1, clf1, clf2, clf3, clf4], meta_classifier=lr)
 params = {'meta_classifier__C': [0.1,1.0,5.0,10.0] ,
'use_features_in_secondary' : [True, False]
}
with ignore_warnings(category=DeprecationWarning):
st_clf_grid = GridSearchCV(estimator=st_clf, param_grid=params, cv=5, refit=True)
 st_clf_grid.fit(X_sc, y_sc)
 sc_st_clf = get_best_score(st_clf_grid )
densenet_model.fit(X, Y, batch_size=16, epochs=5, verbose=1, validation_split=0.2 )<prepare_output>
with ignore_warnings(category=DeprecationWarning):
pred_all_stack = st_clf_grid.predict(X_test_sc)
 sub_stack = pd.DataFrame()
sub_stack['PassengerId'] = df_test['PassengerId']
sub_stack['Survived'] = pred_all_stack
sub_stack.to_csv('stack_clf.csv',index=False )
predictions = test_pred()
predictions = pd.DataFrame(predictions, columns=["id", "has_cactus"])
 predictions.head()<save_to_csv>
list_scores = [sc_knn, sc_rf, sc_extr, sc_svc, sc_gbc, sc_xgb,
sc_ada, sc_cat, sc_lgbm, sc_vot2_cv, sc_st_clf]
list_classifiers = ['KNN','RF','EXTR','SVC','GBC','XGB',
'ADA','CAT','LGBM','VOT2','STACK']
predictions.to_csv("densetmodel_submissions.csv", index=False )<load_pretrained>
score_subm_svc = 0.80861
score_subm_vot2 = 0.78947
score_subm_ada = 0.78468
score_subm_lgbm = 0.78468
score_subm_rf = 0.77990
score_subm_xgb = 0.77033
score_subm_dtree = 0.76076
score_subm_extr = 0.76076
score_subm_gbc = 0.74641
score_subm_cat = 0.74162
score_subm_knn = 0.69856
score_subm_stack = 0.76076
kf = np.load(".. /input/split-dataset/GroupMultilabelStratifiedKfold.npy", allow_pickle=True )<load_from_csv>
subm_scores = [score_subm_knn, score_subm_rf, score_subm_extr, score_subm_svc,
score_subm_gbc, score_subm_xgb, score_subm_ada, score_subm_cat,
score_subm_lgbm, score_subm_vot2, score_subm_stack]
<define_variables><EOS> 
predictions = {'KNN': pred_all_knn, 'RF': pred_all_rf, 'EXTR': pred_all_extr,
'SVC': pred_all_svc, 'GBC': pred_all_gbc, 'XGB': pred_all_xgb,
'ADA': pred_all_ada, 'CAT': pred_all_cat, 'LGBM': pred_all_lgbm,
'VOT2': pred_all_vot2, 'STACK': pred_all_stack}
df_predictions = pd.DataFrame(data=predictions)
 df_predictions.corr()