Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
%matplotlib inline
X_test[['id','demand']].to_csv('result.csv',index=False )<load_from_csv>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 combine = [train_df, test_df]
training_data, training_labels = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.train')
 testing_data, testing_labels = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.test')
 testing_data = csr_matrix(( testing_data.data, testing_data.indices, testing_data.indptr), shape=(testing_data.shape[0], training_data.shape[1]))<normalization>
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )
class SVM:
def __init__(self, learning_rate=0.1, reg_const=0.05, l1=False, num_iters=10, batch_size=200):
self.W = None
self.learning_rate = learning_rate
self.reg_const = reg_const
self.l1 = l1
if self.l1:
self.reg_fun = lambda x: np.max(np.sum(np.abs(x), axis=0))
self.reg_fun_grad = lambda x: np.sum([np.sign(val)for val in x.flatten() if not np.isclose(val,0, atol=1e-5)])
 else:
self.reg_fun = lambda x: 0.5*np.sum(x*x)
 self.reg_fun_grad=lambda x:x
self.num_iters = num_iters
self.batch_size = batch_size
def loss(self,X_batch, y_batch):
num_train = X_batch.shape[0]
num_classes = self.W.shape[1]
scores = X_batch.dot(self.W)
 correct_class_scores = scores[range(num_train), y_batch].reshape(-1,1)
 margins = np.maximum(0, scores + 1 - correct_class_scores)
 margins[range(num_train), y_batch] =0
loss = np.sum(margins)/ num_train + self.reg_const * self.reg_fun(self.W)
 coeff_mat = np.zeros(( num_train, num_classes))
coeff_mat[margins > 0] = 1
coeff_mat[range(num_train), y_batch] = - np.sum(coeff_mat, axis=1)
 dW =(X_batch.T ).dot(coeff_mat)
 dW = dW/num_train + self.reg_const * self.reg_fun_grad(self.W)
 return loss, dW
def fit(self, X, y, **kwargs):
if kwargs:
self.learning_rate = kwargs['learning_rate']
self.reg_const = kwargs['reg_const']
self.l1 = kwargs['l1']
if self.l1:
self.reg_fun = lambda x: np.max(np.sum(np.abs(x), axis=0))
self.reg_fun_grad = lambda x: np.sum([np.sign(val)for val in x.flatten() if not np.isclose(val,0, atol=1e-5)])
 else:
self.reg_fun = lambda x: 0.5*np.sum(x*x)
 self.reg_fun_grad=lambda x:x
self.num_iters = kwargs['num_iters']
self.batch_size = kwargs['batch_size']
num_train, dim = X.shape
num_classes = 2
if self.W is None:
self.W = 0.001 * np.random.randn(int(dim), int(num_classes))
for it in trange(self.num_iters, mininterval=5):
batch_idx = np.random.choice(num_train, self.batch_size, replace = True)
 X_batch = X[batch_idx]
y_batch = y[batch_idx]
loss, grad = self.loss(X_batch, y_batch.astype(int))
self.W += - self.learning_rate * grad
def get_params(self, deep = False):
return {'learning_rate': self.learning_rate, 'reg_const': self.reg_const,
'l1': self.l1, 'num_iters':self.num_iters, 'batch_size':self.batch_size}
def set_params(self, **parameters):
for parameter, value in parameters.items() :
setattr(self, parameter, value)
 return self
def predict(self, X):
scores = X.dot(self.W)
 y_pred = np.argmax(scores, axis =1)
 return y_pred
def score(self, X, y):
assert X.shape[0] == y.shape[0]
correct = sum(1 for actual, prediction in zip(y, self.predict(X)) if np.sign(actual)== np.sign(prediction))
return correct/X.shape[0]<train_model>
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
svm = SVM()
svm.fit(training_data, training_labels, learning_rate=0.01, reg_const=0.05, l1=False, num_iters=100_000, batch_size=200 )<compute_test_metric>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df.head()
svm.score(testing_data, testing_labels )<train_on_grid>
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape
clf = GridSearchCV(SVM() , param_grid={'learning_rate':[0.0001, 0.0005, 0.001], 'reg_const':[0.0001,0.001,0.01,0.1], 'l1': [False], 'num_iters':[200_000], 'batch_size':[200]}, cv=5, scoring='accuracy', n_jobs=-1)
 clf.fit(testing_data, testing_labels )<find_best_score>
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()
print(clf.best_params_)
 print(clf.best_score_ )<compute_train_metric>
guess_ages = np.zeros(( 2,3))
guess_ages
cross_val_score(SVM() , training_data, y=training_labels, scoring='accuracy',
fit_params=clf.best_params_, cv=5, n_jobs=-1 )<compute_test_metric>
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 train_df.head()
clf.best_estimator_.score(testing_data, testing_labels )<compute_train_metric>
train_df['AgeBand'] = pd.cut(train_df['Age'], 5)
 train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )
svm = clf.best_estimator_
eval_data, _ = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.eval.anon')
 eval_data = csr_matrix(( eval_data.data, eval_data.indices, eval_data.indptr), shape=(eval_data.shape[0], training_data.shape[1]))
submission_pred = svm.predict(eval_data )<save_to_csv>
for dataset in combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
train_df.head()
with open('submission.csv', 'w')as submission:
with open('.. /input/movie-ratings/movie-ratings/data-splits/data.eval.anon.id', 'r')as example_ids:
submission.write('example_id,label
')
 for example_id, label in zip(example_ids, submission_pred):
submission.write('{},{}
'.format(example_id.strip() , int(label)) )<set_options>
train_df = train_df.drop(['AgeBand'], axis=1)
 combine = [train_df, test_df]
train_df.head()
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<define_variables>
for dataset in combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
PATH = ".. /input/"
TMP_PATH = "/tmp/tmp"
MODEL_PATH = "/tmp/model/"
sz=224<set_options>
for dataset in combine:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()
torch.cuda.is_available()<set_options>
train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 combine = [train_df, test_df]
train_df.head()
torch.backends.cudnn.enabled<define_variables>
for dataset in combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
train_df.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10 )
fnames = np.array([f'train/{f}' for f in sorted(os.listdir(f'{PATH}train')) ])
 labels = np.array([(0 if 'cat' in fname else 1)for fname in fnames] )<load_pretrained>
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port
arch=resnet34
data=ImageClassifierData.from_paths(path=PATH,val_name='valid',test_name='test',tfms=tfms_from_model(arch, sz))<load_pretrained>
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
learn = ConvLearner.pretrained(arch, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH )<train_model>
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()
learn.fit(0.01, 2 )<predict_on_test>
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()
log_preds = learn.predict(is_test=True )<prepare_output>
train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
 train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False ).mean().sort_values(by='FareBand', ascending=True )
lps = np.argmax(log_preds,axis=1 )<define_variables>
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 combine = [train_df, test_df]
train_df.head(10 )
preds_classes = [data.classes[i] for i in lps]<load_from_csv>
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import BaggingClassifier
sub_frame = pd.read_csv('.. /input/Sample_Sub.csv')
 sub_frame['Category'] = preds_classes<save_to_csv>
from sklearn.model_selection import train_test_split
sub_frame.to_csv('Submission.csv',index=False )<import_modules>
from sklearn.model_selection import train_test_split
import numpy as np
import pandas as pd
from pathlib import Path
from fastai import *
from fastai.vision import *
import torchvision
import torch<define_variables>
X_train, X_test, y_train, y_test = train_test_split(train_df.drop("Survived", axis=1), train_df["Survived"], test_size=0.3 )
data_root_path = Path(".. /input" )<load_from_csv>
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
train_df = pd.read_csv(data_root_path/"train.csv")
 test_df = pd.read_csv(data_root_path/"sample_submission.csv" )<load_pretrained>
clf = DecisionTreeClassifier(random_state=42)
 clf.fit(X_train, y_train)
 print_score(clf, X_train, y_train, X_test, y_test, train=True)
 print_score(clf, X_train, y_train, X_test, y_test, train=False)
transforms = get_transforms(
 do_flip=True,
flip_vert=True,
max_rotate=15.0,
max_lighting=0.15,
max_warp=0.2 
)
 train_imgs = ImageList.from_df(train_df, path=data_root_path/'train', folder='train')
 test_imgs = ImageList.from_df(test_df, path=data_root_path/'test', folder='test')
 train_imgs =(train_imgs 
.split_by_rand_pct(0.01)
.label_from_df() 
.add_test(test_imgs)
.transform(transforms, size=128)
.databunch(path='.', bs=64, device= torch.device('cuda:0')) 
.normalize(imagenet_stats))<choose_model_class>
bag_clf = BaggingClassifier(base_estimator=clf, n_estimators=1000,
bootstrap=True, n_jobs=-1,
random_state=42)
 bag_clf.fit(X_train, y_train)
 print_score(bag_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(bag_clf, X_train, y_train, X_test, y_test, train=False )
learn = cnn_learner(train_imgs, torchvision.models.densenet161, metrics=[error_rate, accuracy] )<train_model>
bag_clf = BaggingClassifier(base_estimator=clf, n_estimators=1000,
bootstrap=True, oob_score=True,
n_jobs=-1, random_state=42 )
lr = 3e-02
learn.fit_one_cycle(8, slice(lr))<predict_on_test>
bag_clf.fit(X_train, y_train )
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
bag_clf.oob_score_
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
print_score(bag_clf, X_train, y_train, X_test, y_test, train=True )
test_df.to_csv('submission.csv', index=False )<set_options>
print_score(bag_clf, X_train, y_train, X_test, y_test, train=False )
plt.style.use('ggplot')
 np.random.seed(7)
 torch.cuda.manual_seed_all(7 )<load_from_csv>
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
train_dir=".. /input/train/train"
test_dir=".. /input/test/test"
train = pd.read_csv('.. /input/train.csv')
 sub_file = pd.read_csv(".. /input/sample_submission.csv")
 data_folder = Path(".. /input" )<normalization>
rf_clf = RandomForestClassifier(random_state=42 )
trfm = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1, max_lighting=0.2, max_warp=0.2, p_affine=0.75, p_lighting=0.75 )<categorify>
rf_clf.fit(X_train, y_train )
test_img = ImageList.from_df(sub_file, path=data_folder/'test', folder='test')
 databunch =(ImageList.from_df(train, path=data_folder/'train', folder='train')
.split_by_rand_pct(0.01)
.label_from_df() 
.add_test(test_img)
.transform(trfm, size=48)
.databunch(path='.', bs=64, device= torch.device('cuda:0')) 
.normalize(imagenet_stats)
 )<define_variables>
print_score(rf_clf, X_train, y_train, X_test, y_test, train=True )
databunch.show_batch(rows=3, figsize=(8,8))<train_model>
print_score(rf_clf, X_train, y_train, X_test, y_test, train=False )
learn = cnn_learner(databunch, models.resnet34, metrics=[error_rate, accuracy])
 learn.fit_one_cycle(5 )<train_model>
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
learn.unfreeze()
learn.fit_one_cycle(5, max_lr=slice(1e-03))<find_best_params>
rf_clf = RandomForestClassifier(random_state=42 )
learn.show_results(rows=3 )<find_best_params>
params_grid = {"max_depth": [3, None],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [True, False],
"criterion": ['gini', 'entropy']}
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(databunch.valid_ds)==len(losses)==len(idxs )<predict_on_test>
grid_search = GridSearchCV(rf_clf, params_grid,
n_jobs=-1, cv=5,
verbose=1, scoring='accuracy' )
predictions1=learn.get_preds(DatasetType.Test)
 predictions2=learn.get_preds(DatasetType.Test)
 predictions3=learn.get_preds(DatasetType.Test)
 predictions4=learn.get_preds(DatasetType.Test)
 predictions5=learn.get_preds(DatasetType.Test)
 predictions6=learn.get_preds(DatasetType.Test)
 predictions7=learn.get_preds(DatasetType.Test)
 predictions8=learn.get_preds(DatasetType.Test)
 comb_output=[predictions1[0],predictions2[0],predictions3[0],predictions4[0],
predictions5[0],predictions6[0],predictions7[0],predictions8[0]]
comb_output=torch.sum(torch.stack(comb_output),dim=0 )<save_to_csv>
grid_search.fit(X_train, y_train )
sub_file.has_cactus = comb_output.numpy() [:, 0]
sub_file.to_csv('submission.csv', index=False )<install_modules>
grid_search.best_score_
!pip install -q --upgrade pip
!pip install -q -U tensorflow-gpu==2.0.0-alpha0<import_modules>
grid_search.best_estimator_.get_params()
AUTOTUNE = tf.data.experimental.AUTOTUNE<load_from_csv>
print_score(grid_search, X_train, y_train, X_test, y_test, train=True )
train_csv = pd.read_csv('.. /input/train.csv')
 train_csv.head()<define_variables>
print_score(grid_search, X_train, y_train, X_test, y_test, train=False )
display(Image('.. /input/train/train/0004be2cfeaba1c0361d39e2b000257b.jpg'))
display(Image('.. /input/train/train/000c8a36845c0208e833c79c1bffedd1.jpg'))
display(Image('.. /input/train/train/002134abf28af54575c18741b89dd2a4.jpg'))
display(Image('.. /input/train/train/0024320f43bdd490562246435af4f90b.jpg'))<define_variables>
from sklearn.ensemble import ExtraTreesClassifier
images = ['.. /input/train/train/' + fname for fname in train_csv['id']]
labels = train_csv['has_cactus'].tolist()<split>
xt_clf = ExtraTreesClassifier(random_state=42)
 xt_clf.fit(X_train, y_train)
 print_score(xt_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(xt_clf, X_train, y_train, X_test, y_test, train=False )
X_train, X_dev, y_train, y_dev = train_test_split(images, labels, test_size=0.1, random_state=42)
 n_train = len(X_train)
 n_dev = len(X_dev )<normalization>
Y_pred = xt_clf.predict(test_df.drop('PassengerId',axis=1))
Y_pred
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submissions_xt.csv', index=False )
IMAGE_SIZE = 96
def preprocess_image(fname, label=None):
img = tf.io.read_file(fname)
 img = tf.image.decode_jpeg(img)
 img = tf.cast(img, tf.float32)
 img =(img / 127.5)- 1
img = tf.image.resize(img, size=(IMAGE_SIZE, IMAGE_SIZE))
if label is not None:
return img, label
else:
return img<split>
ada_clf = AdaBoostClassifier()
ada_clf.fit(X_train, y_train)
 print_score(ada_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(ada_clf, X_train, y_train, X_test, y_test, train=False )
BATCH_SIZE = 32
ds_train =(tf.data.Dataset.from_tensor_slices(( X_train, y_train)) 
.map(preprocess_image, num_parallel_calls=AUTOTUNE)
.shuffle(n_train)
.batch(BATCH_SIZE)
.prefetch(buffer_size=AUTOTUNE)
)
 ds_dev =(tf.data.Dataset.from_tensor_slices(( X_dev, y_dev)) 
.map(preprocess_image, num_parallel_calls=AUTOTUNE)
.shuffle(n_dev)
.batch(BATCH_SIZE)
.prefetch(buffer_size=AUTOTUNE)
 )<choose_model_class>
Y_pred = ada_clf.predict(test_df.drop('PassengerId',axis=1))
Y_pred
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submissions_ada.csv', index=False )
base_model = tf.keras.applications.MobileNetV2(
 input_shape =(IMAGE_SIZE, IMAGE_SIZE, 3),
include_top = False,
weights = 'imagenet' 
)
 base_model.trainable = False
base_model.summary()<choose_model_class>
from sklearn.ensemble import RandomForestClassifier
pooling_layer = tf.keras.layers.GlobalMaxPooling2D()
final_layer = tf.keras.layers.Dense(units=1, activation='sigmoid' )<choose_model_class>
ada_clf = AdaBoostClassifier(RandomForestClassifier())
 ada_clf.fit(X_train, y_train)
 print_score(ada_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(ada_clf, X_train, y_train, X_test, y_test, train=False)
 ada_clf = AdaBoostClassifier(base_estimator=RandomForestClassifier())
 ada_clf.fit(X_train, y_train)
 print_score(ada_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(ada_clf, X_train, y_train, X_test, y_test, train=False )
model = tf.keras.Sequential([
base_model,
pooling_layer,
final_layer
] )<choose_model_class>
Y_pred = ada_clf.predict(test_df.drop('PassengerId',axis=1))
Y_pred
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submissions_ada_random.csv', index=False )
learning_rate = 0.0001
model.compile(
 optimizer=tf.optimizers.Adam(learning_rate=learning_rate),
loss='binary_crossentropy',
metrics=['accuracy']
)<train_model>
from sklearn.ensemble import GradientBoostingClassifier
initial_epochs = 32
steps_per_epoch =(tf.math.ceil(n_train/BATCH_SIZE))
model.fit(
 ds_train.repeat() ,
epochs=initial_epochs,
steps_per_epoch=steps_per_epoch,
validation_data=ds_dev
)<set_options>
gbc_clf = GradientBoostingClassifier()
gbc_clf.fit(X_train, y_train )
base_model.trainable = True
len(base_model.layers )<train_model>
print_score(gbc_clf, X_train, y_train, X_test, y_test, train=True )
fine_tune_at = 100
for layer in base_model.layers[:fine_tune_at]:
layer.trainable = False<choose_model_class>
print_score(gbc_clf, X_train, y_train, X_test, y_test, train=False )
model.compile(
 optimizer=tf.optimizers.Adam(learning_rate=learning_rate/10),
loss='binary_crossentropy',
metrics=['accuracy']
)<train_model>
Y_pred = gbc_clf.predict(test_df.drop('PassengerId',axis=1))
Y_pred
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submissions_gbc.csv', index=False )
fine_epochs = 32
total_epochs = initial_epochs + fine_epochs
model.fit(
 ds_train.repeat() ,
epochs=total_epochs,
initial_epoch=initial_epochs,
steps_per_epoch=steps_per_epoch,
validation_data=ds_dev
)<define_variables>
import xgboost as xgb
test_image_names = tf.io.gfile.listdir('.. /input/test/test/')
 n_test = len(test_image_names)
 test_image_paths = list(map(lambda s: '.. /input/test/test/' + s, test_image_names))<create_dataframe>
xgb_clf = xgb.XGBClassifier(max_depth=5, n_estimators=10000, learning_rate=0.3,
n_jobs=-1 )
ds_test =(tf.data.Dataset.from_tensor_slices(test_image_paths)
.map(preprocess_image, num_parallel_calls=AUTOTUNE)
.batch(BATCH_SIZE)
.prefetch(buffer_size=AUTOTUNE)
 )<predict_on_test>
xgb_clf.fit(X_train, y_train )
predictions = model.predict(ds_test )<create_dataframe>
print_score(xgb_clf, X_train, y_train, X_test, y_test, train=True )
final_df = pd.DataFrame()
final_df['id'] = test_image_names
final_df['has_cactus'] = predictions<save_to_csv>
print_score(xgb_clf, X_train, y_train, X_test, y_test, train=False )
final_df.to_csv('submission.csv', index=False )<set_options>
Y_pred = xgb_clf.predict(test_df.drop('PassengerId',axis=1))
Y_pred
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submissions_xgb.csv', index=False )
%matplotlib inline<load_from_csv>
tree_clf = DecisionTreeClassifier()
tree_clf.fit(X_train, y_train)
 print_score(tree_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(tree_clf, X_train, y_train, X_test, y_test, train=False )
train_df = pd.read_csv(".. /input/train.csv")
 train_df.head()<define_variables>
rf_clf = RandomForestClassifier()
rf_clf.fit(X_train, y_train.ravel())
 print_score(rf_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(rf_clf, X_train, y_train, X_test, y_test, train=False )
data_folder = Path(".. /input")
 train_img = ImageList.from_df(train_df, path=data_folder, folder='train/train' )<feature_engineering>
en_en = pd.DataFrame()
transformations = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1, max_lighting=0.2,
max_warp=0.2, p_affine=0.75, p_lighting=0.75 )<load_from_csv>
tree_clf.predict_proba(X_train )
test_df = pd.read_csv(".. /input/sample_submission.csv")
 test_img = ImageList.from_df(test_df, path=data_folder, folder='test/test' )<normalization>
en_en['tree_clf'] = pd.DataFrame(tree_clf.predict_proba(X_train)) [1]
en_en['rf_clf'] = pd.DataFrame(rf_clf.predict_proba(X_train)) [1]
col_name = en_en.columns
en_en = pd.concat([en_en, pd.DataFrame(y_train ).reset_index(drop=True)], axis=1 )
train_img = train_img.split_by_rand_pct(0.01 ).label_from_df().add_test(test_img ).transform(transformations, size=128 ).databunch(path='.', bs=64, device=torch.device('cuda:0')).normalize(imagenet_stats )<choose_model_class>
tmp = list(col_name)
 tmp.append('ind')
 en_en.columns = tmp
learn = cnn_learner(train_img, models.densenet161, metrics=[error_rate, accuracy] )<train_model>
m_clf = LogisticRegression(fit_intercept=False)
 m_clf.fit(en_en[['tree_clf', 'rf_clf']], en_en['ind'] )
lr = 3e-02
learn.fit_one_cycle(5, slice(lr))<save_to_csv>
en_test = pd.DataFrame()
preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 test_df.has_cactus = preds.numpy() [:,0]
test_df.to_csv('submission.csv', index=False )<set_options>
en_test['tree_clf'] = pd.DataFrame(tree_clf.predict_proba(X_test)) [1]
en_test['rf_clf'] = pd.DataFrame(rf_clf.predict_proba(X_test)) [1]
col_name = en_en.columns
en_test['combined'] = m_clf.predict(en_test[['tree_clf', 'rf_clf']] )
import random
import os
import sklearn.utils
from tqdm import tqdm, tqdm_notebook
import pandas as pd
import cv2 as cv
from tqdm import tqdm
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Conv2D, Dense, Flatten, BatchNormalization, Dropout, LeakyReLU, DepthwiseConv2D, Flatten
from tensorflow.keras.layers import GlobalAveragePooling2D
from tensorflow.keras.callbacks import ModelCheckpoint,ReduceLROnPlateau,EarlyStopping
import json
import logging
import numpy as np
import matplotlib.pyplot as plt<set_options>
en_test = pd.concat([en_test, pd.DataFrame(y_test ).reset_index(drop=True)], axis=1 )
seed(1372)
 set_random_seed(1372)
<train_model>
en_test.columns = tmp
def resize_and_save(filename, input_dir, output_dir, size=32):

 image = Image.open(os.path.join(input_dir, filename))
image.save(os.path.join(output_dir, filename))<load_pretrained>
print(round(accuracy_score(en_test['ind'], en_test['combined']), 4))
class Params() :

 def __init__(self, json_path):
self.update(json_path)
 def save(self, json_path):

 with open(json_path, 'w')as f:
json.dump(self.__dict__, f, indent=4)
 def update(self, json_path):

 with open(json_path)as f:
params = json.load(f)
 self.__dict__.update(params)
 @property
def dict(self):

 return self.__dict__
def set_logger(log_path):

 logger = logging.getLogger()
logger.setLevel(logging.INFO)
 if not logger.handlers:
file_handler = logging.FileHandler(log_path)
 file_handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s: %(message)s'))
logger.addHandler(file_handler)
 stream_handler = logging.StreamHandler()
stream_handler.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(stream_handler)
 def save_dict_to_json(d, json_path):

 with open(json_path, 'w')as f:
d = {k: float(v)for k, v in d.items() }
json.dump(d, f, indent=4)
<data_type_conversions>
print(classification_report(en_test['ind'], en_test['combined']))
df = pd.read_csv(csv_dir)
 df['id'] = dataset_dir + '/' + df['id'].astype(str)
 filenames = df['id']
labels = df['has_cactus'].astype(np.float32)
 df = sklearn.utils.shuffle(df,random_state=1372)
 df = df.reset_index(drop=True )<categorify>
def _parse_function(filename, label, size):

 image_string = tf.read_file(filename)
 image_decoded = tf.image.decode_jpeg(image_string, channels=3)
 image = tf.image.convert_image_dtype(image_decoded, tf.float32)
 resized_image = tf.image.resize_images(image, [size, size])
 return resized_image, label<train_model>
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import AdaBoostClassifier
def input_fn(is_training, filenames, labels, params):

 num_samples = len(filenames)
 assert len(filenames)== len(labels), "Filenames and labels should have same length"
parse_fn = lambda f, l: _parse_function(f, l, params.image_size)
 train_fn = lambda f, l: train_preprocess(f, l, params.use_random_flip)
 if is_training:
dataset =(tf.data.Dataset.from_tensor_slices(( tf.constant(filenames), tf.constant(labels)))
.shuffle(num_samples)
.map(parse_fn, num_parallel_calls=params.num_parallel_calls)
.map(train_fn, num_parallel_calls=params.num_parallel_calls)
.batch(params.batch_size)
.repeat() 
.prefetch(32)
)
 else:
dataset =(tf.data.Dataset.from_tensor_slices(( tf.constant(filenames), tf.constant(labels)))
.map(parse_fn)
.batch(params.batch_size)
.repeat() 
.prefetch(32)
)
 return dataset<split>
pd.Series(list(y_train)).value_counts() / pd.Series(list(y_train)).count()
params = Params(json_path)
 split = int(len(filenames)*0.15)
 train_dataset = input_fn(True, filenames[:split], labels[:split], params)
 valid_dataset = input_fn(False , filenames[split:], labels[split:], params )<choose_model_class>
class_weight = {0:0.61, 1:0.38}
model = Sequential()
model.add(Conv2D(3, kernel_size = 3, activation = 'relu', input_shape =(32, 32, 3)))
 model.add(Conv2D(filters = 16, kernel_size = 3, activation = 'relu'))
model.add(Conv2D(filters = 16, kernel_size = 3, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 32, kernel_size = 1, activation = 'relu'))
model.add(Conv2D(filters = 64, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 2, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 128, kernel_size = 1, activation = 'relu'))
model.add(Conv2D(filters = 256, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 256, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 512, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 2, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 512, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 512, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 1024, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 1024, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Flatten())
 model.add(Dense(512, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(256, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(128, activation = 'elu'))
model.add(Dense(1, activation = 'sigmoid'))<choose_model_class>
forest = RandomForestClassifier(class_weight=class_weight )
Adam = tf.keras.optimizers.Adam(lr=params.learning_rate , amsgrad=True)
 model.compile(optimizer = Adam, loss = tf.losses.log_loss, metrics = ['accuracy'])
 model.summary()<choose_model_class>
ada = AdaBoostClassifier(base_estimator=forest, n_estimators=100,
learning_rate=0.5, random_state=42 )
file_path = 'weights-aerial-cactus.h5'
callbacks = [
ModelCheckpoint(file_path, monitor = 'val_acc', verbose = 1, save_best_only = True, mode = 'max'),
ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 1, mode = 'min', min_lr = 0.00001),
EarlyStopping(monitor = 'val_loss', min_delta = 1e-10, patience = 15, verbose = 1, restore_best_weights = True)
 ]<train_model>
ada.fit(X_train, y_train.ravel() )
history = model.fit(train_dataset, validation_data=valid_dataset,
epochs=50,verbose=True,
steps_per_epoch=int(( len(filenames)- split)/params.batch_size),
validation_steps=int(split/params.batch_size),
callbacks = callbacks )<load_pretrained>
print_score(ada, X_train, y_train, X_test, y_test, train=True)
 print_score(ada, X_train, y_train, X_test, y_test, train=False )
model.load_weights(file_path )<save_to_csv>
bag_clf = BaggingClassifier(base_estimator=ada, n_estimators=50,
max_samples=1.0, max_features=1.0, bootstrap=True,
bootstrap_features=False, n_jobs=-1,
random_state=42 )
test_df = pd.read_csv('.. /input/sample_submission.csv')
 X_test = []
images_test = test_df['id'].values
for img_id in tqdm_notebook(images_test):
X_test.append(cv.imread('.. /input/test/test/' + img_id))
X_test = np.asarray(X_test)
 X_test = X_test.astype('float32')
 X_test /= 255
y_test_pred = model.predict_proba(X_test)
 test_df['has_cactus'] = y_test_pred
test_df.to_csv('aerial-cactus-submission_1.csv', index = False)
 for i in range(len(y_test_pred)) :
if y_test_pred[i][0] >= 0.5:
y_test_pred[i][0] = 1.0
else:
y_test_pred[i][0] = 0.0
test_df['has_cactus'] = y_test_pred
test_df.to_csv('aerial-cactus-submission_2.csv', index = False )<import_modules>
bag_clf.fit(X_train, y_train.ravel() )
from pathlib import Path
from fastai import *
from fastai.vision import *
import torch<define_variables>
print_score(bag_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(bag_clf, X_train, y_train, X_test, y_test, train=False )
<load_from_csv><EOS> 
Y_pred = bag_clf.predict(test_df.drop('PassengerId',axis=1))
Y_pred
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submissions_bag_last.csv', index=False )