Kaggle competition: Titanic - Machine Learning from Disaster
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 50_000
df['batch_index'] = df.index -(df.batch * 50_000)
 df['batch_slices'] = df['batch_index'] // 5_000
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal_undrifted'].mean()
d['median'+c] = df.groupby([c])['signal_undrifted'].median()
d['max'+c] = df.groupby([c])['signal_undrifted'].max()
d['min'+c] = df.groupby([c])['signal_undrifted'].min()
d['std'+c] = df.groupby([c])['signal_undrifted'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal_undrifted'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal_undrifted'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal_undrifted'].apply(lambda x: np.min(np.abs(x)))
 for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['range'+c] = df['max'+c] - df['min'+c]
df['maxtomin'+c] = df['max'+c] / df['min'+c]
df['abs_avg'+c] =(df['abs_min'+c] + df['abs_max'+c])/ 2
df['signal_shift_+1'] = [0,] + list(df['signal_undrifted'].values[:-1])
 df['signal_shift_-1'] = list(df['signal_undrifted'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
df['signal_shift_+2'] = [0,] + [1,] + list(df['signal_undrifted'].values[:-2])
 df['signal_shift_-2'] = list(df['signal_undrifted'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal_undrifted', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]:
df[c+'_msignal'] = df[c] - df['signal_undrifted']
return df
train = features(train)
 test = features(test )<compute_train_metric>
%matplotlib inline
sns.set(style='white', context='notebook', palette='deep' )
def f1_score_calc(y_true, y_pred):
return f1_score(y_true, y_pred, average="macro")
 def lgb_Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average="macro")
 return('KaggleMetric', score, True)
 def train_model_classification(X, X_test, y, params, model_type='lgb', eval_metric='f1score',
columns=None, plot_feature_importance=False, model=None,
verbose=50, early_stopping_rounds=200, n_estimators=2000):
columns = X.columns if columns == None else columns
X_test = X_test[columns]
metrics_dict = {
'f1score': {'lgb_metric_name': lgb_Metric,}
}
result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(( len(X_test)))
 scores = []
feature_importance = pd.DataFrame()

 if True:
X_train, X_valid, y_train, y_valid = model_selection.train_test_split(X, y, test_size=0.3, random_state=7)
 if model_type == 'lgb':
model = lgb.train(params, lgb.Dataset(X_train, y_train),
n_estimators, lgb.Dataset(X_valid, y_valid),
verbose_eval=verbose, early_stopping_rounds=early_stopping_rounds, feval=lgb_Metric)
 preds = model.predict(X, num_iteration=model.best_iteration)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 if model_type == 'xgb':
train_set = xgb.DMatrix(X_train, y_train)
 val_set = xgb.DMatrix(X_valid, y_valid)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=verbose, early_stopping_rounds=early_stopping_rounds)
 preds = model.predict(xgb.DMatrix(X))
y_pred = model.predict(xgb.DMatrix(X_test))
if model_type == 'cat':
model = CatBoostRegressor(params)
 model.fit(X_train, y_train)
 y_pred_valid = np.round(np.clip(preds, 0, 10)).astype(int)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 y_pred = np.round(np.clip(y_pred, 0, 10)).astype(int)
 oof = preds
scores.append(f1_score_calc(y, np.round(np.clip(preds,0,10)).astype(int)))
 prediction += y_pred
if model_type == 'lgb' and plot_feature_importance:
fold_importance = pd.DataFrame()
fold_importance["feature"] = columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 print('FINAL score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
result_dict['model'] = model
if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= folds.n_splits
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
return result_dict<prepare_x_and_y>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
good_columns = [c for c in train.columns if c not in ['time', 'signal','open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]
X = train[good_columns].copy()
y = train['open_channels']
X_test = test[good_columns].copy()
del train, test<train_model>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
params_xgb = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse'}
result_dict_xgb = train_model_classification(X=X[0:500000*8-1], X_test=X_test, y=y[0:500000*8-1], params=params_xgb, model_type='xgb', eval_metric='f1score', plot_feature_importance=False,
verbose=50, early_stopping_rounds=250 )<train_model>
train.isnull().sum()
params_lgb = {'learning_rate': 0.1, 'max_depth': 7, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1}
result_dict_lgb = train_model_classification(X=X[0:500000*8-1], X_test=X_test, y=y[0:500000*8-1], params=params_lgb, model_type='lgb', eval_metric='f1score', plot_feature_importance=False,
verbose=50, early_stopping_rounds=250, n_estimators=3000 )<define_variables>
train.isnull().sum()
preds_ensemble = 0.50 * result_dict_lgb['prediction'] + 0.50 * result_dict_xgb['prediction']<save_to_csv>
test.isnull().sum()
sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv')
 sub['open_channels'] = np.array(np.round(preds_ensemble,0), np.int)
 sub.to_csv('submission.csv', index=False, float_format='%.4f')
 sub.head(10 )<import_modules>
test.isnull().sum()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.stats import mode
from sklearn.metrics import f1_score, accuracy_score
from cuml.neighbors import KNeighborsClassifier, NearestNeighbors
import cuml; cuml.__version__<prepare_x_and_y>
train_test_data = [train,test]
for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-za-a]+)\.',expand=False )
%%time
KNN = 100
batch = 1000
test_pred = np.zeros(( test.shape[0]),dtype=np.int8)
 for g in [0,1,2,3,4]:
print('Infering group %i'%g)
 data = train.loc[train.group==g]
X_train = np.zeros(( len(data)-6,7))
X_train[:,0] = 0.25*data.signal[:-6]
X_train[:,1] = 0.5*data.signal[1:-5]
X_train[:,2] = 1.0*data.signal[2:-4]
X_train[:,3] = 4.0*data.signal[3:-3]
X_train[:,4] = 1.0*data.signal[4:-2]
X_train[:,5] = 0.5*data.signal[5:-1]
X_train[:,6] = 0.25*data.signal[6:]
y_train = data.open_channels[3:].values
data = test.loc[test.group==g]
X_test = np.zeros(( len(data)-6,7))
X_test[:,0] = 0.25*data.signal[:-6]
X_test[:,1] = 0.5*data.signal[1:-5]
X_test[:,2] = 1.0*data.signal[2:-4]
X_test[:,3] = 4.0*data.signal[3:-3]
X_test[:,4] = 1.0*data.signal[4:-2]
X_test[:,5] = 0.5*data.signal[5:-1]
X_test[:,6] = 0.25*data.signal[6:]
model = NearestNeighbors(n_neighbors=KNN)
 model.fit(X_train)
 distances, indices = model.kneighbors(X_test)
 ct = indices.shape[0]
pred = np.zeros(( ct+6),dtype=np.int8)
 it = ct//batch + int(ct%batch!=0)
 for k in range(it):
a = batch*k; b = batch*(k+1); b = min(ct,b)
 pred[a+3:b+3] = mode(y_train[ indices[a:b].astype(int)], axis=1)[0][:,0]
test_pred[test.group==g] = pred<save_to_csv>
train['Title'].value_counts()
sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv')
 sub.open_channels = test_pred
sub.to_csv('submission.csv',index=False,float_format='%.4f')
 res=200
plt.figure(figsize=(20,5))
plt.plot(sub.time[::res],sub.open_channels[::res])
 plt.show()<import_modules>
test['Title'].value_counts()
AUTOTUNE = tf.data.experimental.AUTOTUNE
print("Tensorflow version " + tf.__version__ )<define_variables>
title_mapping = {"Mr":0, "Miss":1, "Mrs":2, "Master":3, "Dr":3, "Rev":3, "Col":3, "Major":3, "Mile":3, "Jonkheer":3, "Countess":3, "Sir":3, "Ms":3, "Lady":3, "Mme":3, "Don":3, "Capt":3, "Dona":3}
BATCH_SIZE = 64
TOTAL_DIGITS = 60000
VALIDATION_DIGITS = 8000<categorify>
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )
column_defaults = [tf.constant(0, dtype=tf.int32)for i in range(28*28+1)]
def decode_csv_line(line, has_labels):
data = tf.io.decode_csv(line, column_defaults)
 first_col = data[0]
if has_labels:
label = tf.one_hot(first_col, 10)
 else:
label = first_col
pixels = data[1:]
image = tf.reshape(pixels, [28,28])
 image = tf.cast(image, tf.float32)/ 255.0
return image, label
def augment(image, label):
std = 0.1
image = tf.expand_dims(image, axis=-1)
 batch = tf.shape(image)[0]
dy1 = tf.random.normal([batch], stddev=std)
 dx1 = tf.random.normal([batch], stddev=std)
 dy2 = tf.random.normal([batch], stddev=std)
 dx2 = tf.random.normal([batch], stddev=std)
 y1 = tf.zeros([batch])+ dy1
x1 = tf.zeros([batch])+ dx1
y2 = tf.ones([batch])+ dy2
x2 = tf.ones([batch])+ dx2
box = tf.stack([y1, x1, y2, x2], axis=-1)
 image = tf.image.crop_and_resize(image, box, tf.range(batch), [28, 28])
 return tf.squeeze(image, axis=-1), label
def blur(image, label):
image = tf.expand_dims(image, axis=-1)
 image = tf.image.resize(image, [56,56], method=tf.image.ResizeMethod.BICUBIC, antialias=True)
 image = tf.image.resize(image, [28,28], method=tf.image.ResizeMethod.BICUBIC, antialias=True)
 image = tf.squeeze(image, axis=-1)
 return(image, label)
 def load_dataset(filename, has_labels):
dataset = tf.data.TextLineDataset(filename)
 dataset = dataset.skip(1)
 dataset = dataset.map(lambda line: decode_csv_line(line, has_labels), num_parallel_calls=AUTOTUNE)
 return dataset
def get_training_dataset(filename, batch_size):
dataset = load_dataset(filename, has_labels=True ).skip(VALIDATION_DIGITS)
 dataset = dataset.cache()
dataset = dataset.shuffle(5000, reshuffle_each_iteration=True)
 dataset = dataset.repeat()
dataset = dataset.batch(batch_size)
 dataset = dataset.map(blur)
 dataset = dataset.map(augment)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def get_validation_dataset(filename, batch_size):
dataset = load_dataset(filename, has_labels=True ).take(VALIDATION_DIGITS)
 dataset = dataset.cache()
dataset = dataset.batch(batch_size)
 dataset = dataset.map(blur)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def get_submission_dataset(filename, batch_size):
dataset = load_dataset(filename, has_labels=False)
 dataset = dataset.cache()
dataset = dataset.batch(batch_size)
 dataset = dataset.map(blur)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset<load_from_csv>
train.drop('Name', axis=1, inplace=True)
 test.drop('Name', axis=1, inplace=True )
train_dataset = get_training_dataset("/kaggle/input/Kannada-MNIST/train.csv", batch_size=BATCH_SIZE)
 valid_dataset = get_validation_dataset("/kaggle/input/Kannada-MNIST/train.csv", batch_size=VALIDATION_DIGITS)
 submit_dataset = get_submission_dataset("/kaggle/input/Kannada-MNIST/test.csv", batch_size=1000)
 steps_per_epoch =(TOTAL_DIGITS - VALIDATION_DIGITS)// BATCH_SIZE<categorify>
sex_mapping = {'male':0, 'female':1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )
PER_ROW = 24
N = PER_ROW*4
train_digits, train_labels = dataset_to_numpy_util(train_dataset, N)
 valid_digits, valid_labels = dataset_to_numpy_util(valid_dataset, N)
 display_rows_of_digits(train_digits, train_labels, train_labels, "training digits and their labels", N//PER_ROW, PER_ROW)
 display_rows_of_digits(valid_digits, valid_labels, valid_labels, "validation digits and their labels", N//PER_ROW, PER_ROW )<choose_model_class>
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )
def make_model() :
model = tf.keras.Sequential(
 [
tf.keras.layers.Reshape(input_shape=(28,28), target_shape=(28, 28, 1), name="image"),
tf.keras.layers.SeparableConv2D(filters=12, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=16, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=20, kernel_size=6, padding='same', use_bias=False, strides=2, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=24, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=28, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=32, kernel_size=6, padding='same', use_bias=False, strides=2, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=36, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(100, use_bias=False),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(10, activation='softmax')
 ])
 model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model
model = make_model()
model.summary()
def lr_fn(epoch):
return LEARNING_RATE * LEARNING_RATE_EXP_DECAY**epoch
lr_decay = tf.keras.callbacks.LearningRateScheduler(lr_fn, verbose=True )<train_model>
for dataset in train_test_data:
dataset.loc[dataset['Age']<=16, 'Age'] = 0
dataset.loc[(dataset['Age']>16)&(dataset['Age']<=23), 'Age'] =1
dataset.loc[(dataset['Age']>23)&(dataset['Age']<=34), 'Age'] =2
dataset.loc[(dataset['Age']>34)&(dataset['Age']<=42), 'Age'] =3
dataset.loc[(dataset['Age']>42)&(dataset['Age']<=58), 'Age'] =4
dataset.loc[dataset['Age']>58, 'Age'] = 5
EPOCHS = 60
LEARNING_RATE = 0.01
LEARNING_RATE_EXP_DECAY = 0.95
display_lr_schedule(lr_fn, EPOCHS )<train_model>
Pclass1 = train[train['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train[train['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train[train['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class', '2nd class', '3rd class']
df.plot(kind='bar', stacked=True, figsize=(10,5))
history = model.fit(train_dataset, steps_per_epoch=steps_per_epoch, validation_data=valid_dataset, validation_steps=1, epochs=EPOCHS, callbacks=[lr_decay] )<predict_on_test>
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 dataset['Title'] = dataset['Title'].fillna(0 )
digits, labels = dataset_to_numpy_util(valid_dataset, VALIDATION_DIGITS)
 probabilities = model.predict(digits, steps=1)
 predicted_labels = np.argmax(probabilities, axis=1)
 display_top_unrecognized(digits, predicted_labels, labels, n_rows=12, digits_per_row=24 )<save_to_csv>
train.isnull().sum()
results = None
for images, indices in submit_dataset:
predictions = model.predict(images)
 df = pd.DataFrame({'id':indices.numpy() , 'label':np.argmax(predictions, axis=-1)})
 results = results.append(df)if results is not None else df
results.to_csv("submission.csv", index=False)
 !ls -al "submission.csv"
!wc -l "submission.csv"
!head "submission.csv" -n 15
!echo "..."
!tail "submission.csv" -n 15<set_options>
train.isnull().sum()
def random_seed(seed_value, use_cuda):
np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 random.seed(seed_value)
 if use_cuda: torch.cuda.manual_seed_all(seed_value)
 random_seed(42, True )<load_from_csv>
test.isnull().sum()
train_csv = pd.read_csv(PATH/'train.csv')
 train_csv.head().T<load_from_csv>
test.isnull().sum()
def get_data_labels(csv,label):
fileraw = pd.read_csv(csv)
 labels = fileraw[label].to_numpy()
data = fileraw.drop([label],axis=1 ).to_numpy(dtype=np.float32 ).reshape(( fileraw.shape[0],28,28))
data = np.expand_dims(data, axis=1)
 return data, labels
train_data, train_labels = get_data_labels(PATH/'train.csv','label')
 test_data, test_labels = get_data_labels(PATH/'test.csv','id')
 other_data, other_labels = get_data_labels(PATH/'Dig-MNIST.csv','label' )<train_model>
embarked_mapping = {'S': 0, "C": 1, "Q": 2}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )
plt.title(f'Training Label: {train_labels[43]}')
 plt.imshow(train_data[43,0],cmap='gray');<define_variables>
train["Fare"].fillna(train.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 test["Fare"].fillna(test.groupby("Pclass")["Fare"].transform("median"), inplace=True )
np.random.seed(42)
 ran_20_pct_idx =(np.random.random_sample(train_labels.shape)) <.2
train_80_labels = train_labels[np.invert(ran_20_pct_idx)]
train_80_data = train_data[np.invert(ran_20_pct_idx)]
valid_20_labels = train_labels[ran_20_pct_idx]
valid_20_data = train_data[ran_20_pct_idx]<create_dataframe>
for dataset in train_test_data:
dataset.loc[dataset['Fare']<=17, 'Fare'] = 0
dataset.loc[(dataset['Fare']>17)&(dataset['Fare']<=29), 'Fare']= 1
dataset.loc[(dataset['Fare']>29)&(dataset['Fare']<=100), 'Fare']= 2
dataset.loc[dataset['Fare']>100, 'Fare'] = 3
train_ds = ArrayDataset(train_80_data,train_80_labels)
 valid_ds = ArrayDataset(valid_20_data,valid_20_labels)
 other_ds = ArrayDataset(other_data, other_labels)
 test_ds = ArrayDataset(test_data, test_labels )<train_model>
train.Cabin.value_counts()
bs = 64
data = DataBunch.create(train_ds, valid_ds, test_ds=test_ds, bs=bs )<define_variables>
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].str[:1]
MODEL_DIR = Path('.. /working/models/' )<choose_model_class>
cabin_mapping = {"A":0, "B":0.4, "C":0.8, "D":1.2, "E":1.6, "F":2.0, "G":2.4, "T":2.8}
for dataset in train_test_data:
dataset['Cabin']=dataset['Cabin'].map(cabin_mapping )
class ConvNet1(nn.Module):
def __init__(self):
super(ConvNet1, self ).__init__()
self.layer1 = nn.Sequential(
 nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(32)
)
 self.layer2 = nn.Sequential(
 nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(32)
)
 self.layer3 = nn.Sequential(
 nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5, padding=1, stride=2),
nn.ReLU() ,
nn.BatchNorm2d(64)
)
 self.layer4 = nn.Sequential(
 nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(64)
)
 self.layer5 = nn.Sequential(
 nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(64)
)
 self.layer6 = nn.Sequential(
 nn.Conv2d(in_channels=64, out_channels=128, kernel_size=5, padding=1, stride=2),
nn.ReLU() ,
nn.BatchNorm2d(128), 
)
 self.drop_out = nn.Dropout(0.2)
 self.relu = nn.ReLU()
self.fc1 = nn.Linear(4608, 128)
 self.fc2 = nn.Linear(128, 10)
 self.bn1d = nn.BatchNorm1d(128)
 self.output = nn.Softmax(dim=1)
 def forward(self, x):
out = self.layer1(x)
 out = self.layer2(out)
 out = self.drop_out(self.layer3(out))
out = self.layer4(out)
 out = self.layer5(out)
 out = self.drop_out(self.layer6(out))
out = out.view(out.shape[0], -1)
 out = self.fc1(out)
 out = self.relu(out)
 out = self.bn1d(out)
 out = self.drop_out(out)
 out = self.fc2(out)
 out = self.output(out)
 return out<train_on_grid>
train['Cabin'].fillna(train.groupby("Pclass")["Cabin"].transform("median"), inplace=True)
 test['Cabin'].fillna(test.groupby("Pclass")["Cabin"].transform("median"), inplace=True )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 conv_net_1 = ConvNet1()
conv_net_1 = conv_net_1.to(device )<choose_model_class>
train['FamilySize'] = train['SibSp'] + train['Parch'] + 1
test['FamilySize'] = test['SibSp'] + test['Parch'] + 1
class ConvNet2(nn.Module):
def __init__(self):
super(ConvNet2, self ).__init__()
self.layer1 = nn.Sequential(
 nn.Conv2d(1, 32, kernel_size=5, stride=1, padding=2),
nn.ReLU() ,
nn.MaxPool2d(kernel_size=2, stride=2))
self.layer2 = nn.Sequential(
 nn.Conv2d(32, 64, kernel_size=5, stride=1, padding=2),
nn.ReLU() ,
nn.MaxPool2d(kernel_size=2, stride=2))
self.drop_out = nn.Dropout(0.5)
 self.fc1 = nn.Linear(7 * 7 * 64, 1000)
 self.fc2 = nn.Linear(1000, 10)
 self.relu = nn.ReLU()
def forward(self, x):
out = self.layer1(x)
 out = self.layer2(out)
 out = out.reshape(out.size(0), -1)
 out = self.drop_out(out)
 out = self.fc1(out)
 out = self.relu(out)
 out = self.drop_out(out)
 out = self.fc2(out)
 return out<train_on_grid>
family_mapping = {1:0, 2:0.4, 3:0.8, 4:1.2, 5:1.6, 6:2.0, 7:2.4, 8:2.8, 9:3.2, 10:3.6, 11:4.0}
for dataset in train_test_data:
dataset['FamilySize']=dataset['FamilySize'].map(family_mapping )
conv_net_2 = ConvNet2()
conv_net_2 = conv_net_2.to(device )<concatenate>
features_drop = ['SibSp', 'Ticket', 'Parch']
train = train.drop(features_drop, axis=1)
 test = test.drop(features_drop, axis=1)
 train = train.drop(['PassengerId'], axis=1 )
def conv2(ni,nf,stride=2,ks=3): return conv_layer(ni,nf,stride=stride,ks=ks )<train_on_grid>
train_data = train.drop(['Survived'], axis=1)
 target = train['Survived']
conv_net_3 = nn.Sequential(
 conv2(1,32,stride=1,ks=3),
conv2(32,32,stride=1,ks=3),
conv2(32,32,stride=2,ks=5),
nn.Dropout(0.4),
conv2(32,64,stride=1,ks=3),
conv2(64,64,stride=1,ks=3),
conv2(64,64,stride=2,ks=5),
nn.Dropout(0.4),
Flatten() ,
nn.Linear(3136, 128),
relu(inplace=True),
nn.BatchNorm1d(128),
nn.Dropout(0.4),
nn.Linear(128,10)
 )<load_pretrained>
k_fold = StratifiedKFold(n_splits=10 )
rn18 = models.resnet18(pretrained=False)
 rn18.load_state_dict(torch.load('.. /input/pytorch-pretrained-models/resnet18-5c106cde.pth'))
rn18.conv1 = Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False )<choose_model_class>
random_state = 2
classifiers = []
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state,learning_rate=0.1))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, train_data, y = target, scoring = "accuracy", cv = k_fold, n_jobs=-1))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","AdaBoost",
"RandomForest","ExtraTrees","GradientBoosting"]})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores" )
learner1 = Learner(data,
conv_net_1,
metrics=accuracy,
model_dir=MODEL_DIR,
opt_func=Adam,
loss_func=nn.CrossEntropyLoss()
)<find_best_params>
DTC = DecisionTreeClassifier(max_depth=5)
 adaDTC = AdaBoostClassifier(DTC, random_state=7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[1,10,20,30,40,50],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 1, 10, 100]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=k_fold, scoring="accuracy", n_jobs= -1, verbose = 1,return_train_score = True)
 gsadaDTC.fit(train_data, target)
 ada_best = gsadaDTC.best_estimator_
learner1.lr_find()
learner1.recorder.plot(suggestion=True )<train_model>
gsadaDTC.best_score_
%%time
learner1.fit_one_cycle(50,
slice(1e-03),
callbacks=[SaveModelCallback(learner1,
every='improvement',
monitor='accuracy',
name='best_model_1')]
)<save_to_csv>
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [5],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[500,1000],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=k_fold, scoring="accuracy", n_jobs= -1, verbose = 1)
 gsExtC.fit(train_data,target)
 ExtC_best = gsExtC.best_estimator_
learner1.load('best_model_1')
 preds, ids = learner1.get_preds(DatasetType.Test)
 y = torch.argmax(preds, dim=1)
 submission_1 = pd.DataFrame({ 'id': ids,'label': y })
 submission_1.to_csv("submission.csv", index=False)
 submission_1.to_csv("submission_1.csv", index=False )<choose_model_class>
gsExtC.best_score_
learner2 = Learner(data,
conv_net_2,
metrics=accuracy,
model_dir=MODEL_DIR,
opt_func=RAdam,
loss_func=nn.CrossEntropyLoss()
)<find_best_params>
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [2],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,500],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=k_fold, scoring="accuracy", n_jobs=-1, verbose = 1)
 gsRFC.fit(train_data,target)
 RFC_best = gsRFC.best_estimator_
learner2.lr_find()
learner2.recorder.plot(suggestion=True )<train_model>
gsRFC.best_score_
%%time
learner2.fit_one_cycle(50,
slice(1e-03),
callbacks=[SaveModelCallback(learner2,
every='improvement',
monitor='accuracy',
name='best_model_2')]
)<save_to_csv>
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [1000,2000,3000],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [19],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=k_fold, scoring="accuracy", n_jobs= -1, verbose = 1)
 gsGBC.fit(train_data,target)
 GBC_best = gsGBC.best_estimator_
learner2.load('best_model_2')
 preds, ids = learner2.get_preds(DatasetType.Test)
 y = torch.argmax(preds, dim=1)
 submission_2 = pd.DataFrame({ 'id': ids,'label': y })
 submission_2.to_csv("submission_2.csv", index=False )<choose_model_class>
gsGBC.best_score_
learner3 = Learner(data,
conv_net_3,
metrics=accuracy,
model_dir=MODEL_DIR,
opt_func=Adam,
loss_func=nn.CrossEntropyLoss()
)<find_best_params>
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [ 0.001, 0.01, 0.1, 1, 10, 100, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=k_fold, scoring="accuracy", n_jobs=-1, verbose = 1)
 gsSVMC.fit(train_data,target)
 SVMC_best = gsSVMC.best_estimator_
learner3.lr_find()
learner3.recorder.plot(suggestion=True )<train_model>
gsSVMC.best_score_
%%time
learner3.fit_one_cycle(50,
slice(8e-03),
callbacks=[SaveModelCallback(learner3,
every='improvement',
monitor='accuracy',
name='best_model_3')]
)<save_to_csv>
votingC = VotingClassifier(estimators=[('rfc', RFC_best),('extc', ExtC_best), 
('svc', SVMC_best),('adac',ada_best),('gbc',GBC_best)], voting='soft', n_jobs=-1)
 votingC = votingC.fit(train_data, target )
learner3.load('best_model_3')
 preds, ids = learner3.get_preds(DatasetType.Test)
 y = torch.argmax(torch.exp(preds), dim=1)
 submission_3 = pd.DataFrame({ 'id': ids,'label': y })
 submission_3.to_csv("submission_3.csv", index=False )<choose_model_class>
test_data = test.drop("PassengerId", axis=1 ).copy()
prediction = votingC.predict(test_data )
<find_best_params><EOS> 
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": prediction
})
 submission.to_csv('submission.csv', index=False )