Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<set_options>

 print(os.listdir(".. /input"))
warnings.filterwarnings('ignore')
df_train = reduce_mem_usage(df_train)
 df_test = reduce_mem_usage(df_test)
 gc.collect()<drop_column>


try:
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 print('Files are loaded')
 except:
print('Something went wrong.' )
train_columns = list(df_test.columns)
 train_idx = df_train.Id
test_idx = df_test.Id
train_columns.remove("Id")
 train_columns.remove("matchId")
 train_columns.remove("groupId" )<prepare_x_and_y>
ship = df_train.append(df_test, ignore_index=True)
x_train = df_train[train_columns]
x_test = df_test[train_columns]
y_train = df_train["winPlacePerc"].astype('float' )<categorify>
df_train['Survived'].value_counts(normalize=True )
encoded_train = pd.get_dummies(x_train.matchType, prefix=x_train.matchType.name ,prefix_sep="_")
 encoded_test = pd.get_dummies(x_test.matchType, prefix=x_test.matchType.name ,prefix_sep="_")
 encoded_train.head()<merge>
print('"ship" columns with null values:
', ship.isnull().sum() )
x_train = x_train.merge(encoded_train, right_index=True, left_index=True)
 x_test = x_test.merge(encoded_test, right_index=True, left_index=True )<drop_column>
try:
ship_orig = ship.copy()
print('Copies have been made')
 except:
print('Something went wrong.' )
del x_train['matchType']
del x_test['matchType']<drop_column>
columns_to_drop = ['PassengerId', 'Ticket']
ship = ship.drop(columns_to_drop, axis=1)
 print('The following columns have been dropped: ', columns_to_drop)
 del columns_to_drop
del df_train
del df_test
gc.collect()<split>
print('Missing Values in "ship" data: ', list(col for col in ship.columns if ship[col].isnull().any() == True))
folds = KFold(n_splits=3,random_state=6)
 oof_preds = np.zeros(x_train.shape[0])
 sub_preds = np.zeros(x_test.shape[0])
 start = time.time()
valid_score = 0
importances = pd.DataFrame()
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(x_train, y_train)) :
trn_x, trn_y = x_train.iloc[trn_idx], y_train[trn_idx]
val_x, val_y = x_train.iloc[val_idx], y_train[val_idx]
train_data = lgb.Dataset(data=trn_x, label=trn_y)
 valid_data = lgb.Dataset(data=val_x, label=val_y)
 params = {"objective" : "regression",
"metric" : "mae",
'n_estimators':10000,
'early_stopping_rounds':100,
"num_leaves" : 30,
"learning_rate" : 0.3,
"bagging_fraction" : 0.9,
"bagging_seed" : 0}
lgb_model = lgb.train(params, train_data, valid_sets=[train_data, valid_data], verbose_eval=1000)
 oof_preds[val_idx] = lgb_model.predict(val_x, num_iteration=lgb_model.best_iteration)
 oof_preds[oof_preds>1] = 1
oof_preds[oof_preds<0] = 0
sub_pred = lgb_model.predict(x_test, num_iteration=lgb_model.best_iteration)/ folds.n_splits
sub_pred[sub_pred>1] = 1
sub_pred[sub_pred<0] = 0
sub_preds += sub_pred
print('Fold %2d RMSE : %.6f' %(n_fold + 1, mean_absolute_error(val_y, oof_preds[val_idx])))
 valid_score += mean_absolute_error(val_y, oof_preds[val_idx] )<train_model>
ship['Embarked'].value_counts()
print('Done' )<save_to_csv>
ship["Embarked"] = ship["Embarked"].fillna("S")
 print('Embarked column blanks have been filled with the value "S"' )
test_pred = pd.DataFrame({"Id":test_idx})
 test_pred["winPlacePerc"] = sub_preds
test_pred.columns = ["Id", "winPlacePerc"]
test_pred.to_csv("lgb_model_181204.csv", index=False )<set_options>
median_value = ship["Fare"].median()
print('The median value for the "Fare" column is: ', median_value)
 ship["Fare"].fillna(median_value, inplace=True)
 print('Fare column blanks have been filled with the median value of the column')
 del median_value
warnings.filterwarnings('ignore')
 %matplotlib inline
py.init_notebook_mode(connected=True)
<load_from_csv>
print('Number of null values in Age column:', ship['Age'].isnull().sum())
 print('The mean of the Age column: ', ship['Age'].mean() )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<filter>
imputer_tool = SimpleImputer()
ship_numeric = ship.select_dtypes(exclude=['object'])
 cols_with_missing = ['Age']
for col in cols_with_missing:
ship_numeric.loc[:, col + '_was_missing'] = ship_numeric[col].isnull()
ship_numeric_imp = imputer_tool.fit_transform(ship_numeric.values)
 ship_numeric = pd.DataFrame(ship_numeric_imp, index=ship_numeric.index, columns=ship_numeric.columns)
 ship_numeric.head()
df_train[df_train['groupId']==24]<filter>
ship_numeric.loc[lambda df: df.Age_was_missing == 1, :][:5]
len(df_train[df_train['matchId']==0] )<count_values>
ship['Age'] = ship_numeric['Age'].copy()
print('Are there any null values?', ship['Age'].isnull().any())
 ship['Age'] = ship_numeric['Age'].copy()
ship['Age_was_missing'] = ship_numeric['Age_was_missing'].copy()
del ship_numeric, ship_numeric_imp
print('Age data has been copied back to main dataset')
 print('Age_was_missing column was added to main dataset' )
df_train['roadKills'].value_counts()<count_values>
print('NaN "Cabin" values in "ship" dataset: %s out of %s' %(ship['Cabin'].isnull().sum() , len(ship)) )
df_train['teamKills'].value_counts()<count_values>
cabin_notnull = df_train['Cabin'].loc[df_train['Cabin'].notnull() ].astype(str ).str[0]
cabin_notnull = pd.DataFrame([cabin_notnull, df_train['Survived'].loc[df_train['Cabin'].notnull() ]] ).T
df_train['headshotKills'].value_counts()<count_values>
cabin_notnull['Cabin'].value_counts()
df_train['vehicleDestroys'].value_counts()<feature_engineering>
ship.drop("Cabin", axis=1, inplace=True)
 print('Cabin column has been dropped.' )
headshot = df_train[['kills','winPlacePerc','headshotKills']]
headshot['headshotrate'] = headshot['headshotKills']/headshot['kills']<feature_engineering>
embark_dummies_titanic = pd.get_dummies(ship['Embarked'])
 embark_dummies_titanic.drop(['S'], axis=1, inplace=True)
 ship = ship.join(embark_dummies_titanic)
 ship.drop(['Embarked'], axis=1,inplace=True)
 print('Embarked column has been dropped.C and Q columns have been added.' )
df_train['headshotrate'] = df_train['headshotKills']/df_train['kills']
df_test['headshotrate'] = df_test['headshotKills']/df_test['kills']<feature_engineering>
pclass_dummies_titanic = pd.get_dummies(ship['Pclass'])
 pclass_dummies_titanic.columns = ['Class_1','Class_2','Class_3']
pclass_dummies_titanic.drop(['Class_3'], axis=1, inplace=True)
 ship.drop(['Pclass'],axis=1,inplace=True)
 ship = ship.join(pclass_dummies_titanic)
 print('Pclass column has been changed.' )
killStreak = df_train[['kills','winPlacePerc','killStreaks']]
killStreak['killStreakrate'] = killStreak['killStreaks']/killStreak['kills']
killStreak.corr()<feature_engineering>
ship['Family'] = ship["Parch"] + ship["SibSp"]
ship['Family'].loc[ship['Family'] > 0] = 1
ship['Family'].loc[ship['Family'] == 0] = 0


ship = ship.drop(['SibSp','Parch'], axis=1)
 print('SibSp and Parch columns have been dropped.Family column has been added.' )
del killStreak
df_train['killStreakrate'] = -(df_train['killStreaks']/df_train['kills'])
 df_test['killStreakrate'] = -(df_test['killStreaks']/df_test['kills'])
 del df_train['killStreaks']; del df_test['killStreaks']<feature_engineering>
def get_person(passenger):
age, sex = passenger
return 'child' if age < 18 else sex
ship['Person'] = ship[['Age','Sex']].apply(get_person, axis=1)
 ship.drop(['Sex'],axis=1,inplace=True)
 person_dummies_titanic = pd.get_dummies(ship['Person'])
 person_dummies_titanic.columns = ['Child','Female','Male']
person_dummies_titanic.drop(['Male'], axis=1, inplace=True)
 ship = ship.join(person_dummies_titanic)
 ship.drop(['Person'],axis=1,inplace=True)
 print('Age and Sex columns have been dropped.Child and Female columns have been added.' )
df_train['hacker_pt'] = 0
df_test['hacker_pt'] = 0<feature_engineering>
ship['Title'] = ship['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
print('Title column has been created with values')
 ship.head()
df_train['total_Distance'] = df_train['rideDistance'] + df_train["walkDistance"] + df_train["swimDistance"]
df_test['total_Distance'] = df_test['rideDistance'] + df_test["walkDistance"] + df_test["swimDistance"]<feature_engineering>
ship['Title'].value_counts()
df_train['hacker_pt'][(df_train['heals'] + df_train['boosts'] < 1)&(df_train['total_Distance'] < 100)&(df_train['kills'] > 20)] = 1
df_test['hacker_pt'][(df_test['heals'] + df_test['boosts'] < 1)&(df_test['total_Distance'] < 100)&(df_test['kills'] > 20)] = 1<filter>
ship.groupby('Title', sort=False)['Age'].agg(['mean', 'min', 'median', 'max', 'count'] )
df_train['hacker_pt'][(df_train['kills'] > 10)&(df_train['weaponsAcquired'] >= 10)&(df_train['total_Distance'] == 0)] += 1
df_test['hacker_pt'][(df_test['kills'] > 10)&(df_test['weaponsAcquired'] >= 10)&(df_test['total_Distance'] == 0)] += 1<drop_column>
ship['Title'].loc[ship['Title'] == 'the Countess'] = 'Mrs'
ship['Title'].loc[ship['Title'] == 'Ms'] = 'Mrs'
ship['Title'].loc[ship['Title'] == 'Lady'] = 'Mrs'
ship['Title'].loc[ship['Title'] == 'Dona'] = 'Mrs'
ship['Title'].loc[ship['Title'] == 'Mlle'] = 'Miss'
ship['Title'].loc[ship['Title'] == 'Mme'] = 'Miss'
print('Rare lady titles have been added to larger group titles' )
del healthitems<feature_engineering>
print(ship['Title'].value_counts())
 stat_min = 10
title_names =(ship['Title'].value_counts() < stat_min)
 ship['Title'] = ship['Title'].apply(lambda x: 'Misc' if title_names.loc[x] == True else x)
 print(ship['Title'].value_counts())
 print("-"*10)
 print('Rare title names have been combined into a "Misc" option' )
kills = df_train[['assists','winPlacePerc','kills']]
kills['kills_assists'] =(kills['kills'] + kills['assists'])
 kills.corr()<drop_column>
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
del kills
df_train['kills_assists'] = df_train['kills'] + df_train['assists']
df_test['kills_assists'] = df_test['kills'] + df_test['assists']
del df_train['kills']; del df_test['kills']<drop_column>
label = LabelEncoder()
ship['Title_Code'] = label.fit_transform(ship['Title'])
 print('Fit Transform function has been run.')
 ship['Title_Code'].head()
df_train = reduce_mem_usage(df_train)
 df_test = reduce_mem_usage(df_test)
 gc.collect()<groupby>
ship.drop(['Name'],axis=1,inplace=True)
 print('Name column has been dropped.' )
df_train_size = df_train.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_test_size = df_test.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_train_mean = df_train.groupby(['matchId','groupId'] ).mean().reset_index()
df_test_mean = df_test.groupby(['matchId','groupId'] ).mean().reset_index()
df_train_max = df_train.groupby(['matchId','groupId'] ).max().reset_index()
df_test_max = df_test.groupby(['matchId','groupId'] ).max().reset_index()
df_train_min = df_train.groupby(['matchId','groupId'] ).min().reset_index()
df_test_min = df_test.groupby(['matchId','groupId'] ).min().reset_index()

 df_train_match_mean = df_train.groupby(['matchId'] ).mean().reset_index()
df_test_match_mean = df_test.groupby(['matchId'] ).mean().reset_index()

 df_train = pd.merge(df_train, df_train_mean, suffixes=["", "_mean"], how='left', on=['matchId', 'groupId'])
 df_test = pd.merge(df_test, df_test_mean, suffixes=["", "_mean"], how='left', on=['matchId', 'groupId'])
 del df_train_mean
del df_test_mean
df_train = pd.merge(df_train, df_train_max, suffixes=["", "_max"], how='left', on=['matchId', 'groupId'])
 df_test = pd.merge(df_test, df_test_max, suffixes=["", "_max"], how='left', on=['matchId', 'groupId'])
 del df_train_max
del df_test_max
df_train = pd.merge(df_train, df_train_min, suffixes=["", "_min"], how='left', on=['matchId', 'groupId'])
 df_test = pd.merge(df_test, df_test_min, suffixes=["", "_min"], how='left', on=['matchId', 'groupId'])
 del df_train_min
del df_test_min
df_train = pd.merge(df_train, df_train_match_mean, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 df_test = pd.merge(df_test, df_test_match_mean, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 del df_train_match_mean
del df_test_match_mean
df_train = pd.merge(df_train, df_train_size, how='left', on=['matchId', 'groupId'])
 df_test = pd.merge(df_test, df_test_size, how='left', on=['matchId', 'groupId'])
 del df_train_size
del df_test_size
gc.collect()
df_train = reduce_mem_usage(df_train)
 df_test = reduce_mem_usage(df_test)
 gc.collect()<set_options>
title_data = {}
title_data['Mr - Survived'] = ship['Title_Code'].loc[(ship['Title_Code'] == 3)&(ship['Survived'] == 1)].count()
title_data['Mr - Deceased'] = ship['Title_Code'].loc[(ship['Title_Code'] == 3)&(ship['Survived'] == 0)].count()
title_data['Miss - Survived'] = ship['Title_Code'].loc[(ship['Title_Code'] == 2)&(ship['Survived'] == 1)].count()
title_data['Miss - Deceased'] = ship['Title_Code'].loc[(ship['Title_Code'] == 2)&(ship['Survived'] == 0)].count()
title_data['Mrs - Survived'] = ship['Title_Code'].loc[(ship['Title_Code'] == 4)&(ship['Survived'] == 1)].count()
title_data['Mrs - Deceased'] = ship['Title_Code'].loc[(ship['Title_Code'] == 4)&(ship['Survived'] == 0)].count()
title_data['Master - Survived'] = ship['Title_Code'].loc[(ship['Title_Code'] == 0)&(ship['Survived'] == 1)].count()
title_data['Master - Deceased'] = ship['Title_Code'].loc[(ship['Title_Code'] == 0)&(ship['Survived'] == 0)].count()
title_data['Misc - Survived'] = ship['Title_Code'].loc[(ship['Title_Code'] == 1)&(ship['Survived'] == 1)].count()
title_data['Misc - Deceased'] = ship['Title_Code'].loc[(ship['Title_Code'] == 1)&(ship['Survived'] == 0)].count()
plt.figure(figsize=(16,4))
plt.bar(list(title_data.keys()),(list(title_data.values())))
 plt.show()
title_data
warnings.filterwarnings("ignore")
 color = sns.color_palette() 
<drop_column>
title_dummies_titanic = pd.get_dummies(ship['Title'])
 title_dummies_titanic.drop(['Misc', 'Mr'], axis=1, inplace=True)
 ship.drop(['Title', 'Title_Code'],axis=1,inplace=True)
 ship = ship.join(title_dummies_titanic)
 print('Title column has been dropped."Master", "Mrs", and "Miss" has been added.' )
train_columns = list(df_test.columns)
 train_idx = df_train.Id
test_idx = df_test.Id
train_columns.remove("Id")
 train_columns.remove("matchId")
 train_columns.remove("groupId" )<prepare_x_and_y>
print('Number of people who survived with a Fare > 50: ',
len(ship['Survived'].loc[(ship['Fare'] > 50)&(ship['Survived'] == 1)]))
print('Number of people who died with a Fare > 50: ',
len(ship['Survived'].loc[(ship['Fare'] > 50)&(ship['Survived'] == 0)]))
x_train = df_train[train_columns]
x_test = df_test[train_columns]
y_train = df_train["winPlacePerc"].astype('float' )<set_options>
ship['High_Fare'] = [1 if x > 50 else 0 for x in ship['Fare']]
print('High Fare column created.' )
del df_train; del df_test
gc.collect()<split>
ship.drop(['Age_was_missing'],axis=1,inplace=True)
 print('Dropped "Age_was_missing" column from dataset.' )
folds = KFold(n_splits=5,random_state=6)
 oof_preds = np.zeros(x_train.shape[0])
 sub_preds = np.zeros(x_test.shape[0])
 start = time.time()
valid_score = 0
importances = pd.DataFrame()
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(x_train, y_train)) :
trn_x, trn_y = x_train.iloc[trn_idx], y_train[trn_idx]
val_x, val_y = x_train.iloc[val_idx], y_train[val_idx]
train_data = lgb.Dataset(data=trn_x, label=trn_y)
 valid_data = lgb.Dataset(data=val_x, label=val_y)
 params = {"objective" : "regression", "metric" : "mae", 'n_estimators':10000, 'early_stopping_rounds':100,
"num_leaves" : 30, "learning_rate" : 0.3, "bagging_fraction" : 0.9,
"bagging_seed" : 0}
lgb_model = lgb.train(params, train_data, valid_sets=[train_data, valid_data], verbose_eval=1000)
 oof_preds[val_idx] = lgb_model.predict(val_x, num_iteration=lgb_model.best_iteration)
 oof_preds[oof_preds>1] = 1
oof_preds[oof_preds<0] = 0
sub_pred = lgb_model.predict(x_test, num_iteration=lgb_model.best_iteration)/ folds.n_splits
sub_pred[sub_pred>1] = 1
sub_pred[sub_pred<0] = 0
sub_preds += sub_pred
print('Fold %2d RMSE : %.6f' %(n_fold + 1, mean_absolute_error(val_y, oof_preds[val_idx])))
 valid_score += mean_absolute_error(val_y, oof_preds[val_idx] )<save_to_csv>
ship.drop(['Age'],axis=1,inplace=True )
test_pred = pd.DataFrame({"Id":test_idx})
 test_pred["winPlacePerc"] = sub_preds
test_pred.columns = ["Id", "winPlacePerc"]
test_pred.to_csv("lgb_base_model.csv", index=False )<set_options>
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
def is_interactive() :
return 'runtime' in get_ipython().config.IPKernelApp.connection_file
print('Interactive?', is_interactive())
 debug = False
use_amp = True<set_options>
df_tr = ship[ship.Survived.notnull() ]
df_tr.head()
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(21 )<import_modules>
df_te = ship[ship.Survived.isnull() ]
df_te.drop(['Survived'], axis=1, inplace=True)
 df_te = df_te.reset_index(drop=True)
 df_te.head()
torch.__version__, ignite.__version__<set_options>
x_train = df_tr.drop("Survived",axis=1)
 y_train = df_tr["Survived"]
x_test = df_te.copy()
print('Test and Train ML variables are ready.' )
%%time
if use_amp:
try:
except ImportError:
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/nvidia-apex/repository/* 
<normalization>
logreg = LogisticRegression()
logreg.fit(x_train, y_train)
 y_pred = logreg.predict(x_test)
 logreg.score(x_train, y_train )
class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 class Flatten(nn.Module):
def forward(self, x):
return x.reshape(x.shape[0], -1 )<define_search_model>
random_forest = RandomForestClassifier(n_estimators=300)
 random_forest.fit(x_train, y_train)
 y_pred = random_forest.predict(x_test ).astype(int)
 random_forest.score(x_train, y_train )
class SqueezeExcitation(nn.Module):
def __init__(self, inplanes, se_planes):
super(SqueezeExcitation, self ).__init__()
self.reduce_expand = nn.Sequential(
 nn.Conv2d(inplanes, se_planes,
kernel_size=1, stride=1, padding=0, bias=True),
Swish() ,
nn.Conv2d(se_planes, inplanes,
kernel_size=1, stride=1, padding=0, bias=True),
nn.Sigmoid() 
)
 def forward(self, x):
x_se = torch.mean(x, dim=(-2, -1), keepdim=True)
 x_se = self.reduce_expand(x_se)
 return x_se * x 
<choose_model_class>
df_test['Survived'] = y_pred.astype(int)
 df_test.to_csv('SurvivedList.csv')
 df_test
class MBConv(nn.Module):
def __init__(self, inplanes, planes, kernel_size, stride,
expand_rate=1.0, se_rate=0.25,
drop_connect_rate=0.2):
super(MBConv, self ).__init__()
expand_planes = int(inplanes * expand_rate)
 se_planes = max(1, int(inplanes * se_rate))
self.expansion_conv = None
if expand_rate > 1.0:
self.expansion_conv = nn.Sequential(
 nn.Conv2d(inplanes, expand_planes,
kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),
Swish() 
)
 inplanes = expand_planes
self.depthwise_conv = nn.Sequential(
 nn.Conv2d(inplanes, expand_planes,
kernel_size=kernel_size, stride=stride,
padding=kernel_size // 2, groups=expand_planes,
bias=False),
nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),
Swish() 
)
 self.squeeze_excitation = SqueezeExcitation(expand_planes, se_planes)
 self.project_conv = nn.Sequential(
 nn.Conv2d(expand_planes, planes,
kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(planes, momentum=0.01, eps=1e-3), 
)
 self.with_skip = stride == 1
self.drop_connect_rate = torch.tensor(drop_connect_rate, requires_grad=False)
 def _drop_connect(self, x):
keep_prob = 1.0 - self.drop_connect_rate
drop_mask = torch.rand(x.shape[0], 1, 1, 1)+ keep_prob
drop_mask = drop_mask.type_as(x)
 drop_mask.floor_()
return drop_mask * x / keep_prob
def forward(self, x):
z = x
if self.expansion_conv is not None:
x = self.expansion_conv(x)
 x = self.depthwise_conv(x)
 x = self.squeeze_excitation(x)
 x = self.project_conv(x)
 if x.shape == z.shape and self.with_skip:
if self.training and self.drop_connect_rate is not None:
self._drop_connect(x)
 x += z
return x<normalization>
submission = df_test[["PassengerId", "Survived"]]
<choose_model_class><EOS> 
submission.to_csv('titanic.csv', index=False, header=['PassengerID', 'Survived'] )