Kaggle competition: Titanic - Machine Learning from Disaster
print('The sizes of the datasets are:')
 print('Training Dataset: ', df_train.shape)
 print('Testing Dataset: ', df_test.shape )<sort_values>
warnings.filterwarnings('ignore')
group_tmp = df_train[df_train['matchId']=='df014fbee741c6']['groupId'].value_counts().sort_values(ascending=False )<set_options>
train = pd.read_csv(".. /input/train.csv")
 train['label'] = 'train'
test = pd.read_csv(".. /input/test.csv")
 test['label'] = 'test'
test_passengerId = test.PassengerId
df = train.append(test)
 df.sample(2 )
warnings.filterwarnings('ignore' )<load_from_csv>
df.isnull().sum()
def feature_engineering(is_train=True,debug=True):
test_idx = None
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 test_idx = df.Id
df = reduce_mem_usage(df)
 print("remove some columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
df['Embarked'].fillna('S', inplace = True )
X_train, y_train, train_columns, _ = feature_engineering(True,False)
 X_test, _, _ , test_idx = feature_engineering(False,True )<drop_column>
df[df.Fare.isnull() ]
X_train =reduce_mem_usage(X_train)
 X_test = reduce_mem_usage(X_test )<train_model>
print(df[df.Pclass == 1].Fare.quantile([0.25, 0.50, 0.75]))
print(df[df.Pclass == 2].Fare.quantile([0.25, 0.50, 0.75]))
print(df[df.Pclass == 3].Fare.quantile([0.25, 0.50, 0.75]))
LR_model = LinearRegression(n_jobs=4, normalize=True)
 LR_model.fit(X_train,y_train )<compute_test_metric>
df['Fare'].fillna(df[df.Pclass == 3].Fare.median() , inplace = True )
LR_model.score(X_train,y_train )<predict_on_test>
print("Age column has", df.Age.isnull().sum() , "missing values out of", len(df), ".Missing value percentage =", df.Age.isnull().sum() /len(df)*100 )
y_pred_train = LR_model.predict(X_train)
 y_pred_test = LR_model.predict(X_test )<feature_engineering>
df.pivot_table(values = 'Age', index = ['Pclass', 'SibSp', 'Parch'], aggfunc = 'median' )
y_pred_test[y_pred_test>1] = 1
y_pred_test[y_pred_test<0] = 0<save_to_csv>
df.Age.isnull().sum()
df_test['winPlacePerc'] = y_pred_test
submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<load_from_csv>
age_null = df.Age.isnull()
group_med_age = df.pivot_table(values = 'Age', index = ['Pclass', 'SibSp'], aggfunc = 'median')
 df.loc[age_null, 'Age'] = df.loc[age_null, ['Pclass', 'SibSp']].apply(lambda x: group_med_age.loc[(group_med_age.index.get_level_values('Pclass')== x.Pclass)&(group_med_age.index.get_level_values('SibSp')== x.SibSp)].Age.values[0], axis = 1 )
train=pd.read_csv(".. /input/train_V2.csv")
 test=pd.read_csv(".. /input/test_V2.csv" )<categorify>
df.Age.isnull().sum()
le=LabelEncoder()
enc=OneHotEncoder()
train.loc[(train.matchType!='solo')&(train.matchType!='duo')&(train.matchType!='squad')&(train.matchType!='solo-fpp')&(train.matchType!='duo-fpp')&(train.matchType!='squad-fpp'),'matchType']='other'
train['matchType']=train['matchType'].map({'solo':0 , 'duo':1, 'squad':2, 'solo-fpp':3, 'duo-fpp':4, 'squad-fpp':5,'other':6} )<count_missing_values>
print("Cabin has", df.Cabin.isnull().sum() , "missing values out of", len(df))
train.dropna(inplace=True)
 train.isnull().sum()<categorify>
df.Cabin.fillna('O', inplace = True )
data=enc.fit(train[['matchType']])
 temp=enc.transform(train[['matchType']] )<create_dataframe>
df.isnull().sum()
temp1=pd.DataFrame(temp.toarray() ,columns=["solo", "duo", "squad", "solo-fpp", "duo-fpp", "squad-fpp","other"])
 temp1=temp1.set_index(train.index.values)
 temp1
train=pd.concat([train,temp1],axis=1 )<drop_column>
df['Fare_log'] = df.Fare.map(lambda i: np.log(i)if i > 0 else 0 )
train['killsasist']=train['kills']+train['assists']+train['roadKills']
train['total_distance']=train['swimDistance']+train['rideDistance']+train['walkDistance']
train['external_booster']=train['boosts']+train['weaponsAcquired']+train['heals']
train=train.drop(['assists','kills','swimDistance','rideDistance','walkDistance','boosts','weaponsAcquired','heals','roadKills','rankPoints'],axis=1 )<drop_column>
df['Family_size'] = 1 + df.Parch + df.SibSp
df['Alone'] = np.where(df.Family_size == 1, 1, 0 )
train=train.drop(['killPoints','maxPlace','winPoints'],axis=1 )<categorify>
print(df.Family_size.value_counts())
 print(df.Alone.value_counts() )
train['Players_all']=train.groupby('matchId')['Id'].transform('count')
 train['players_group']=train.groupby('groupId')['Id'].transform('count' )<prepare_x_and_y>
df.loc[df['Family_size'] == 1, 'Family_size_bin'] = 0
df.loc[(df['Family_size'] >= 2)&(df['Family_size'] <= 4), 'Family_size_bin'] = 1
df.loc[df['Family_size'] >=5, 'Family_size_bin'] = 2
Y=train.winPlacePerc
train = train.drop(["Id", "groupId", "matchId","winPlacePerc"], axis=1)
 del train['matchType']
train.head()<train_model>
df['Title'] = df.Name.str.split(", ", expand = True)[1].str.split(".", expand = True)[0]
df.Title.value_counts()
d_train = lgb.Dataset(train, label=Y)
 params = {}
params['learning_rate'] = 0.05
params['boosting_type'] = 'gbdt'
params['objective'] = 'regression'
params['metric'] = 'mae'
params['sub_feature'] = 0.9
params['num_leaves'] = 511
params['min_data'] = 1
params['max_depth'] = 30
params['min_gain_to_split']= 0.00001
clf = lgb.train(params, d_train,2000 )<categorify>
minor_titles = df.Title.value_counts() <= 4
df['Title'] = df.Title.apply(lambda x: 'Others' if minor_titles.loc[x] == True else x)
 df.Title.value_counts()
test.loc[(test.matchType!='solo')&(test.matchType!='duo')&(test.matchType!='squad')&(test.matchType!='solo-fpp')&(test.matchType!='duo-fpp')&(test.matchType!='squad-fpp'),'matchType']='other'
test['matchType']=test['matchType'].map({'solo':0 , 'duo':1, 'squad':2, 'solo-fpp':3, 'duo-fpp':4, 'squad-fpp':5,'other':6} )<categorify>
df['Fare_bin'] = pd.qcut(df.Fare, 4, labels = [0,1,2,3] ).astype(int)
 df['Age_bin'] = pd.cut(df.Age.astype(int), 5, labels = [0,1,2,3,4] ).astype(int )
data_test=enc.fit(test[['matchType']])
 temp_test=enc.transform(test[['matchType']])
 temp2=pd.DataFrame(temp_test.toarray() ,columns=["solo", "duo", "squad", "solo-fpp", "duo-fpp", "squad-fpp","other"])
 temp2=temp2.set_index(test.index.values)
 temp2
test=pd.concat([test,temp2],axis=1)
 del test['matchType'] 
<drop_column>
label = LabelEncoder()
df['Title'] = label.fit_transform(df.Title)
 df['Sex'] = label.fit_transform(df.Sex)
 df['Embarked'] = label.fit_transform(df.Embarked)
 df['Cabin'] = label.fit_transform(df.Cabin )
test['killsasist']=test['kills']+test['assists']+test['roadKills']
test['total_distance']=test['swimDistance']+test['rideDistance']+test['walkDistance']
test['external_booster']=test['boosts']+test['weaponsAcquired']+test['heals']
test=test.drop(['assists','kills','swimDistance','rideDistance','walkDistance','boosts','weaponsAcquired','heals','roadKills','rankPoints'],axis=1 )<drop_column>
corr_columns = list(df.drop(['Name', 'PassengerId', 'Ticket', 'label'], axis = 1 ).columns )
test=test.drop(['killPoints','maxPlace','winPoints'],axis=1 )<categorify>
df['Ticket'] = df.Ticket.map(lambda x: re.sub(r'\W+', '', x))
test['Players_all']=test.groupby('matchId')['Id'].transform('count')
 test['players_group']=test.groupby('groupId')['Id'].transform('count' )<drop_column>
Ticket = []
for i in list(df.Ticket):
if not i.isdigit() :
Ticket.append(i[:2])
 else:
Ticket.append("X")
 df['Ticket'] = Ticket
test_id=test.Id
test = test.drop(["Id", "groupId", "matchId"], axis=1)
 test.head()<save_to_csv>
df = pd.get_dummies(df, columns = ['Ticket'], prefix = 'T' )
out=clf.predict(test)
 submission=pd.DataFrame({'Id':test_id,'winPlacePerc':out})
 submission.to_csv('submission.csv', index=False )<import_modules>
cat_variables = [x for x in df.columns if df.dtypes[x] == 'object']
cat_variables
from IPython.display import Image<install_modules>
df.drop(['Name', 'PassengerId'], axis = 1, inplace = True )
!pip install image-classifiers<import_modules>
train = df.loc[df.label == 'train'].drop('label', axis = 1)
 test = df.loc[df.label == 'test'].drop(['label', 'Survived'], axis = 1 )
tf.__version__<define_variables>
X_train = train.drop(['Survived'], axis = 1)
 y_train = train['Survived'].astype(int)
 X_test = test
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )<choose_model_class>
kfold = StratifiedKFold(n_splits = 5 )
ResNet34, preprocess_input = Classifiers.get('resnet34' )<init_hyperparams>
classifiers = []
classifiers.append(KNeighborsClassifier())
 classifiers.append(LinearDiscriminantAnalysis())
 classifiers.append(LogisticRegression(random_state = 0))
classifiers.append(LinearSVC(random_state = 0))
classifiers.append(SVC(random_state = 0))
classifiers.append(RandomForestClassifier(random_state = 0))
classifiers.append(ExtraTreesClassifier(random_state = 0))
classifiers.append(XGBClassifier(random_state = 0))
classifiers.append(LGBMClassifier(random_state = 0))
classifiers.append(MLPClassifier())
 cv_results = []
for classifier in classifiers:
cv_results.append(cross_val_score(classifier, X_train, y_train, scoring = 'accuracy', cv = kfold, n_jobs = -1))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({'CV_means':cv_means, 'CV_std':cv_std, 'Algorithm':['KNN', 'LinearDiscriminantAnalysis', 'LogisticRegression', 'LinearSVC', 'SVC', 'RandomForest', 'ExtraTrees', 'XGB', 'LGB', 'MLP']})
cfg = {
'parse_params': {
'cut_time': 10,
},
'data_params': {
'sample_time': 6,
'spec_fmax': 24000.0,
'spec_fmin': 40.0,
'spec_mel': 384,
'mel_power': 2,
'img_shape':(384, 768)
 },
'model_params': {
'batchsize_per_tpu': 18,
'iteration_per_epoch': 64,
'epoch': 18,
'arch': ResNet34,
'arch_preprocess': preprocess_input,
'freeze_to': 0,
'loss': {
'fn': tfa.losses.SigmoidFocalCrossEntropy,
'params': {},
},
'optim': {
'fn': tfa.optimizers.RectifiedAdam,
'params': {'lr': 2e-3, 'total_steps': 18*64, 'warmup_proportion': 0.3, 'min_lr': 1e-6},
},
'mixup': True
}
}<define_variables>
LDA_best = LinearDiscriminantAnalysis().fit(X_train, y_train )
strategy = tf.distribute.experimental.TPUStrategy(tpu)
 AUTOTUNE = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('rfcx-species-audio-detection')
 TRAIN_TFREC = GCS_DS_PATH + "/tfrecords/train"
TEST_TFREC = GCS_DS_PATH + "/tfrecords/test"<define_variables>
RF = RandomForestClassifier(random_state = 0)
 RF_params = {'n_estimators' : [10,50,100],
'criterion' : ['gini', 'entropy'],
'max_depth' : [5,8,None],
'min_samples_split' : [2,5,8],
'min_samples_leaf' : [1,3,5],
'max_features' : ['auto', 'log2', None]}
GS_RF = GridSearchCV(RF, param_grid = RF_params, cv = kfold, scoring = 'accuracy', n_jobs = -1, verbose = 1)
 GS_RF.fit(X_train, y_train)
 RF_best = GS_RF.best_estimator_
print("Best parameters :", RF_best)
 print("Best score :", GS_RF.best_score_ )
CUT = cfg['parse_params']['cut_time']
SR = 48000
TIME = cfg['data_params']['sample_time']
FMAX = cfg['data_params']['spec_fmax']
FMIN = cfg['data_params']['spec_fmin']
N_MEL = cfg['data_params']['spec_mel']
HEIGHT, WIDTH = cfg['data_params']['img_shape']
CLASS_N = 24<create_dataframe>
ET = ExtraTreesClassifier(random_state = 0)
 ET_params = {'n_estimators' : [10,50,100],
'criterion' : ['gini', 'entropy'],
'max_depth' : [5,8,None],
'min_samples_split' : [2,5,8],
'min_samples_leaf' : [1,3,5],
'max_features' : ['auto', 'log2', None]}
GS_ET = GridSearchCV(ET, param_grid = ET_params, cv= kfold, scoring = 'accuracy', n_jobs = -1, verbose = 1)
 GS_ET.fit(X_train, y_train)
 ET_best = GS_ET.best_estimator_
print('Best parameters :', ET_best)
 print('Best score :', GS_ET.best_score_ )
raw_dataset = tf.data.TFRecordDataset([TRAIN_TFREC + '/00-148.tfrec'])
 raw_dataset<categorify>
warnings.filterwarnings('ignore' )
feature_description = {
'recording_id': tf.io.FixedLenFeature([], tf.string, default_value=''),
'audio_wav': tf.io.FixedLenFeature([], tf.string, default_value=''),
'label_info': tf.io.FixedLenFeature([], tf.string, default_value=''),
}
parse_dtype = {
'audio_wav': tf.float32,
'recording_id': tf.string,
'species_id': tf.int32,
'songtype_id': tf.int32,
't_min': tf.float32,
'f_min': tf.float32,
't_max': tf.float32,
'f_max':tf.float32,
'is_tp': tf.int32
}
@tf.function
def _parse_function(example_proto):
sample = tf.io.parse_single_example(example_proto, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 label_info = tf.strings.split(sample['label_info'], sep='"')[1]
labels = tf.strings.split(label_info, sep=';')
 @tf.function
def _cut_audio(label):
items = tf.strings.split(label, sep=',')
 spid = tf.squeeze(tf.strings.to_number(items[0], tf.int32))
soid = tf.squeeze(tf.strings.to_number(items[1], tf.int32))
tmin = tf.squeeze(tf.strings.to_number(items[2]))
fmin = tf.squeeze(tf.strings.to_number(items[3]))
tmax = tf.squeeze(tf.strings.to_number(items[4]))
fmax = tf.squeeze(tf.strings.to_number(items[5]))
tp = tf.squeeze(tf.strings.to_number(items[6], tf.int32))
tmax_s = tmax * tf.cast(SR, tf.float32)
 tmin_s = tmin * tf.cast(SR, tf.float32)
 cut_s = tf.cast(CUT * SR, tf.float32)
 all_s = tf.cast(60 * SR, tf.float32)
 tsize_s = tmax_s - tmin_s
cut_min = tf.cast(
 tf.maximum(0.0,
tf.minimum(tmin_s -(cut_s - tsize_s)/ 2,
tf.minimum(tmax_s +(cut_s - tsize_s)/ 2, all_s)- cut_s)
), tf.int32 
)
 cut_max = cut_min + CUT * SR
_sample = {
'audio_wav': tf.reshape(wav[cut_min:cut_max], [CUT*SR]),
'recording_id': sample['recording_id'],
'species_id': spid,
'songtype_id': soid,
't_min': tmin - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),
'f_min': fmin,
't_max': tmax - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),
'f_max': fmax,
'is_tp': tp
}
return _sample
samples = tf.map_fn(_cut_audio, labels, dtype=parse_dtype)
 return samples
parsed_dataset = raw_dataset.map(_parse_function ).unbatch()<feature_engineering>
XGB = XGBClassifier(random_state = 0)
 XGB_params = {'n_estimators' : [100,200,500],
'max_depth' : [3,4,5],
'learning_rate' : [0.01,0.05,0.1,0.2],
'booster' : ['gbtree', 'gblinear', 'dart']}
GS_XGB = GridSearchCV(XGB, param_grid = XGB_params, cv= kfold, scoring = 'accuracy', n_jobs = -1, verbose = 1)
 GS_XGB.fit(X_train, y_train)
 XGB_best = GS_XGB.best_estimator_
print('Best parameters :', XGB_best)
 print('Best score :', GS_XGB.best_score_ )
@tf.function
def _cut_wav(x):
cut_min = tf.random.uniform([], maxval=(CUT-TIME)*SR, dtype=tf.int32)
 cut_max = cut_min + TIME * SR
cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])
 y = {}
y.update(x)
 y['audio_wav'] = cutwave
y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32)/ SR)
 y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32)/ SR)
 return y
@tf.function
def _cut_wav_val(x):
cut_min =(CUT-TIME)*SR // 2
cut_max = cut_min + TIME * SR
cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])
 y = {}
y.update(x)
 y['audio_wav'] = cutwave
y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32)/ SR)
 y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32)/ SR)
 return y<feature_engineering>
LGB = LGBMClassifier(random_state = 0)
 LGB_params = {'n_estimators' : [100,200,500],
'max_depth' : [5,8,-1],
'learning_rate' : [0.01,0.05,0.1,0.2],
'boosting_type' : ['gbdt', 'goss', 'dart']}
GS_LGB = GridSearchCV(LGB, param_grid = LGB_params, cv= kfold, scoring = 'accuracy', n_jobs = -1, verbose = 1)
 GS_LGB.fit(X_train, y_train)
 LGB_best = GS_LGB.best_estimator_
print('Best parameters :', LGB_best)
 print('Best score :', GS_LGB.best_score_ )
@tf.function
def _filtTP(x):
return x['is_tp'] == 1<data_type_conversions>
MLP = MLPClassifier(random_state = 0)
 MLP_params = {'hidden_layer_sizes' : [[10], [10,10], [10,100], [100,100]],
'activation' : ['relu', 'tanh', 'logistic'],
'alpha' : [0.0001,0.001,0.01]}
GS_MLP = GridSearchCV(MLP, param_grid = MLP_params, cv= kfold, scoring = 'accuracy', n_jobs = -1, verbose = 1)
 GS_MLP.fit(X_train, y_train)
 MLP_best = GS_MLP.best_estimator_
print('Best parameters :', MLP_best)
 print('Best score :', GS_MLP.best_score_ )
def show_wav(sample, ax):
wav = sample["audio_wav"].numpy()
rate = SR
ax.plot(np.arange(len(wav)) / rate, wav)
 ax.set_title(
 sample["recording_id"].numpy().decode()
+("/%d" % sample["species_id"])
 +("TP" if sample["is_tp"] else "FP"))
return Audio(( wav * 2**15 ).astype(np.int16), rate=rate)
 fig, ax = plt.subplots(figsize=(15, 3))
show_wav(next(iter(parsed_dataset)) , ax )<normalization>
voting = VotingClassifier(estimators = [['LDA', LDA_best], ["MLP", MLP_best],
['RFC', RF_best], ['ETC', ET_best],
['XGB', XGB_best], ['LGB', LGB_best]], voting = 'soft', n_jobs = -1)
 voting = voting.fit(X_train, y_train )
<categorify><EOS> 
results = pd.DataFrame(test_passengerId, columns = ['PassengerId'] ).assign(Survived = pd.Series(voting.predict(X_test)))
 results.to_csv('models_voting.csv', index = None )