Kaggle competition: Titanic - Machine Learning from Disaster
agg_cols = ['card1','card2','card3','card5','uid','uid2','uid3','uid4']
def add_agg_col(col_prefix,agg_col,col_suffix='TransactionAmt'):
if isinstance(agg_col,list):
temp_df=pd.concat([train[[col_prefix,col_suffix]],test[[col_prefix,col_suffix]]])
 temp_df=temp_df.groupby(col_prefix)[col_suffix].agg(agg_col)
 for c in agg_col:
new_col=col_prefix+'_'+c+'_'+col_suffix
train[new_col]=train[col_prefix].map(temp_df[c])
 test[new_col]=test[col_prefix].map(temp_df[c])
 else:
raise TypeError('agg_col must be List')
 for i in agg_cols:
add_agg_col(i,['mean','std'])
 print(f'{i} for ['mean','std'] aggregate is done!' )<categorify>
pd.set_option('display.expand_frame_repr', False)
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
def id02(data):
data['id_02_to_mean_card1'] = data['id_02'] / data.groupby(['card1'])['id_02'].transform('mean')
 data['id_02_to_mean_card4'] = data['id_02'] / data.groupby(['card4'])['id_02'].transform('mean')
 data['id_02_to_std_card1'] = data['id_02'] / data.groupby(['card1'])['id_02'].transform('std')
 data['id_02_to_std_card4'] = data['id_02'] / data.groupby(['card4'])['id_02'].transform('std')
 id02(train)
 id02(test )<categorify>
import matplotlib.pyplot as plt
import seaborn as sns
def D15(data):
data['D15_to_mean_card1'] = data['D15'] / data.groupby(['card1'])['D15'].transform('mean')
 data['D15_to_mean_card4'] = data['D15'] / data.groupby(['card4'])['D15'].transform('mean')
 data['D15_to_std_card1'] = data['D15'] / data.groupby(['card1'])['D15'].transform('std')
 data['D15_to_std_card4'] = data['D15'] / data.groupby(['card4'])['D15'].transform('std')
 data['D15_to_mean_addr1'] = data['D15'] / data.groupby(['addr1'])['D15'].transform('mean')
 data['D15_to_mean_addr2'] = data['D15'] / data.groupby(['addr2'])['D15'].transform('mean')
 data['D15_to_std_addr1'] = data['D15'] / data.groupby(['addr1'])['D15'].transform('std')
 data['D15_to_std_addr2'] = data['D15'] / data.groupby(['addr2'])['D15'].transform('std')
 D15(train)
 D15(test )<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 validation = pd.read_csv(".. /input/test.csv" )
def device(data):
data['device_name'] = data['DeviceInfo'].str.split('/', expand=True)[0]
data['device_version'] = data['DeviceInfo'].str.split('/', expand=True)[1]
device(train)
 device(test )<feature_engineering>
print(train.dtypes)
 print()
print(train.isna().sum() )
def browser_OS(data):
data['OS_id_30'] = data['id_30'].str.split(' ', expand=True)[0]
data['version_id_30'] = data['id_30'].str.split(' ', expand=True)[1]
data['browser_id_31'] = data['id_31'].str.split(' ', expand=True)[0]
data['version_id_31'] = data['id_31'].str.split(' ', expand=True)[1]
browser_OS(train)
 browser_OS(test )<feature_engineering>
print(validation.dtypes)
 print()
print(validation.isna().sum() )
freq_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','DeviceType',
'id_30','id_33',
'uid','uid2','uid3','uid4'
]+date_col
def set_freq_col(train,test,col):
prefix='_fq'
temp_df=pd.concat([train[[col]],test[[col]]])
 fq=temp_df[col].value_counts(dropna=False)
 train[col+prefix]=train[col].map(fq)
 test[col+prefix]=test[col].map(fq)
 for c in freq_cols:
set_freq_col(train,test,c)
 periods = ['month','year_weekday','weekday_hour']
uids = ['uid','uid2','uid3','uid4']
def set_uid_period(train,test,periods,uids):
for period in periods:
for col in uids:
new_column = col + '_' + period
temp_df = pd.concat([train[[col,period]], test[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period+'_fq']
test[new_column] /= test[period+'_fq']
set_uid_period(train,test,periods,uids )<categorify>
for dataset in [train, validation]:
dataset['Title'] = dataset['Name'].str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
print(dataset['Title'].value_counts())
 print()
train=train.replace([np.inf,-np.inf],np.nan)
 test=test.replace([np.inf,-np.inf],np.nan )<count_values>
for df in [train, validation]:
print(df.shape)
 print()
print(df.isna().sum() )
tr_na_count=train.isnull().sum() /len(train)
 tr_drop_cols=[c for c in train.columns if tr_na_count[c]>0.70]
tr_big_cols=[c for c in train.columns if train[c].value_counts(normalize=True,dropna=False ).values[0]>0.85]
te_na_count=test.isnull().sum() /len(test)
 te_drop_cols=[c for c in test.columns if te_na_count[c]>0.70]
te_big_cols=[c for c in test.columns if test[c].value_counts(normalize=True,dropna=False ).values[0]>0.85]
drop_cols=list(set(tr_drop_cols+tr_big_cols+te_drop_cols+te_big_cols))
drop_cols.remove('isFraud')
 print(len(drop_cols))
response=train['isFraud']
train.drop(columns=drop_cols+['isFraud'],inplace=True)
 test.drop(columns=drop_cols,inplace=True )<drop_column>
for df in [train, validation]:
df.dropna(subset = ['Embarked'], inplace = True )
excess_col=['date','TransactionDT']
train.drop(columns=excess_col,inplace=True)
 test.drop(columns=excess_col,inplace=True )<count_missing_values>
def getTitle(series):
return series.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
print(getTitle(train[train.Age.isnull() ].Name ).value_counts())
 mr_mask = train['Title'] == 'Mr'
miss_mask = train['Title'] == 'Miss'
mrs_mask = train['Title'] == 'Mrs'
master_mask = train['Title'] == 'Master'
dr_mask = train['Title'] == 'Dr'
train.loc[mr_mask, 'Age'] = train.loc[mr_mask, 'Age'].fillna(train[train.Title == 'Mr'].Age.mean())
 train.loc[miss_mask, 'Age'] = train.loc[miss_mask, 'Age'].fillna(train[train.Title == 'Miss'].Age.mean())
 train.loc[mrs_mask, 'Age'] = train.loc[mrs_mask, 'Age'].fillna(train[train.Title == 'Mrs'].Age.mean())
 train.loc[master_mask, 'Age'] = train.loc[master_mask, 'Age'].fillna(train[train.Title == 'Master'].Age.mean())
 train.loc[dr_mask, 'Age'] = train.loc[dr_mask, 'Age'].fillna(train[train.Title == 'Dr'].Age.mean())
 print()
print(getTitle(train[train.Age.isnull() ].Name ).value_counts() )
na_count_post=train.isna().sum()
print(na_count_post[na_count_post>0]/train.shape[0] )<define_variables>
print(getTitle(validation[validation.Age.isnull() ].Name ).value_counts())
 mr_mask = validation['Title'] == 'Mr'
miss_mask = validation['Title'] == 'Miss'
mrs_mask = validation['Title'] == 'Mrs'
master_mask = validation['Title'] == 'Master'
ms_mask = validation['Title'] == 'Ms'
validation.loc[mr_mask, 'Age'] = validation.loc[mr_mask, 'Age'].fillna(validation[validation.Title == 'Mr'].Age.mean())
 validation.loc[miss_mask, 'Age'] = validation.loc[miss_mask, 'Age'].fillna(validation[validation.Title == 'Miss'].Age.mean())
 validation.loc[mrs_mask, 'Age'] = validation.loc[mrs_mask, 'Age'].fillna(validation[validation.Title == 'Mrs'].Age.mean())
 validation.loc[master_mask, 'Age'] = validation.loc[master_mask, 'Age'].fillna(validation[validation.Title == 'Master'].Age.mean())
 validation.loc[ms_mask, 'Age'] = validation.loc[ms_mask, 'Age'].fillna(validation[validation.Title == 'Miss'].Age.mean())
 print(getTitle(validation[validation.Age.isnull() ].Name ).value_counts() )
categorical_cols=categorical_cols<define_variables>
print(train.isna().sum())
 print(validation.isna().sum() )
train_ip=train.copy()
test_ip=test.copy()<data_type_conversions>
train.drop(columns=['PassengerId'], inplace = True)
 [df.drop(columns=['Ticket'], inplace = True)for df in [train, validation]]
for col in numerical_cols:
train_ip[col] = train_ip[col].fillna(value=train_ip[col].mean(skipna=True))
test_ip[col] = test_ip[col].fillna(value=test_ip[col].mean(skipna=True))<feature_engineering>
[train, validation] = [pd.get_dummies(data = df, columns = ['Pclass', 'Sex', 'Embarked'])for df in [train, validation]]
for col in categorical_cols:
train_ip[col] = train_ip[col].fillna(train_ip[col].mode() [0])
 test_ip[col] = test_ip[col].fillna(test_ip[col].mode() [0] )<count_missing_values>
for df in [train, validation]:
df['HasCabin'] = df['Cabin'].notna().astype(int)
 df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['IsAlone'] =(df['FamilySize'] > 1 ).astype(int )
na_count_post2=train_ip.isna().sum()
print(na_count_post[na_count_post2>0]/train_ip.shape[0] )<categorify>
[df.drop(columns=['Cabin', 'SibSp', 'Parch'], inplace = True)for df in [train, validation]]
def labelencoder(train,test,col):
cod=list(train[col].values)+list(test[col].values)
 le=LabelEncoder().fit(cod)
 train[col]=le.transform(train[col])
 test[col]=le.transform(test[col])
 for c in categorical_cols:
labelencoder(train_ip,test_ip,c )<init_hyperparams>
train['Title'] = train['Title'].replace('Mlle', 'Miss' ).replace('Ms', 'Miss' ).replace('Mme', 'Mrs' ).replace(['Dr', 'Major', 'Col', 'Rev', 'Lady', 'Jonkheer', 'Don', 'Sir', 'Dona', 'Capt', 'the Countess'], 'Special')
 validation['Title'] = validation['Title'].replace('Mlle', 'Miss' ).replace('Ms', 'Miss' ).replace('Mme', 'Mrs' ).replace(['Dr', 'Major', 'Col', 'Rev', 'Lady', 'Jonkheer', 'Don', 'Sir', 'Dona', 'Capt', 'the Countess'], 'Special')
 [df.drop(columns=['Name'], inplace = True)for df in [train, validation]]
[train, validation] = [pd.get_dummies(data = df, columns = ['Title'])for df in [train, validation]]
params = {'num_leaves': int(( 2**10)*0.72),
'min_child_weight': 0.17,
'feature_fraction': 0.72,
'bagging_fraction': 0.72,
'min_data_in_leaf': 179,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006,
"boosting_type": "gbdt",
"bagging_seed": 13,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3299927210061127,
'reg_lambda': 0.3885237330340494,
'random_state': 4,
}<prepare_x_and_y>
X = train[['Age', 'Fare', 'Pclass_2', 'Pclass_3', 'Sex_female', 'Embarked_C',
'Embarked_S', 'HasCabin', 'FamilySize', 'Title_Master', 'Title_Mr',
'Title_Mrs', 'Title_Special']]
y = train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)
 print(X_train.shape, X_test.shape )
%%time
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = train_ip.columns
splits = folds.split(train_ip, response)
 y_preds = np.zeros(test_ip.shape[0])
 y_oof = np.zeros(train_ip.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = train_ip[columns].iloc[train_index], train_ip[columns].iloc[valid_index]
y_train, y_valid = response.iloc[train_index], response.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=300)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(test_ip)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}" )<load_from_csv>
print(y.value_counts() )
sub = pd.read_csv(f'.. /input/ieee-fraud-detection/sample_submission.csv' )<save_to_csv>
y_default = pd.Series([0] * train['Survived'].shape[0], name = 'Survived')
 print(y_default.value_counts() )
sub['isFraud']= y_preds
sub.to_csv('submission.csv', index=False )<import_modules>
print(confusion_matrix(y, y_default))
print()
print(accuracy_score(y, y_default))
import numpy as np
import pandas as pd
import cudf
import cupy as cp
from cuml.svm import SVC
from sklearn.model_selection import KFold, train_test_split
from sklearn.metrics import roc_auc_score, log_loss 
<import_modules>
classifier = LinearSVC(dual=False)
 classifier.fit(X_train, y_train)
 y_pred = classifier.predict(X_test)
 print(confusion_matrix(y_test, y_pred))
print(accuracy_score(y_test, y_pred))
scores = cross_val_score(classifier, X_train, y_train, cv=10, scoring='accuracy')
 print(scores.mean() )
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import os 
<load_from_csv>
classifier = KNeighborsClassifier(n_neighbors = 2)
 classifier.fit(X_train, y_train)
 y_pred = classifier.predict(X_test)
 print(confusion_matrix(y_test, y_pred))
print(accuracy_score(y_test, y_pred))
scores = cross_val_score(classifier, X_train, y_train, cv=10, scoring='accuracy')
 print(scores.mean() )
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()<define_variables>
print("AdaBoostClassifier")
 ada_boost_classifier = AdaBoostClassifier()
ada_boost_classifier.fit(X_train, y_train)
 y_pred = ada_boost_classifier.predict(X_test)
 print(confusion_matrix(y_test, y_pred))
print(accuracy_score(y_test, y_pred))
scores = cross_val_score(ada_boost_classifier, X_train, y_train, cv=10, scoring='accuracy')
 print(scores.mean())
 print("BaggingClassifier")
 bagging_classifier = BaggingClassifier()
bagging_classifier.fit(X_train, y_train)
 y_pred = bagging_classifier.predict(X_test)
 print(confusion_matrix(y_test, y_pred))
print(accuracy_score(y_test, y_pred))
scores = cross_val_score(bagging_classifier, X_train, y_train, cv=10, scoring='accuracy')
 print(scores.mean())
 print("ExtraTreesClassifier")
 extra_trees_classifier = ExtraTreesClassifier(n_estimators=100)
 extra_trees_classifier.fit(X_train, y_train)
 y_pred = extra_trees_classifier.predict(X_test)
 print(confusion_matrix(y_test, y_pred))
print(accuracy_score(y_test, y_pred))
scores = cross_val_score(extra_trees_classifier, X_train, y_train, cv=10, scoring='accuracy')
 print(scores.mean())
 print("GradientBoostingClassifier")
 gradient_boosting_classifier = GradientBoostingClassifier()
gradient_boosting_classifier.fit(X_train, y_train)
 y_pred = gradient_boosting_classifier.predict(X_test)
 print(confusion_matrix(y_test, y_pred))
print(accuracy_score(y_test, y_pred))
scores = cross_val_score(gradient_boosting_classifier, X_train, y_train, cv=10, scoring='accuracy')
 print(scores.mean())
 print("RandomForestClassifier")
 random_forest_classifier = RandomForestClassifier(n_estimators=100)
 random_forest_classifier.fit(X_train, y_train)
 y_pred = random_forest_classifier.predict(X_test)
 print(confusion_matrix(y_test, y_pred))
print(accuracy_score(y_test, y_pred))
scores = cross_val_score(random_forest_classifier, X_train, y_train, cv=10, scoring='accuracy')
 print(scores.mean() )
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224<load_pretrained>
xgboost_classifier = XGBClassifier()
xgboost_classifier.fit(X_train, y_train)
 y_pred = xgboost_classifier.predict(X_test)
 print(confusion_matrix(y_test, y_pred))
print(accuracy_score(y_test, y_pred))
scores = cross_val_score(xgboost_classifier, X_train, y_train, cv=10, scoring='accuracy')
 print(scores.mean() )
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )<define_variables>
X_validation = validation[['Age', 'Fare', 'Pclass_2', 'Pclass_3', 'Sex_female', 'Embarked_C',
'Embarked_S', 'HasCabin', 'FamilySize', 'Title_Master', 'Title_Mr',
'Title_Mrs', 'Title_Special']]
y_valid = xgboost_classifier.predict(X_validation )
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]<load_pretrained>
validation_pId = validation.loc[:, 'PassengerId']
my_submission = pd.DataFrame(data={'PassengerId':validation_pId, 'Survived':y_valid})
 print(my_submission['Survived'].value_counts() )
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )<load_pretrained>
my_submission.to_csv('submission.csv', index = False )