Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<split>
import numpy as np
import pandas as pd
train = id_split(train)
 test = id_split(test )<drop_column>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv' )
usefull_features = [col for col in train.columns if col not in drop_features]
train = train[usefull_features]
usefull_features.remove('isFraud')
 test = test[usefull_features]<data_type_conversions>
women = train_data[train_data['Sex'] == 'female']['Survived']
rate_women = sum(women)/len(women)
 print('% of women who survived:', rate_women )
train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))<define_variables>
men = train_data[train_data.Sex == 'male']['Survived']
rate_men = sum(men)/len(men)
 print('% of men who survived:', rate_men )
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other',
'optonline.net': 'other', 'hotmail.co.uk': 'microsoft', 'comcast.net': 'other',
'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo', 'yahoo.es': 'yahoo',
'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol',
'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink', 'gmail.com': 'google',
'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other', 'web.de': 'other',
'cfl.rr.com': 'other', 'hotmail.com': 'microsoft', 'protonmail.com': 'other',
'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo', 'servicios-ta.com': 'other',
'netzero.net': 'other', 'suddenlink.net': 'other', 'roadrunner.com': 'other',
'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo',
'msn.com': 'microsoft', 'q.com': 'centurylink', 'prodigy.net.mx': 'att',
'frontier.com': 'yahoo', 'anonymous.com': 'other', 'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo', 'ymail.com': 'yahoo',
'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft',
'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other',
'cox.net': 'other', 'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']<feature_engineering>
train_data[['Sex', 'Survived']].groupby(['Sex'] ).mean()
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<feature_engineering>
train_data[['Pclass', 'Survived']].groupby(['Pclass'] ).mean()
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 def setTime(df):
df['TransactionDT'] = df['TransactionDT'].fillna(df['TransactionDT'].median())
 df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(df['DT'].dt.year-2017)*12 + df['DT'].dt.month
df['DT_W'] =(df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear
df['DT_D'] =(df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear
df['DT_hour'] = df['DT'].dt.hour
df['DT_day_week'] = df['DT'].dt.dayofweek
df['DT_day'] = df['DT'].dt.day
df['D9_not_na'] = np.where(df['D9'].isna() ,0,1)
 df['D8_not_same_day'] = np.where(df['D8']>=1,1,0)
 df['D8_D9_decimal_dist'] = df['D8'].fillna(0)-df['D8'].fillna(0 ).astype(int)
 df['D8_D9_decimal_dist'] =(( df['D8_D9_decimal_dist']-df['D9'])**2)**0.5
df['D8'] = df['D8'].fillna(-1 ).astype(int)
 return df
train=setTime(train)
 test=setTime(test )<data_type_conversions>
women_count = 0
women_survived_count = 0
for idx, row in train_data.iterrows() :
if row['Sex'] == 'female':
women_count += 1
if row['Survived'] == 1:
women_survived_count += 1
women_survived_count / women_count
def addNewFeatures(data):
data['uid'] = data['card1'].astype(str)+'_'+data['card2'].astype(str)
 data['uid2'] = data['uid'].astype(str)+'_'+data['card3'].astype(str)+'_'+data['card5'].astype(str)
 data['uid3'] = data['uid2'].astype(str)+'_'+data['addr1'].astype(str)+'_'+data['addr2'].astype(str)
 return data<feature_engineering>
predictions = []
for idx, row in test_data.iterrows() :
if row['Sex'] == 'female':
if row['Pclass'] <3:
predictions.append(1)
 elif row['Fare'] < 25:
predictions.append(1)
 else:
predictions.append(0)
 else:
if row['Age'] < 10:
predictions.append(1)
 else:
predictions.append(0)
train = addNewFeatures(train)
 test = addNewFeatures(test )<data_type_conversions>
test_data['Survived'] = predictions
i_cols = ['card2','card3','card5','uid','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train[[col, 'TransactionAmt']], test[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train[new_col_name] = train[col].map(temp_df)
 test[new_col_name] = test[col].map(temp_df )<categorify>
test_data[['PassengerId', 'Survived']].to_csv('submission.csv', index=False )