Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
%matplotlib inline
for df in [train_df, test_df]:
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(df['DT'].dt.year-2017)*12 + df['DT'].dt.month
df['DT_W'] =(df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear
df['DT_D'] =(df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear
df['DT_hour'] = df['DT'].dt.hour
df['DT_day_week'] = df['DT'].dt.dayofweek
df['DT_day'] = df['DT'].dt.day
df['D9'] = np.where(df['D9'].isna() ,0,1 )<feature_engineering>
warnings.filterwarnings('ignore' )
i_cols = ['card1']
for col in i_cols:
valid_card = pd.concat([train_df[[col]], test_df[[col]]])
 valid_card = valid_card[col].value_counts()
valid_card = valid_card[valid_card>2]
valid_card = list(valid_card.index)
 train_df[col] = np.where(train_df[col].isin(test_df[col]), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(train_df[col]), test_df[col], np.nan)
 train_df[col] = np.where(train_df[col].isin(valid_card), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(valid_card), test_df[col], np.nan )<data_type_conversions>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
i_cols = ['M1','M2','M3','M5','M6','M7','M8','M9']
for df in [train_df, test_df]:
df['M_sum'] = df[i_cols].sum(axis=1 ).astype(np.int8)
 df['M_na'] = df[i_cols].isna().sum(axis=1 ).astype(np.int8 )<data_type_conversions>
train_df.isna().sum()
train_df['uid'] = train_df['card1'].astype(str)+'_'+train_df['card2'].astype(str)
 test_df['uid'] = test_df['card1'].astype(str)+'_'+test_df['card2'].astype(str)
 train_df['uid2'] = train_df['uid'].astype(str)+'_'+train_df['card3'].astype(str)+'_'+train_df['card5'].astype(str)
 test_df['uid2'] = test_df['uid'].astype(str)+'_'+test_df['card3'].astype(str)+'_'+test_df['card5'].astype(str)
 train_df['uid3'] = train_df['uid2'].astype(str)+'_'+train_df['addr1'].astype(str)+'_'+train_df['addr2'].astype(str)
 test_df['uid3'] = test_df['uid2'].astype(str)+'_'+test_df['addr1'].astype(str)+'_'+test_df['addr2'].astype(str)
 train_df['TransactionAmt_check'] = np.where(train_df['TransactionAmt'].isin(test_df['TransactionAmt']), 1, 0)
 test_df['TransactionAmt_check'] = np.where(test_df['TransactionAmt'].isin(train_df['TransactionAmt']), 1, 0)
 i_cols = ['card1','card2','card3','card5','uid','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train_df[[col, 'TransactionAmt']], test_df[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df)
 test_df[new_col_name] = test_df[col].map(temp_df)
 train_df['TransactionAmt'] = np.log1p(train_df['TransactionAmt'])
 test_df['TransactionAmt'] = np.log1p(test_df['TransactionAmt'] )<feature_engineering>
test_df.isna().sum()
p = 'P_emaildomain'
r = 'R_emaildomain'
uknown = 'email_not_provided'
for df in [train_df, test_df]:
df[p] = df[p].fillna(uknown)
 df[r] = df[r].fillna(uknown)
 df['email_check'] = np.where(( df[p]==df[r])&(df[p]!=uknown),1,0)
 df[p+'_prefix'] = df[p].apply(lambda x: x.split('.')[0])
 df[r+'_prefix'] = df[r].apply(lambda x: x.split('.')[0] )<data_type_conversions>
train_df['source'] = 'train'
test_df['source'] = 'test'
for df in [train_identity, test_identity]:
df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['DeviceInfo_device'] = df['DeviceInfo'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
df['DeviceInfo_version'] = df['DeviceInfo'].apply(lambda x: ''.join([i for i in x if i.isnumeric() ]))
df['id_30'] = df['id_30'].fillna('unknown_device' ).str.lower()
df['id_30_device'] = df['id_30'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
df['id_30_version'] = df['id_30'].apply(lambda x: ''.join([i for i in x if i.isnumeric() ]))
df['id_31'] = df['id_31'].fillna('unknown_device' ).str.lower()
df['id_31_device'] = df['id_31'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))<merge>
dataset = pd.concat([train_df, test_df], ignore_index=True )
temp_df = train_df[['TransactionID']]
temp_df = temp_df.merge(train_identity, on=['TransactionID'], how='left')
 del temp_df['TransactionID']
train_df = pd.concat([train_df,temp_df], axis=1)
 temp_df = test_df[['TransactionID']]
temp_df = temp_df.merge(test_identity, on=['TransactionID'], how='left')
 del temp_df['TransactionID']
test_df = pd.concat([test_df,temp_df], axis=1 )<define_variables>
dataset.isnull().sum()
i_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','DeviceInfo_device','DeviceInfo_version',
'id_30','id_30_device','id_30_version',
'id_31_device',
'id_33',
'uid','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train_df[col+'_fq_enc'] = train_df[col].map(fq_encode)
 test_df[col+'_fq_enc'] = test_df[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train_df[col+'_total'] = train_df[col].map(fq_encode)
 test_df[col+'_total'] = test_df[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train_df[[col,period]], test_df[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train_df[new_column] =(train_df[col].astype(str)+ '_' + train_df[period].astype(str)).map(fq_encode)
 test_df[new_column] =(test_df[col].astype(str)+ '_' + test_df[period].astype(str)).map(fq_encode)
 train_df[new_column] /= train_df[period+'_total']
test_df[new_column] /= test_df[period+'_total']<categorify>
train_df.isnull().sum()
for col in ['ProductCD','M4']:
temp_dict = train_df.groupby([col])[TARGET].agg(['mean'] ).reset_index().rename(
 columns={'mean': col+'_target_mean'})
 temp_dict.index = temp_dict[col].values
temp_dict = temp_dict[col+'_target_mean'].to_dict()
train_df[col] = train_df[col].map(temp_dict)
 test_df[col] = test_df[col].map(temp_dict )<categorify>
train_df['Survived'].value_counts()
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )<define_variables>
train_df.Parch.value_counts()
rm_cols = [
'TransactionID','TransactionDT',
TARGET,
'uid','uid2','uid3',
'bank_type',
'DT','DT_M','DT_W','DT_D',
'DT_hour','DT_day_week','DT_day',
'DT_D_total','DT_W_total','DT_M_total',
'id_30','id_31','id_33',
]<init_hyperparams>
train_df.SibSp.value_counts() /891*100
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.85,
'subsample_freq':1,
'subsample':0.85,
'n_estimators':2**9,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
'reg_alpha':0.3,
'reg_lamdba':0.243
}<predict_on_test>
train_df.Pclass.value_counts() /train_df.shape[0]*100
if LOCAL_TEST:
lgb_params['learning_rate'] = 0.01
lgb_params['n_estimators'] = 20000
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params)
 print(metrics.roc_auc_score(test_predictions[TARGET], test_predictions['prediction']))
else:
lgb_params['learning_rate'] = 0.007
lgb_params['n_estimators'] = 1800
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params, NFOLDS=6 )<save_to_csv>
grid_pivot1 = train_df.pivot_table(columns='Survived', values='Age', aggfunc='mean' )
if not LOCAL_TEST:
test_predictions['isFraud'] = test_predictions['prediction']
test_predictions[['TransactionID','isFraud']].to_csv('submission.csv', index=False )<set_options>
warnings.filterwarnings('ignore' )<define_variables>
grid_pivot2 = train_df.pivot_table(index='Sex', columns='Survived', values='Age', aggfunc='mean')
 grid_pivot2
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )<predict_on_test>
grid_pivot3 = train_df.pivot_table(index='Pclass',columns='Survived', values='Age' )
def make_predictions(tr_df, tt_df, features_columns, target, lgb_params, NFOLDS=2):
folds = KFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 X,y = tr_df[features_columns], tr_df[target]
P,P_y = tt_df[features_columns], tt_df[target]
tt_df = tt_df[['TransactionID',target]]
predictions = np.zeros(len(tt_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, vl_y = X.iloc[val_idx,:], y[val_idx]
print(len(tr_x),len(vl_x))
tr_data = lgb.Dataset(tr_x, label=tr_y)
 if LOCAL_TEST:
vl_data = lgb.Dataset(P, label=P_y)
 else:
vl_data = lgb.Dataset(vl_x, label=vl_y)
 estimator = lgb.train(
 lgb_params,
tr_data,
valid_sets = [tr_data, vl_data],
verbose_eval = 200, 
)
 pp_p = estimator.predict(P)
 predictions += pp_p/NFOLDS
if LOCAL_TEST:
feature_imp = pd.DataFrame(sorted(zip(estimator.feature_importance() ,X.columns)) , columns=['Value','Feature'])
 print(feature_imp)
 del tr_x, tr_y, vl_x, vl_y, tr_data, vl_data
gc.collect()
tt_df['prediction'] = predictions
return tt_df 
<define_variables>
train_df.pivot_table(index=['Embarked','Pclass'], columns='Sex', values='Survived' )
SEED = 42
seed_everything(SEED)
 LOCAL_TEST = False
TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )<feature_engineering>
def extract_titles(name):
tit = re.findall('([A-Za-z]+)\.', name)
 return tit[0]
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 if LOCAL_TEST:
train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
test_df = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 train_df = train_df[train_df['DT_M']<(train_df['DT_M'].max() -1)].reset_index(drop=True)
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = train_identity[train_identity['TransactionID'].isin(
 test_df['TransactionID'])].reset_index(drop=True)
 train_identity = train_identity[train_identity['TransactionID'].isin(
 train_df['TransactionID'])].reset_index(drop=True)
 del train_df['DT_M'], test_df['DT_M']
else:
test_df = pd.read_pickle('.. /input/ieee-data-minification/test_transaction.pkl')
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = pd.read_pickle('.. /input/ieee-data-minification/test_identity.pkl')
 base_columns = list(train_df)+ list(train_identity)
 print('Shape control:', train_df.shape, test_df.shape )<feature_engineering>
dataset['Title'] = dataset['Name'].apply(lambda x: extract_titles(x))
for df in [train_df, test_df]:
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(df['DT'].dt.year-2017)*12 + df['DT'].dt.month
df['DT_W'] =(df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear
df['DT_D'] =(df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear
df['DT_hour'] = df['DT'].dt.hour
df['DT_day_week'] = df['DT'].dt.dayofweek
df['DT_day'] = df['DT'].dt.day
df['D9'] = np.where(df['D9'].isna() ,0,1 )<feature_engineering>
dataset[dataset['source'] == 'train'].isnull().sum()
i_cols = ['card1']
for col in i_cols:
valid_card = pd.concat([train_df[[col]], test_df[[col]]])
 valid_card = valid_card[col].value_counts()
valid_card = valid_card[valid_card>2]
valid_card = list(valid_card.index)
 train_df[col] = np.where(train_df[col].isin(test_df[col]), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(train_df[col]), test_df[col], np.nan)
 train_df[col] = np.where(train_df[col].isin(valid_card), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(valid_card), test_df[col], np.nan )<data_type_conversions>
dataset[dataset['source'] == 'test'].isnull().sum()
i_cols = ['M1','M2','M3','M5','M6','M7','M8','M9']
for df in [train_df, test_df]:
df['M_sum'] = df[i_cols].sum(axis=1 ).astype(np.int8)
 df['M_na'] = df[i_cols].isna().sum(axis=1 ).astype(np.int8 )<categorify>
dataset['Title'] = dataset['Title'].replace(['Don', 'Rev', 'Dr','Major', 'Lady', 'Sir','Col', 'Capt', 'Countess',
'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace(['Ms', 'Mlle'], 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme','Mrs' )
for col in ['ProductCD','M4']:
temp_dict = train_df.groupby([col])[TARGET].agg(['mean'] ).reset_index().rename(
 columns={'mean': col+'_target_mean'})
 temp_dict.index = temp_dict[col].values
temp_dict = temp_dict[col+'_target_mean'].to_dict()
train_df[col+'_target_mean'] = train_df[col].map(temp_dict)
 test_df[col+'_target_mean'] = test_df[col].map(temp_dict )<data_type_conversions>
dataset[dataset['source'] == 'train'].Title.value_counts()
train_df['uid'] = train_df['card1'].astype(str)+'_'+train_df['card2'].astype(str)
 test_df['uid'] = test_df['card1'].astype(str)+'_'+test_df['card2'].astype(str)
 train_df['uid2'] = train_df['uid'].astype(str)+'_'+train_df['card3'].astype(str)+'_'+train_df['card4'].astype(str)
 test_df['uid2'] = test_df['uid'].astype(str)+'_'+test_df['card3'].astype(str)+'_'+test_df['card4'].astype(str)
 train_df['uid3'] = train_df['uid2'].astype(str)+'_'+train_df['addr1'].astype(str)+'_'+train_df['addr2'].astype(str)
 test_df['uid3'] = test_df['uid2'].astype(str)+'_'+test_df['addr1'].astype(str)+'_'+test_df['addr2'].astype(str)
 train_df['TransactionAmt_check'] = np.where(train_df['TransactionAmt'].isin(test_df['TransactionAmt']), 1, 0)
 test_df['TransactionAmt_check'] = np.where(test_df['TransactionAmt'].isin(train_df['TransactionAmt']), 1, 0)
 i_cols = ['card1','card2','card3','card5','uid','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train_df[[col, 'TransactionAmt']], test_df[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df)
 test_df[new_col_name] = test_df[col].map(temp_df)
 train_df['TransactionAmt'] = np.log1p(train_df['TransactionAmt'])
 test_df['TransactionAmt'] = np.log1p(test_df['TransactionAmt'] )<feature_engineering>
p = 'P_emaildomain'
r = 'R_emaildomain'
uknown = 'email_not_provided'
for df in [train_df, test_df]:
df[p] = df[p].fillna(uknown)
 df[r] = df[r].fillna(uknown)
 df['email_check'] = np.where(( df[p]==df[r])&(df[p]!=uknown),1,0)
 df[p+'_prefix'] = df[p].apply(lambda x: x.split('.')[0])
 df[r+'_prefix'] = df[r].apply(lambda x: x.split('.')[0])
<data_type_conversions>
sex_dummy = pd.get_dummies(dataset['Sex'] )
for df in [train_identity, test_identity]:
df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['DeviceInfo_device'] = df['DeviceInfo'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
df['DeviceInfo_version'] = df['DeviceInfo'].apply(lambda x: ''.join([i for i in x if i.isnumeric() ]))
df['id_30'] = df['id_30'].fillna('unknown_device' ).str.lower()
df['id_30_device'] = df['id_30'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
df['id_30_version'] = df['id_30'].apply(lambda x: ''.join([i for i in x if i.isnumeric() ]))
df['id_31'] = df['id_31'].fillna('unknown_device' ).str.lower()
df['id_31_device'] = df['id_31'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))<merge>
dataset = dataset.join(sex_dummy )
temp_df = train_df[['TransactionID']]
temp_df = temp_df.merge(train_identity, on=['TransactionID'], how='left')
 del temp_df['TransactionID']
train_df = pd.concat([train_df,temp_df], axis=1)
 temp_df = test_df[['TransactionID']]
temp_df = temp_df.merge(test_identity, on=['TransactionID'], how='left')
 del temp_df['TransactionID']
test_df = pd.concat([test_df,temp_df], axis=1 )<define_variables>
grid_pivot = dataset[dataset['source'] == 'train'].pivot_table(index='Pclass',columns='Sex', values='Age', aggfunc='median' )
i_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','DeviceInfo_device','DeviceInfo_version',
'id_30','id_30_device','id_30_version',
'id_31_device',
'id_33',
'uid','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train_df[col+'_fq_enc'] = train_df[col].map(fq_encode)
 test_df[col+'_fq_enc'] = test_df[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train_df[col+'_total'] = train_df[col].map(fq_encode)
 test_df[col+'_total'] = test_df[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train_df[[col,period]], test_df[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train_df[new_column] =(train_df[col].astype(str)+ '_' + train_df[period].astype(str)).map(fq_encode)
 test_df[new_column] =(test_df[col].astype(str)+ '_' + test_df[period].astype(str)).map(fq_encode)
 train_df[new_column] /= train_df[period+'_total']
test_df[new_column] /= test_df[period+'_total']<categorify>
guess_ages = np.zeros(( 2,3))
guess_ages
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )<init_hyperparams>
dataset['AgeBand'] = pd.cut(dataset['Age'], 5 )
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':0.7,
'n_estimators':800,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
}<predict_on_test>
dataset.AgeBand.value_counts()
if LOCAL_TEST:
lgb_params['learning_rate'] = 0.01
lgb_params['n_estimators'] = 20000
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params)
 print(metrics.roc_auc_score(test_predictions[TARGET], test_predictions['prediction']))
else:
lgb_params['learning_rate'] = 0.01
lgb_params['n_estimators'] = 800
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params, NFOLDS=2 )<save_to_csv>
dataset['AgeBand'] = dataset['AgeBand'].astype(str )
if not LOCAL_TEST:
test_predictions['isFraud'] = test_predictions['prediction']
test_predictions[['TransactionID','isFraud']].to_csv('submission.csv', index=False )<set_options>
warnings.filterwarnings('ignore' )<define_variables>
def ageclass(x):
if x <= 16:
return 0
elif x > 16 and x <= 32:
return 1
elif x > 32 and x <= 48:
return 2
elif x > 48 and x <= 64:
return 3
else:
x > 64
return 4
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
<define_variables>
dataset['AgeClass'] = dataset['Age'].apply(ageclass )
SEED = 42
seed_everything(SEED)
 LOCAL_TEST = False
TARGET = 'isFraud'<load_pretrained>
dataset['Family_Size'] = dataset['Parch'] + dataset['SibSp'] + 1
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 if LOCAL_TEST:
test_df = train_df.iloc[-100000:,].reset_index(drop=True)
 train_df = train_df.iloc[:400000,].reset_index(drop=True)
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = train_identity[train_identity['TransactionID'].isin(test_df['TransactionID'])].reset_index(drop=True)
 train_identity = train_identity[train_identity['TransactionID'].isin(train_df['TransactionID'])].reset_index(drop=True)
 else:
test_df = pd.read_pickle('.. /input/ieee-data-minification/test_transaction.pkl')
 test_identity = pd.read_pickle('.. /input/ieee-data-minification/test_identity.pkl' )<feature_engineering>
dataset.pivot_table(index='Family_Size', values='Survived' ).sort_values(by='Survived',ascending=False )
valid_card = train_df['card1'].value_counts()
valid_card = valid_card[valid_card>10]
valid_card = list(valid_card.index)
 train_df['card1'] = np.where(train_df['card1'].isin(valid_card), train_df['card1'], np.nan)
 test_df['card1'] = np.where(test_df['card1'].isin(valid_card), test_df['card1'], np.nan )<count_values>
def isalone(x):
if x == 1:
return 1
else:
return 0
i_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8','D9',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain'
]
for col in i_cols:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train_df[col+'_fq_enc'] = train_df[col].map(fq_encode)
 test_df[col+'_fq_enc'] = test_df[col].map(fq_encode )<categorify>
dataset['IsAlone'] = dataset['Family_Size'].apply(isalone )
for col in ['ProductCD','M4']:
temp_dict = train_df.groupby([col])[TARGET].agg(['mean'] ).reset_index().rename(
 columns={'mean': col+'_target_mean'})
 temp_dict.index = temp_dict[col].values
temp_dict = temp_dict[col+'_target_mean'].to_dict()
train_df[col+'_target_mean'] = train_df[col].map(temp_dict)
 test_df[col+'_target_mean'] = test_df[col].map(temp_dict )<categorify>
dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace=True )
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )<init_hyperparams>
dataset.isnull().sum()
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':1,
'n_estimators':800,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
}<predict_on_test>
embarked_dummy = pd.get_dummies(dataset['Embarked'] )
def make_predictions(tr_df, tt_df, features_columns, target, lgb_params, NFOLDS=2):
folds = KFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 X,y = tr_df[features_columns], tr_df[target]
P,P_y = tt_df[features_columns], tt_df[target]
tt_df = tt_df[['TransactionID',target]]
predictions = np.zeros(len(tt_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, vl_y = X.iloc[val_idx,:], y[val_idx]
print(len(tr_x),len(vl_x))
tr_data = lgb.Dataset(tr_x, label=tr_y)
 if LOCAL_TEST:
vl_data = lgb.Dataset(P, label=P_y)
 else:
vl_data = lgb.Dataset(vl_x, label=vl_y)
 estimator = lgb.train(
 lgb_params,
tr_data,
valid_sets = [tr_data, vl_data],
verbose_eval = 200, 
)
 pp_p = estimator.predict(P)
 predictions += pp_p/NFOLDS
if LOCAL_TEST:
feature_imp = pd.DataFrame(sorted(zip(estimator.feature_importance() ,X.columns)) , columns=['Value','Feature'])
 print(feature_imp)
 del tr_x, tr_y, vl_x, vl_y, tr_data, vl_data
gc.collect()
tt_df['prediction'] = predictions
return tt_df 
<compute_train_metric>
dataset = dataset.join(embarked_dummy )
if LOCAL_TEST:
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params)
 print(metrics.roc_auc_score(test_predictions[TARGET], test_predictions['prediction']))
else:
lgb_params['learning_rate'] = 0.005
lgb_params['n_estimators'] = 2000
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params, NFOLDS=10 )<save_to_csv>
dataset['Fare'].fillna(dataset['Fare'].median() , inplace=True )
if not LOCAL_TEST:
test_predictions['isFraud'] = test_predictions['prediction']
test_predictions[['TransactionID','isFraud']].to_csv('submission.csv', index=False )<set_options>
le = LabelEncoder()
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore' )<load_from_csv>
dataset['FareBand'] = pd.qcut(dataset['Fare'], 4 )
%%time
folder_path = '.. /input/'
print('Loading data...')
 train_identity = pd.read_csv(f'{folder_path}train_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_identity!')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_transaction!')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_identity!')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_transaction!')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 print('\tSuccessfully loaded sample_submission!')
 print('Data was successfully loaded!
' )<feature_engineering>
dataset['FareClass'] = le.fit_transform(dataset['FareBand'] )
def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe<split>
train_identity = id_split(train_identity)
 test_identity = id_split(test_identity )<merge>
dataset['Title'] = le.fit_transform(dataset['Title'] )
print('Merging data...')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print('Data was successfully merged!
')
 del train_identity, train_transaction, test_identity, test_transaction
print(f'Train dataset has {train.shape[0]} rows and {train.shape[1]} columns.')
 print(f'Test dataset has {test.shape[0]} rows and {test.shape[1]} columns.')
 gc.collect()<define_variables>
dataset['Age*Class'] = dataset['AgeClass'] * dataset['Pclass']
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'device_name', 'device_version', 'OS_id_30', 'version_id_30',
'browser_id_31', 'version_id_31', 'screen_width', 'screen_height', 'had_id']<drop_column>
drop_these = 'Age Cabin Embarked Fare Name Parch Ticket Sex SibSp AgeBand Family_Size FareBand'.split(' ' )
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('isFraud')
 cols_to_drop.remove('TransactionDT' )<drop_column>
dataset = dataset.drop(drop_these, axis=1 )
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )<categorify>
train_cleaned = dataset[dataset['source'] == 'train']
test_cleaned = dataset[dataset['source'] == 'test']
columns_a = ['TransactionAmt', 'id_02', 'D15']
columns_b = ['card1', 'card4', 'addr1']
for col_a in columns_a:
for col_b in columns_b:
for df in [train, test]:
df[f'{col_a}_to_mean_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('mean')
 df[f'{col_a}_to_std_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('std' )<feature_engineering>
train_cleaned['Survived'] = train_cleaned['Survived'].astype(int )
train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))<feature_engineering>
def predict_model(dtrain, dtest, predictor, outcome, model):
model.fit(dtrain[predictor], dtrain[outcome])
 dtrain_pred = model.predict(dtest[predictor])
 score = model.score(dtrain[predictor], dtrain[outcome])*100
return score, dtrain_pred
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<data_type_conversions>
predictors_var = ['Pclass','Title', 'female','male', 'AgeClass', 'IsAlone', 'C', 'Q', 'S', 'FareClass', 'Age*Class']
outcome_var = 'Survived'
traindf = train_cleaned
testdf = test_cleaned
%%time
for col in train.columns:
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))<drop_column>
logreg = LogisticRegression()
%%time
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<prepare_x_and_y>
predict_model(traindf, testdf, predictors_var, outcome_var, logreg )
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT'], axis=1)
 del train, test
gc.collect()<import_modules>
coef1.sort_values(ascending=False )
from sklearn.model_selection import KFold
import lightgbm as lgb<init_hyperparams>
svc = SVC()
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47,
}<split>
predict_model(traindf, testdf, predictors_var, outcome_var, svc )
%%time
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=500)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )<save_to_csv>
knn = KNeighborsClassifier(n_neighbors=3 )
sub['isFraud'] = y_preds
sub.to_csv("submission.csv", index=False )<save_to_csv>
predict_model(traindf, testdf, predictors_var, outcome_var, knn )
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('50 TOP feature importance over {} folds average'.format(folds.n_splits)) ;<define_variables>
gaussian = GaussianNB()
start_time = time.time()
SUBMIT_MODE = True 
<compute_test_metric>
predict_model(traindf, testdf, predictors_var, outcome_var, gaussian )
def rmse(predicted, actual):
return np.sqrt(((predicted - actual)** 2 ).mean())
 def split_cat(text):
try:
return text.split("/")
 except:
return("No Label", "No Label", "No Label" )<categorify>
decision_tree = DecisionTreeClassifier()
class TargetEncoder:
def __repr__(self):
return 'TargetEncoder'
def __init__(self, cols, smoothing=1, min_samples_leaf=1, noise_level=0, keep_original=False):
self.cols = cols
self.smoothing = smoothing
self.min_samples_leaf = min_samples_leaf
self.noise_level = noise_level
self.keep_original = keep_original
@staticmethod
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def encode(self, train, test, target):
for col in self.cols:
if self.keep_original:
train[col + '_te'], test[col + '_te'] = self.encode_column(train[col], test[col], target)
 else:
train[col], test[col] = self.encode_column(train[col], test[col], target)
 return train, test
def encode_column(self, trn_series, tst_series, target):
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - self.min_samples_leaf)/ self.smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(['mean', 'count'], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return self.add_noise(ft_trn_series, self.noise_level), self.add_noise(ft_tst_series, self.noise_level )<categorify>
predict_model(traindf, testdf, predictors_var, outcome_var, decision_tree )
def to_number(x):
try:
if not x.isdigit() :
return 0
x = int(x)
 if x > 100:
return 100
else:
return x
except:
return 0
def sum_numbers(desc):
if not isinstance(desc, str):
return 0
try:
return sum([to_number(s)for s in desc.split() ])
 except:
return 0<string_transform>
random_forest = RandomForestClassifier(n_estimators=100 )
stopwords = {x: 1 for x in stopwords.words('english')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 non_alphanumpunct = re.compile(u'[^A-Za-z0-9\.?!,; \(\)\[\]'"\$]+')
 RE_PUNCTUATION = '|'.join([re.escape(x)for x in string.punctuation])
 def normalize_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] \
if len(x)> 1 and x not in stopwords])
 def clean_name(x):
if len(x):
x = non_alphanums.sub(' ', x ).split()
if len(x):
return x[0].lower()
return ''
print('[{}] Finished defining stuff'.format(time.time() - start_time))<load_from_csv>
predict_model(traindf, testdf, predictors_var, outcome_var, random_forest )
train = pd.read_table('.. /input/train.tsv', engine='c',
dtype={'item_condition_id': 'category',
'shipping': 'category',
},
converters={'category_name': split_cat})
 test = pd.read_table('.. /input/test.tsv', engine='c',
dtype={'item_condition_id': 'category',
'shipping': 'category',
},
converters={'category_name': split_cat})
 print('[{}] Finished load data'.format(time.time() - start_time))<prepare_x_and_y>
predict_model(traindf, testdf, predictors_var, outcome_var, random_forest)[1]
train['is_train'] = 1
test['is_train'] = 0
print('[{}] Compiled train / test'.format(time.time() - start_time))
print('Train shape: ', train.shape)
 print('Test shape: ', test.shape)
 train = train[train.price != 0].reset_index(drop=True)
 print('[{}] Removed nonzero price'.format(time.time() - start_time))
print('Train shape: ', train.shape)
 print('Test shape: ', test.shape)
 y = np.log1p(train['price'])
 nrow_train = train.shape[0]
merge = pd.concat([train, test])
 submission = test[['test_id']]
print('[{}] Compiled merge'.format(time.time() - start_time))
print('Merge shape: ', merge.shape )<drop_column>
results = predict_model(traindf, testdf, predictors_var, outcome_var, random_forest)[1]
del train
del test
merge.drop(['train_id', 'test_id', 'price'], axis=1, inplace=True)
 gc.collect()
print('[{}] Garbage collection'.format(time.time() - start_time))<data_type_conversions>
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": results
} )
<categorify><EOS> 
submission.to_csv('submission_updated.csv', index=False )