Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
assert sys.version_info >=(3, 5)
 assert sklearn.__version__ >= "0.20"
%matplotlib inline
warnings.filterwarnings(action="ignore" )
variables = list(categoricalCols)
 na_vals = np.sum(raw_train_data.loc[:,variables]=='missing')/raw_train_data.shape[0]
goodCategoricalVars = []
for i_var in variables:
if na_vals[i_var] < 1:
goodCategoricalVars.append(i_var )<prepare_x_and_y>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
featuresToUse = goodNumericVars+goodCategoricalVars
train_data = raw_train_data.loc[:,featuresToUse]
target_data = raw_train_data['isFraud']<categorify>
passengerID = test['PassengerId'].copy()
train_data['TransactionAmt_to_mean_card1'] = train_data['TransactionAmt'] / train_data.groupby(['card1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_card4'] = train_data['TransactionAmt'] / train_data.groupby(['card4'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_card5'] = train_data['TransactionAmt'] / train_data.groupby(['card5'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_addr1'] = train_data['TransactionAmt'] / train_data.groupby(['addr1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_id31'] = train_data['TransactionAmt'] / train_data.groupby(['id_31'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_devicename'] = train_data['TransactionAmt'] / train_data.groupby(['device_name'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C1'] = train_data['TransactionAmt'] / train_data.groupby(['C1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C3'] = train_data['TransactionAmt'] / train_data.groupby(['C3'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C5'] = train_data['TransactionAmt'] / train_data.groupby(['C5'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C13'] = train_data['TransactionAmt'] / train_data.groupby(['C13'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_std_card1'] = train_data['TransactionAmt'] / train_data.groupby(['card1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_card4'] = train_data['TransactionAmt'] / train_data.groupby(['card4'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_card5'] = train_data['TransactionAmt'] / train_data.groupby(['card5'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_addr1'] = train_data['TransactionAmt'] / train_data.groupby(['addr1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_id31'] = train_data['TransactionAmt'] / train_data.groupby(['id_31'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_devicename'] = train_data['TransactionAmt'] / train_data.groupby(['device_name'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C1'] = train_data['TransactionAmt'] / train_data.groupby(['C1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C3'] = train_data['TransactionAmt'] / train_data.groupby(['C3'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C5'] = train_data['TransactionAmt'] / train_data.groupby(['C5'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_decimal'] =(( train_data['TransactionAmt'] - train_data['TransactionAmt'].astype(int)) * 1000 ).astype(int )<data_type_conversions>
train["split"] = "train"
test["split"] = "test"
data = pd.concat([train , test], ignore_index=True)
 data.set_index("PassengerId")
 data["Survived"].fillna(0.0,inplace=True)
 print(train.shape, test.shape, data.shape )
train_data['card1_card2'] = train_data['card1'].astype(str)+ '_' + train_data['card2'].astype(str)
 train_data['addr1_dist1'] = train_data['addr1'].astype(str)+ '_' + train_data['dist1'].astype(str)
 train_data['card1_addr1'] = train_data['card1'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card1_addr2'] = train_data['card1'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['card2_addr1'] = train_data['card2'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card2_addr2'] = train_data['card2'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['card4_addr1'] = train_data['card4'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card4_addr2'] = train_data['card4'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['DeviceInfo_P_emaildomain'] = train_data['DeviceInfo'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['P_emaildomain_addr1'] = train_data['P_emaildomain'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['id01_addr1'] = train_data['id_01'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card1_P_emaildomain'] = train_data['card1'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['card5_P_emaildomain'] = train_data['card5'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['card2_P_emaildomain'] = train_data['card2'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['card4_P_emaildomain'] = train_data['card4'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['id_01_P_emaildomain'] = train_data['id_01'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str )<define_variables>
data.apply(lambda x: len(x.unique()))
scale_pos_weight = np.sqrt(len(target_data.loc[target_data==0])/len(target_data.loc[target_data==1]))
del raw_train_data<load_from_csv>
data['FamilySize'] = data['SibSp'] + data['Parch'] + 1
data['IsAlone'] = 0
data.loc[data['FamilySize'] == 1, 'IsAlone'] = 1
test_identity_data = pd.read_csv('.. /input/test_identity.csv')
 test_transaction_data = pd.read_csv('.. /input/test_transaction.csv' )<count_missing_values>
data.groupby('Fare')['Fare'].unique().value_counts()
na_columns = test_transaction_data.isna().sum()
print(na_columns[na_columns==0])
 print(na_columns[na_columns>0]/test_transaction_data.shape[0] )<feature_engineering>
data['Fare'].fillna(data['Fare'].median() , inplace=True)
 data.loc[data['Fare'] <= 7.91, 'Fare'] = 0
data.loc[(data['Fare'] > 7.91)&(data['Fare'] <= 14.454), 'Fare'] = 1
data.loc[(data['Fare'] > 14.454)&(data['Fare'] <= 31), 'Fare'] = 2
data.loc[ data['Fare'] > 31, 'Fare'] = 3
data['Fare'] = data['Fare'].astype(int )
test_transaction_data['Transaction_dow'] = np.floor(( test_transaction_data['TransactionDT'] /(3600 * 24)- 1)% 7)
 test_transaction_data['Transaction_hour'] = np.floor(test_transaction_data['TransactionDT'] / 3600)% 24
transaction_data_columns = test_transaction_data.columns
numericCols = test_transaction_data._get_numeric_data().columns
categoricalCols = list(set(transaction_data_columns)- set(numericCols))
test_transaction_data[categoricalCols] = test_transaction_data[categoricalCols].replace({ np.nan:'missing'})
 test_transaction_data[numericCols] = test_transaction_data[numericCols].replace({ np.nan:-1} )<feature_engineering>
data['Has_Cabin'] = data["Cabin"].apply(lambda x: 0 if type(x)== float else 1 )
identity_data_columns = test_identity_data.columns
numericCols = test_identity_data._get_numeric_data().columns
categoricalCols = list(set(identity_data_columns)- set(numericCols))
test_identity_data[categoricalCols] = test_identity_data[categoricalCols].replace({ np.nan:'missing'})
 test_identity_data[numericCols] = test_identity_data[numericCols].replace({ np.nan:-1})
 test_identity_data['id_31'] = test_identity_data['id_31'].str.replace('\d+', '')
 test_identity_data.loc[test_identity_data['id_30'].str.contains('Mac', na=False), 'id_30'] = 'mac'
test_identity_data.loc[test_identity_data['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Android', na=False), 'id_30'] = 'android'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Linux', na=False), 'id_30'] = 'Linux'
test_identity_data['device_name'] = test_identity_data['DeviceInfo'].str.split('/', expand=True)[0]
test_identity_data.loc[test_identity_data['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
test_identity_data.loc[test_identity_data['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
test_identity_data.loc[test_identity_data['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
test_identity_data.loc[test_identity_data['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
test_identity_data.loc[test_identity_data['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
test_identity_data.loc[test_identity_data['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
test_identity_data.loc[test_identity_data['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
test_identity_data.loc[test_identity_data.device_name.isin(test_identity_data.device_name.value_counts() [test_identity_data.device_name.value_counts() < 200].index), 'device_name'] = "Others"<merge>
avg = data['Age'].mean()
std = data['Age'].std()
null_count = data['Age'].isnull().sum()
random_list = np.random.randint(avg - std, avg + std, size = null_count)
 data['Age'][np.isnan(data['Age'])] = random_list
data['Age'] = data['Age'].astype(int )
raw_test_data = pd.merge(test_transaction_data, test_identity_data, on='TransactionID', how='left')
 transactionID = raw_test_data.loc[:,'TransactionID']
del test_identity_data,test_transaction_data<feature_engineering>
data.loc[ data['Age'] <= 16, 'Age'] = 0
data.loc[(data['Age'] > 16)&(data['Age'] <= 32), 'Age'] = 1
data.loc[(data['Age'] > 32)&(data['Age'] <= 48), 'Age'] = 2
data.loc[(data['Age'] > 48)&(data['Age'] <= 64), 'Age'] = 3
data.loc[data['Age'] > 64, 'Age'] = 4
raw_test_data_columns = raw_test_data.columns
numericCols = raw_test_data._get_numeric_data().columns
categoricalCols = list(set(raw_test_data_columns)- set(numericCols))
print('The categorical columns in training data are: ',categoricalCols)
 raw_test_data[categoricalCols] = raw_test_data[categoricalCols].replace({ np.nan:'missing'})
 raw_test_data[numericCols] = raw_test_data[numericCols].replace({ np.nan:-1})
 raw_test_data = reduce_memory_usage(raw_test_data, deep=True, verbose=True )<categorify>
data['Sex'] = data['Sex'].map({'female': 0, 'male': 1} ).astype(int )
test_data = raw_test_data.loc[:,featuresToUse]
test_data['TransactionAmt_to_mean_card1'] = test_data['TransactionAmt'] / test_data.groupby(['card1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_card4'] = test_data['TransactionAmt'] / test_data.groupby(['card4'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_card5'] = test_data['TransactionAmt'] / test_data.groupby(['card5'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_addr1'] = test_data['TransactionAmt'] / test_data.groupby(['addr1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_id31'] = test_data['TransactionAmt'] / test_data.groupby(['id_31'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_devicename'] = test_data['TransactionAmt'] / test_data.groupby(['device_name'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C1'] = test_data['TransactionAmt'] / test_data.groupby(['C1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C3'] = test_data['TransactionAmt'] / test_data.groupby(['C3'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C5'] = test_data['TransactionAmt'] / test_data.groupby(['C5'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C13'] = test_data['TransactionAmt'] / test_data.groupby(['C13'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_std_card1'] = test_data['TransactionAmt'] / test_data.groupby(['card1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_card4'] = test_data['TransactionAmt'] / test_data.groupby(['card4'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_card5'] = test_data['TransactionAmt'] / test_data.groupby(['card5'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_addr1'] = test_data['TransactionAmt'] / test_data.groupby(['addr1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_id31'] = test_data['TransactionAmt'] / test_data.groupby(['id_31'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_devicename'] = test_data['TransactionAmt'] / test_data.groupby(['device_name'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C1'] = test_data['TransactionAmt'] / test_data.groupby(['C1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C3'] = test_data['TransactionAmt'] / test_data.groupby(['C3'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C5'] = test_data['TransactionAmt'] / test_data.groupby(['C5'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_decimal'] =(( test_data['TransactionAmt'] - test_data['TransactionAmt'].astype(int)) * 1000 ).astype(int )<data_type_conversions>
data['Embarked'].fillna('S', inplace=True)
 data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )
test_data['card1_card2'] = test_data['card1'].astype(str)+ '_' + test_data['card2'].astype(str)
 test_data['addr1_dist1'] = test_data['addr1'].astype(str)+ '_' + test_data['dist1'].astype(str)
 test_data['card1_addr1'] = test_data['card1'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card1_addr2'] = test_data['card1'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['card2_addr1'] = test_data['card2'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card2_addr2'] = test_data['card2'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['card4_addr1'] = test_data['card4'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card4_addr2'] = test_data['card4'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['DeviceInfo_P_emaildomain'] = test_data['DeviceInfo'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['P_emaildomain_addr1'] = test_data['P_emaildomain'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['id01_addr1'] = test_data['id_01'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card1_P_emaildomain'] = test_data['card1'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['card5_P_emaildomain'] = test_data['card5'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['card2_P_emaildomain'] = test_data['card2'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['card4_P_emaildomain'] = test_data['card4'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['id_01_P_emaildomain'] = test_data['id_01'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str )<choose_model_class>
data['Title'] = data['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
data['Title'] = data['Title'].replace(['Miss', 'Mrs','Ms', 'Mlle', 'Lady', 'Mme', 'the Countess', 'Dona'], 'Miss/Mrs/Ms')
 data['Title'] = data['Title'].replace(['Dr', 'Col', 'Major', 'Jonkheer', 'Capt', 'Sir', 'Don'], 'Dr/Military/Noble')
 data['Title'] = data['Title'].fillna('Mr' ).map({'Mr': 0, 'Mrs': 1, 'Miss/Mrs/Ms': 2, 'Dr/Military/Noble': 3, 'Master': 4, 'Rev': 5} ).astype(int )
cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True )<compute_train_metric>
data['Title'].value_counts()
def compute_roc_auc(clf,index):
y_predict = clf.predict_proba(train_data.iloc[index])[:,1]
fpr, tpr, thresholds = roc_curve(target_data.iloc[index], y_predict)
 auc_score = auc(fpr, tpr)
 return fpr, tpr, auc_score<init_hyperparams>
drop_columns = ["PassengerId", "Ticket","Cabin", "Name", "SibSp"]
data = data.drop(drop_columns, axis=1 )
params = {'bagging_fraction': 0.7982116702024386,
'feature_fraction': 0.1785051643813966,
'max_depth': int(49.17611603427576),
'min_child_weight': 3.2852905549011155,
'min_data_in_leaf': int(31.03480802715621),
'n_estimators': 5000,
'num_leaves': int(52.851307790411965),
'reg_alpha': 0.45963319421692145,
'reg_lambda': 0.6591286807489907,
'metric':'auc',
'boosting_type': 'gbdt',
'colsample_bytree':.8,
'subsample':.9,
'min_split_gain':.01,
'max_bin':127,
'bagging_freq':5,
'learning_rate':0.01,
'early_stopping_rounds':100
}<split>
data.isnull().sum()
fprs_lgb, tprs_lgb, scores_lgb = [], [], []
feature_importances = pd.DataFrame()
feature_importances['feature'] = train_data.columns
predictions = np.zeros(len(test_data))
for(train, test), i in zip(cv.split(train_data, target_data), range(5)) :
lgb_best = LGBMClassifier(boosting = params['boosting_type'],n_estimators = params['n_estimators'],
learning_rate = params['learning_rate'],num_leaves = params['num_leaves'],
colsample_bytree = params['colsample_bytree'],subsample = params['subsample'],
max_depth = params['max_depth'],reg_alpha = params['reg_alpha'],
reg_lambda = params['reg_lambda'],min_split_gain = params['min_split_gain'],
min_child_weight = params['min_child_weight'],max_bin = params['max_bin'],
bagging_freq = params['bagging_freq'],feature_fraction = params['feature_fraction'],
bagging_fraction = params['bagging_fraction'],min_data_in_leaf = params['min_data_in_leaf'],
early_stopping_rounds = params['early_stopping_rounds'])
 lgb_best.fit(train_data.iloc[train,:], target_data.iloc[train],
eval_set = [(train_data.iloc[train,:], target_data.iloc[train]), 
(train_data.iloc[test,:], target_data.iloc[test])],eval_metric='auc',verbose = 200)
 feature_importances['fold_{}'.format(i + 1)] = lgb_best.feature_importances_
_, _, auc_score_train = compute_roc_auc(lgb_best,train)
 fpr, tpr, auc_score = compute_roc_auc(lgb_best,test)
 scores_lgb.append(( auc_score_train, auc_score))
fprs_lgb.append(fpr)
 tprs_lgb.append(tpr)
 predictions += lgb_best.predict_proba(test_data, num_iteration=lgb_best.best_iteration_)[:,1]<feature_engineering>
train = data[data['split'] == 'train'].copy()
test = data[data['split'] == 'test'].copy()
train_labels = train["Survived"].copy()
train.drop(['split','Survived'], axis=1, inplace=True)
 test.drop(['split','Survived'], axis=1, inplace=True )
predictions /= cv.n_splits<compute_test_metric>
train, train_val, train_labels, train_val_labels = train_test_split(train, train_labels, test_size=0.2, random_state=42 )
print('Mean AUC:', np.mean(scores_lgb,axis = 0))<drop_column>
num_pipeline = Pipeline([ 
("imputer", SimpleImputer(strategy="median")) , 
("MinMaxScaler", MinMaxScaler())
 ] )
del train_data,target_data<save_to_csv>
train_prepared = num_pipeline.fit_transform(train)
 train_val = num_pipeline.fit_transform(train_val )
data = {'TransactionID':transactionID,'isFraud':predictions}
submissionDF = pd.DataFrame(data)
 submissionDF.to_csv('sample_submission.csv',index=False )<load_from_csv>
model_lgre = LogisticRegression(random_state=0)
 param_grid = {'C': [0.014,0.012], 'multi_class': ['multinomial'],
'penalty': ['l1'],'solver': ['saga'], 'tol': [0.1] }
GridCV_LR = GridSearchCV(model_lgre, param_grid, verbose=1, cv=5)
 GridCV_LR.fit(train_prepared,train_labels)
 score_grid_LR = GridCV_LR.best_score_
model_lgre = GridCV_LR.best_estimator_
print(score_grid_LR)
 predict_score_lg_clf = GridCV_LR.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_lg_clf )
sub1 = pd.read_csv('.. /input/ensemble/ensemble/All_Blends_9430.csv')
 sub2 = pd.read_csv('.. /input/ensemble/ensemble/stack_median_9427.csv')
 sub3 = pd.read_csv('.. /input/ensemble/ensemble/submission_IEEE_9417.csv')
 sub4 = pd.read_csv('.. /input/lgb-single-model-lb-0-9419/ieee_cis_fraud_detection_v2.csv')
 sub5 = pd.read_csv('.. /input/lgb-starter-r/sub.csv')
 sub6 = pd.read_csv('.. /input/lgbm-baseline-small-fe-no-blend/lgb_sub.csv')
 temp=pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 temp['isFraud'] = 0.22 * sub3['isFraud'] + 0.23 * sub4['isFraud'] + 0.27 * sub5['isFraud'] + 0.28 * sub6['isFraud']
temp.to_csv('submission.csv', index=False )<import_modules>
rand_forest_clf = RandomForestClassifier(random_state=42)
 param_grid = {'max_depth': [15], 'min_samples_split': [5],'n_estimators' : [100] }
GridCV_rd_clf = GridSearchCV(rand_forest_clf, param_grid, verbose=1, cv=5)
 GridCV_rd_clf.fit(train_prepared, train_labels)
 score_grid_rd = GridCV_rd_clf.best_score_
rand_forest_clf = GridCV_rd_clf.best_estimator_
print(score_grid_rd)
 predict_score_rd_clf = GridCV_rd_clf.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_rd_clf )
import os
import numpy as np
import pandas as pd<define_variables>
tree_clf = DecisionTreeClassifier()
sub_path = ".. /input/ieeesubmissions4"
all_files = os.listdir(sub_path)
 all_files<save_to_csv>
params_grid_ada = {}
ada_model = AdaBoostClassifier(tree_clf,n_estimators=3000,
algorithm="SAMME.R", learning_rate=0.05, random_state=42)
 GridCV_ada = GridSearchCV(ada_model, params_grid_ada, verbose=1, cv=5)
 GridCV_ada.fit(train_prepared, train_labels)
 score_grid_ada = GridCV_ada.best_score_
model_ada = GridCV_ada.best_estimator_
print(score_grid_ada)
 predict_score_ada = GridCV_ada.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_ada )
all_blends = pd.read_csv(os.path.join(sub_path, 'All_Blends_9430.csv'), index_col=0)
 stack_median = pd.read_csv(os.path.join(sub_path, 'stack_median_9424.csv'), index_col=0)
 sub = stack_median.copy()
sub.head()
sub['isFraud'] = stack_median['isFraud'].values*0.156+all_blends['isFraud'].values*0.844
sub['isFraud'].head()
sub.head()
sub.to_csv('blend_of_blends.csv', float_format='%.6f' )<import_modules>
params_grid_gb = {}
gbc_model = GradientBoostingClassifier(criterion='friedman_mse', init=None,
learning_rate=0.05, loss='deviance', max_depth=3,
max_features=None, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=10000,
n_iter_no_change=None, presort='auto', random_state=None,
subsample=1.0, tol=0.0001, validation_fraction=0.1,
verbose=0, warm_start=False)
 GridCV_GB = GridSearchCV(gbc_model, params_grid_gb, verbose=1, cv=5)
 GridCV_GB.fit(train_prepared, train_labels)
 score_grid_GB = GridCV_GB.best_score_
model_gbc = GridCV_GB.best_estimator_
print(score_grid_GB)
 predict_score_GB = GridCV_GB.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_GB )
import numpy as np
import pandas as pd
import os
from sklearn import preprocessing
import lightgbm as lgb
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.model_selection import train_test_split
from bayes_opt import BayesianOptimization
import seaborn as sns
from sklearn import metrics
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import mean_squared_error
from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.metrics import auc
from sklearn.metrics import precision_score
from sklearn.metrics import roc_curve
from scipy.interpolate import interp1d
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix<load_from_csv>
mlp_clf = MLPClassifier(activation = "logistic", hidden_layer_sizes=(300,), random_state=42,batch_size=1000)
 param_grid = { 'max_iter': [1200], 'alpha': [1e-4],
'solver': ['sgd'], 'learning_rate_init': [0.05,0.06],'tol': [1e-4] }
GridCV_MLP = GridSearchCV(mlp_clf, param_grid, verbose=1, cv=3)
 GridCV_MLP.fit(train,train_labels)
 score_grid_MLP = GridCV_MLP.best_score_
model_mlp = GridCV_MLP.best_estimator_
print(score_grid_MLP)
 predict_score_MLP = GridCV_MLP.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_MLP )
train_id=pd.read_csv('.. /input/train_identity.csv')
 test_id=pd.read_csv('.. /input/test_identity.csv')
 train_id.head() 
<load_from_csv>
params_grid_search_lgb = {
}
lgb_model = lgb.LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.7,
gamma=0, importance_type='split', learning_rate=0.05, max_depth=3,
min_child_samples=20, min_child_weight=6, min_split_gain=0.0,
n_estimators=20000, n_jobs=-1, nthread=4, num_leaves=31,
objective=None, random_state=None, reg_alpha=0.0, reg_lambda=0.0,
scale_pos_weight=1, seed=29, silent=True, subsample=0.7,
subsample_for_bin=200000, subsample_freq=0)
 GridCV_LGB = GridSearchCV(lgb_model, params_grid_search_lgb, verbose=1, cv=5)
 GridCV_LGB.fit(train_prepared,train_labels)
 score_grid_LGB = GridCV_LGB.best_score_
model_lgb = GridCV_LGB.best_estimator_
print(score_grid_LGB)
 predict_score_LGB = GridCV_LGB.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_LGB )
train_data=pd.read_csv('.. /input/train_transaction.csv')
 test_data=pd.read_csv('.. /input/test_transaction.csv')
<merge>
params_grid_xgb = {}
xgb_model = xgb.XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bytree=0.7, gamma=0.2, learning_rate=0.009,
max_delta_step=0, max_depth=3, min_child_weight=6, missing=None,
n_estimators=10000, n_jobs=1, nthread=4, objective='binary:logistic',
random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,
seed=29, silent=True, subsample=0.7)
 GridCV_XGB = GridSearchCV(xgb_model, params_grid_xgb, verbose=1, cv=5)
 GridCV_XGB.fit(train_prepared, train_labels)
 score_grid_XGB = GridCV_XGB.best_score_
model_xgb = GridCV_XGB.best_estimator_
print(score_grid_XGB)
 predict_score_XGB = GridCV_XGB.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_XGB )
train = train_data.merge(train_id, how='left', on='TransactionID')
 test = test_data.merge(test_id, how='left',on='TransactionID')
 del train_id; gc.collect()
del test_id; gc.collect()
del train_data; gc.collect()
del test_data; gc.collect()<drop_column>
params_grid_vclf = {}
estimators = [ 
("Ada Boosting", model_ada), 
("Gradient Boost", model_gbc), 
("lgb_model", model_lgb), 
("XGBoost", model_xgb), 
("MLP", model_mlp),
]
voting_clf = VotingClassifier(estimators,voting='soft')
 GridCV_voting_clf = GridSearchCV(voting_clf, params_grid_vclf, verbose=1, cv=5)
 GridCV_voting_clf.fit(train_prepared, train_labels)
 score_grid_vclf = GridCV_voting_clf.best_score_
model_vclf = GridCV_voting_clf.best_estimator_
print(score_grid_vclf)
 predict_score_vclf = GridCV_voting_clf.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_vclf)
<features_selection>
test_prepared = num_pipeline.fit_transform(test )
training_missing = train.isna().sum(axis=0)/ train.shape[0]
test_missing = test.isna().sum(axis=0)/ test.shape[0]
change =(training_missing / test_missing ).sort_values(ascending=False)
 change = change[change<1e6]
train_more=change[change>4].reset_index()
train_more.columns=['train_more_id','rate']
test_more=change[change<0.4].reset_index()
test_more.columns=['test_more_id','rate']
train_more_id=train_more['train_more_id'].values
train[train_more_id].head()<categorify>
sub_voting_classifier = model_vclf.predict(test_prepared )
imp = Imputer(missing_values='NaN', strategy='most_frequent', axis=0)
 test[['dist1','dist2','C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14']]=imp.fit_transform(test[['dist1','dist2','C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14']])
 train[['dist1','dist2','C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14']]=imp.fit_transform(train[['dist1','dist2','C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14']] )<count_missing_values>
estimators =[model_ada,model_lgb,model_gbc, model_mlp]
sclf = StackingCVClassifier(classifiers= estimators,
meta_classifier=model_xgb)
 print('3-fold cross validation:
')
 for clf, label in zip([model_ada, model_lgb, model_gbc, model_mlp, sclf],
['Ada Boosting',
'light gradient Boosting(LGBM)',
'Gradient Boosting',
"MLP",
'StackingClassifier(Meta: XGboost)']):
scores = model_selection.cross_val_score(clf, train_prepared, train_labels,
cv=5, scoring='precision')
 print("precision: %0.2f(+/- %0.2f)[%s]"
%(scores.mean() , scores.std() , label))
isna = train.isna().sum(axis=1)
 isna_test = test.isna().sum(axis=1)
 train['isna']=train.isna().sum(axis=1)
 test['isna']=test.isna().sum(axis=1 )<categorify>
sclf.fit(train_prepared, train_labels )
test[test.TransactionDT>2.5e7][train_more_id]=imp.fit_transform(test[test.TransactionDT>2.5e7][train_more_id])
 train[train_more_id]=imp.fit_transform(train[train_more_id] )<feature_engineering>
prediction_sclf = sclf.predict(train_val)
 report_and_confusion_matrix(train_val_labels, prediction_sclf )
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')
 train['TransactionDT'] = train['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds = x)))
 test['TransactionDT'] = test['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds = x)))
 train['hour'] = train['TransactionDT'].dt.hour
test['hour'] = test['TransactionDT'].dt.hour
train['month'] = train['TransactionDT'].dt.month
test['month'] = test['TransactionDT'].dt.month 
<data_type_conversions>
prediction_stacking_clf = sclf.predict(test_prepared )
x=test['hour']<7
x1=test['hour']>19
x.astype(int)
 x1.astype(int)
 x2=np.add(x,x1)
 x2.astype(int)
 test['night']=x2.astype(int )<data_type_conversions>
sub = pd.read_csv('.. /input/gender_submission.csv')
 sub['Survived'] = sub_voting_classifier.astype('int64')
 sub.to_csv("voting_classifier.csv", index=False)
 sub1 = pd.read_csv('.. /input/gender_submission.csv')
 sub1['Survived'] = prediction_stacking_clf.astype('int64')
 sub.to_csv("stacking_clf.csv", index=False )
x=train['hour']<7
x1=train['hour']>19
x.astype(int)
 x1.astype(int)
 x2=np.add(x,x1)
 x2.astype(int)
 train['night']=x2.astype(int )<drop_column>
from keras.models import Sequential
from keras.optimizers import Adam
from keras.layers import Dense, Activation, Dropout,BatchNormalization
from keras.utils import to_categorical
def set_missing_card2(df):
give_df = df[['card2', 'TransactionID', 'card1', 'TransactionAmt', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13']]
known_card2 = give_df[give_df.card2.notnull() ].as_matrix()
unknown_card2 = give_df[give_df.card2.isnull() ].as_matrix()
y = known_card2[:, 0]
X = known_card2[:, 1:]
rfr = RandomForestRegressor(random_state=0, n_estimators=10, n_jobs=-1)
 rfr.fit(X, y)
 predictedcard2= rfr.predict(unknown_card2[:, 1::])
 df.loc[(df.card2.isnull()), 'card2']= predictedcard2
return df, rfr
train, rfr = set_missing_card2(train)
 test, rfr = set_missing_card2(test )<prepare_x_and_y>
model = Sequential()
model.add(Dense(input_dim=train.shape[1], units=128,
kernel_initializer='normal', bias_initializer='zeros'))
BatchNormalization() ,
model.add(Activation('relu'))
for i in range(0, 15):
BatchNormalization() ,
model.add(Dense(units=100, kernel_initializer='he_normal',
bias_initializer='zeros'))
BatchNormalization() ,
model.add(Activation('elu'))
model.add(Dense(units=2))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['accuracy'] )
def set_missing_card3(df):
give_df = df[['card3','TransactionID','card1', 'TransactionAmt','C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13']]
known_card3 = give_df[give_df.card3.notnull() ].as_matrix()
unknown_card3 = give_df[give_df.card3.isnull() ].as_matrix()
y = known_card3[:, 0]
X = known_card3[:, 1:]
rfr = RandomForestRegressor(random_state=0, n_estimators=10, n_jobs=-1)
 rfr.fit(X, y)
 predictedcard3 = rfr.predict(unknown_card3[:, 1::])
 df.loc[(df.card3.isnull()), 'card3' ] = predictedcard3
return df, rfr
train,rfr=set_missing_card3(train)
 test,rfr=set_missing_card3(test )<prepare_x_and_y>
labels = to_categorical(train_labels )
def set_missing_card5(df):
give_df = df[['card5','TransactionID','card1', 'TransactionAmt','C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13']]
known_card5 = give_df[give_df.card5.notnull() ].as_matrix()
unknown_card5 = give_df[give_df.card5.isnull() ].as_matrix()
y = known_card5[:, 0]
X = known_card5[:, 1:]
rfr = RandomForestRegressor(random_state=0, n_estimators=10, n_jobs=-1)
 rfr.fit(X, y)
 predictedcard5 = rfr.predict(unknown_card5[:, 1::])
 df.loc[(df.card5.isnull()), 'card5' ] = predictedcard5
return df, rfr
train,rfr=set_missing_card5(train)
 test,rfr=set_missing_card5(test )<prepare_x_and_y>
model_history = model.fit(train, labels, epochs=200, batch_size=600,validation_split=0.2, verbose=2 )
def set_missing_addr1(df):
give_df = df[['addr1','TransactionID','card1', 'TransactionAmt','C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13']]
known_addr1 = give_df[give_df.addr1.notnull() ].as_matrix()
unknown_addr1 = give_df[give_df.addr1.isnull() ].as_matrix()
y = known_addr1[:, 0]
X = known_addr1[:, 1:]
rfr = RandomForestRegressor(random_state=0, n_estimators=10, n_jobs=-1)
 rfr.fit(X, y)
 predictedaddr1 = rfr.predict(unknown_addr1[:, 1::])
 df.loc[(df.addr1.isnull()), 'addr1' ] = predictedaddr1
return df, rfr
train,rfr=set_missing_addr1(train)
 test,rfr=set_missing_addr1(test )<prepare_x_and_y>
test_prepared = num_pipeline.fit_transform(test )
<drop_column><EOS> 
submission_keras = model.predict_classes(test_prepared)
 sub['Survived'] = submission_keras.astype('int64')
 sub.to_csv("keras.csv", index=False )