Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
pd.DataFrame({ 'key':list(dftest['key']),
'fare_amount':fare } ).set_index('key' ).to_csv('sample_submission.csv', sep=',' )<define_variables>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_sample= pd.read_csv('.. /input/gender_submission.csv' )
traintypes = {'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
cols = list(traintypes.keys() )<load_from_csv>
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
%%time
train = pd.read_csv(TRAIN_PATH, usecols=cols, dtype=traintypes, nrows = 5000000 )<train_model>
Outliers_to_drop = detect_outliers(df_train,2,["Age","SibSp","Parch","Fare"])
 df_train = df_train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )
print("old size: %d" % len(train))
train = train[train.fare_amount >=0]
print("New size: %d" % len(train))<count_missing_values>
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in df_train["Name"]]
df_train["Title"] = pd.Series(dataset_title)
 df_train["Title"] = df_train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df_train["Title"] = df_train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_train["Title"] = df_train["Title"].astype(int)
 df_train.drop(labels = ["Name"], axis = 1, inplace = True )
train.isnull().sum()<train_model>
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age
print("old size: %d" % len(train))
train = train.dropna(how='any', axis=0)
 print("New size after dropping missing value: %d" % len(train))<set_options>
def impute_fare(cols):
Fare = cols[0]
Pclass = cols[1]
if pd.isnull(Fare):
if Pclass == 1:
return 84
elif Pclass == 2:
return 20
else:
return 13
else:
return Fare
% matplotlib inline
plt.style.use('seaborn-whitegrid' )<load_from_csv>
df_train['Age'] = df_train[['Age','Pclass']].apply(impute_age,axis=1 )
test = pd.read_csv(".. /input/test.csv")
 print("shape of test data", test.shape)
 test.head()<filter>
sex = pd.get_dummies(df_train['Sex'],drop_first=True)
 embark = pd.get_dummies(df_train['Embarked'],drop_first=True)
 df_train = pd.concat([df_train,sex,embark],axis=1 )
def select_within_test_boundary(df, BB):
return(df.pickup_longitude >= BB[0])&(df.pickup_longitude <= BB[1])& \ 
(df.pickup_latitude >= BB[2])&(df.pickup_latitude <= BB[3])& \ 
(df.dropoff_longitude >= BB[0])&(df.dropoff_longitude <= BB[1])& \ 
(df.dropoff_latitude >= BB[2])&(df.dropoff_latitude <= BB[3] )<feature_engineering>
df_train["Family"] = df_train["SibSp"] + df_train["Parch"] + 1
df_train['Single'] = df_train['Family'].map(lambda s: 1 if s == 1 else 0)
 df_train['SmallF'] = df_train['Family'].map(lambda s: 1 if s == 2 else 0)
 df_train['MedF'] = df_train['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df_train['LargeF'] = df_train['Family'].map(lambda s: 1 if s >= 5 else 0)
 df_train['Senior'] = df_train['Age'].map(lambda s:1 if s>60 else 0 )
def prepare_time_features(df):
df['pickup_datetime'] = df['pickup_datetime'].str.slice(0, 16)
 df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], utc=True, format='%Y-%m-%d %H:%M')
 df['hour_of_day'] = df.pickup_datetime.dt.hour
df['month'] = df.pickup_datetime.dt.month
df["year"] = df.pickup_datetime.dt.year
df["weekday"] = df.pickup_datetime.dt.weekday
return df<feature_engineering>
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in df_test["Name"]]
df_test["Title"] = pd.Series(dataset_title)
 df_test["Title"] = df_test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df_test["Title"] = df_test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_test["Title"] = df_test["Title"].astype(int)
 df_test.drop(labels = ["Name"], axis = 1, inplace = True )
train = prepare_time_features(train)
 test = prepare_time_features(test )<compute_test_metric>
df_test['Age'] = df_test[['Age','Pclass']].apply(impute_age,axis=1)
 sex = pd.get_dummies(df_test['Sex'],drop_first=True)
 embark = pd.get_dummies(df_test['Embarked'],drop_first=True)
 df_test = pd.concat([df_test,sex,embark],axis=1)
 df_test['Fare'].fillna(value=df_test['Fare'].median() ,inplace=True )
def distance(lat1, lon1, lat2, lon2):
p = 0.017453292519943295
a = 0.5 - np.cos(( lat2 - lat1)* p)/2 + np.cos(lat1 * p)* np.cos(lat2 * p)*(1 - np.cos(( lon2 - lon1)* p)) / 2
return 0.6213712 * 12742 * np.arcsin(np.sqrt(a))<feature_engineering>
df_test['Fare'] = df_test[['Fare','Pclass']].apply(impute_fare,axis=1 )
train['distance_miles'] = distance(train.pickup_latitude, train.pickup_longitude, \
train.dropoff_latitude, train.dropoff_longitude )<compute_test_metric>
df_test["Fare"] = df_test["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )
test['distance_miles'] = distance(test.pickup_latitude, test.pickup_longitude, \
test.dropoff_latitude, test.dropoff_longitude )<categorify>
df_test["Family"] = df_test["SibSp"] + df_test["Parch"] + 1
def transform(data):
jfk =(-73.7781, 40.6413)
 ewr =(-74.1745, 40.6895)
 lgr =(-73.8740, 40.7769)
 data['pickup_distance_to_jfk'] = distance(jfk[1], jfk[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_jfk'] = distance(jfk[1], jfk[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['pickup_distance_to_ewr'] = distance(ewr[1], ewr[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_ewr'] = distance(ewr[1], ewr[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['pickup_distance_to_lgr'] = distance(lgr[1], lgr[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_lgr'] = distance(lgr[1], lgr[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 return data
train = transform(train)
 test = transform(test )<drop_column>
df_test['Single'] = df_test['Family'].map(lambda s: 1 if s == 1 else 0)
 df_test['SmallF'] = df_test['Family'].map(lambda s: 1 if s == 2 else 0)
 df_test['MedF'] = df_test['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df_test['LargeF'] = df_test['Family'].map(lambda s: 1 if s >= 5 else 0)
 df_test['Senior'] = df_test['Age'].map(lambda s:1 if s>60 else 0 )
print("old size: %d" % len(train))
train = train.drop(index= train[train['fare_amount']==0].index, axis=0)
 print("New size: %d" % len(train))<drop_column>
df_train['Person'] = df_train[['Age','Sex']].apply(get_person,axis=1)
 df_test['Person'] = df_test[['Age','Sex']].apply(get_person,axis=1)
 person_dummies_train = pd.get_dummies(df_train['Person'])
 person_dummies_train.columns = ['Child','Female','Male']
person_dummies_train.drop(['Male'], axis=1, inplace=True)
 person_dummies_test = pd.get_dummies(df_test['Person'])
 person_dummies_test.columns = ['Child','Female','Male']
person_dummies_test.drop(['Male'], axis=1, inplace=True)
 df_train = df_train.join(person_dummies_train)
 df_test = df_test.join(person_dummies_test)
 df_train.drop(['Person'],axis=1,inplace=True)
 df_test.drop(['Person'],axis=1,inplace=True )
print("old size: %d" % len(train))
train = train.drop(index= train[train['fare_amount'] < 2.5].index, axis=0)
 print("New size: %d" % len(train))<train_model>
df_train.drop('male',axis=1,inplace=True)
 df_test.drop('male',axis=1,inplace=True )
print("old size: %d" % len(train))
train = train.drop(index= train[train.passenger_count >= 7].index, axis=0)
 print("New size: %d" % len(train))<count_values>
df_train.drop(['Cabin','Ticket'],axis = 1, inplace= True)
 df_test.drop(['Ticket','Cabin'],axis = 1, inplace= True )
pd.cut(test['distance_miles'],np.linspace(0, 70, num = 8)).value_counts()<split>
df_train.drop(['Sex','Embarked'],axis=1,inplace=True)
 df_test.drop(['Sex','Embarked'],axis=1,inplace=True )
X_train, X_test, y_train, y_test = train_test_split(df_train.drop('fare_amount', axis=1),
df_train['fare_amount'], test_size=0.2, random_state = 42)
 print(X_train.shape)
 print(X_test.shape)
 print(y_train.shape)
 print(y_test.shape )<init_hyperparams>
X_train, X_test, y_train, y_test = train_test_split(df_train.drop('Survived',axis=1),
df_train['Survived'], test_size=0.15,
random_state=101 )
params = {
'max_depth': 7,
'gamma' :0,
'eta':.03,
'subsample': 1,
'colsample_bytree': 0.9,
'objective':'reg:linear',
'eval_metric':'rmse',
'silent': 0
}<train_model>
dt = DecisionTreeClassifier()
dt.fit(X_train,y_train);
plt.figure(figsize=(18,18))
plot_tree(dt,filled=True);
def XGBmodel(X_train,X_test,y_train,y_test,params):
matrix_train = xgb.DMatrix(X_train,label=y_train)
 matrix_test = xgb.DMatrix(X_test,label=y_test)
 model=xgb.train(params=params,
dtrain=matrix_train,num_boost_round=5000,
early_stopping_rounds=10,evals=[(matrix_test,'test')])
 return model
model = XGBmodel(X_train,X_test,y_train,y_test,params )<predict_on_test>
XGB = XGBClassifier(max_depth=4,learning_rate=0.005,n_estimators=500,n_jobs=-1,min_child_weight=2)
 XGB.fit(X_train,y_train )
prediction = model.predict(xgb.DMatrix(df_test), ntree_limit = model.best_ntree_limit ).tolist()<save_to_csv>
y_pred = pd.DataFrame(XGB.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_xgb = y_pred
y_pred.to_csv('titanic_pred_xgb.csv',index=False )
test = pd.read_csv(".. /input/test.csv")
 holdout = pd.DataFrame({'key': test['key'], 'fare_amount': prediction})
 holdout.to_csv('xgb_with4mtrainedrows.csv', index=False )<load_from_csv>
Scaler1 = StandardScaler()
Scaler2 = StandardScaler()
X_train_scaled = Scaler1.fit_transform(X_train)
 df_test_scaled = Scaler2.fit_transform(df_test )
<import_modules>
logmodel = LogisticRegression(C=10 ).fit(X_train,y_train)
 y_pred = pd.DataFrame(logmodel.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_lr = y_pred
y_pred.to_csv('titanic_pred_logistic.csv',index=False )
<load_from_csv>
RFC = RandomForestClassifier(n_estimators=500,max_depth=9,min_samples_split=3)
 RFC.fit(X_train,y_train )
dtypes = {'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
val_size = 10_000
input_path = '.. /input/train.csv'
val_df = pd.read_csv(input_path, usecols=dtypes.keys() , dtype=dtypes, nrows=val_size)
 val_df = remove_outliers(val_df)
 val_df = extract_features(val_df)
 X_val = val_df.drop(columns='fare_amount')
 y_val = val_df[['fare_amount']]
dval = xgb.DMatrix(X_val, y_val, feature_names=X_val.columns)
 batch_size = 10_000_000
columns = pd.read_csv(input_path, nrows=0 ).columns
train_df = pd.read_csv(input_path, usecols=dtypes.keys() , dtype=dtypes,
names=columns, skiprows=val_size + 1,
chunksize=batch_size)
 val_df.dtypes<prepare_x_and_y>
y_pred = pd.DataFrame(RFC.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_rf = y_pred
y_pred.to_csv('titanic_pred_rfc.csv',index=False )
params = {'learning_rate': 0.05,
'max_depth': 7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'subsample': 0.8,
'gamma': 1,
'silent': True,
'verbose_eval': True}
num_rounds = 100
model = None
for batch_df in tqdm(train_df):
batch_df = remove_outliers(batch_df)
 batch_df = extract_features(batch_df)
 X_train = batch_df.drop(columns='fare_amount')
 y_train = batch_df[['fare_amount']]
dtrain = xgb.DMatrix(X_train, y_train, feature_names=X_train.columns)
 model = xgb.train(params, dtrain, num_rounds, early_stopping_rounds=5,
evals=[(dtrain, 'train'),(dval, 'eval')],
xgb_model=model )<load_from_csv>
lgb = LGBMClassifier(learning_rate=0.01,max_depth=5,n_estimators=500,num_leaves=3 ).fit(X_train,y_train )
test_df = pd.read_csv('.. /input/test.csv')
 test_df = extract_features(test_df)
 test_df.dtypes<save_to_csv>
y_pred = pd.DataFrame(lgb.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_lgb = y_pred
y_pred.to_csv('titanic_pred_lgb.csv',index=False )
X_test = test_df.drop(columns='key')
 dtest = xgb.DMatrix(X_test, feature_names=X_test.columns)
 y_pred = model.predict(dtest)
 submission = pd.DataFrame({'key': test_df['key'], 'fare_amount': y_pred})
 submission.to_csv('submission.csv', index = False)
 !head submission.csv<import_modules>
print("XGB train score: ",round(XGB.score(X_train,y_train),2), " XGB test score: ",round(XGB.score(X_test,y_test),2))
print("Log-Reg.train score: ",round(logmodel.score(X_train,y_train),2)," Log-Reg.test score: ",round(logmodel.score(X_test,y_test),2))
print("Random Forest's train score: ",round(RFC.score(X_train,y_train),2), " Random Forest test score: ",round(RFC.score(X_test,y_test),2))
print("LGBM train score: ",round(lgb.score(X_train,y_train),2), " LGB Model test score: ",round(lgb.score(X_test,y_test),2))
tqdm.pandas() 
<set_options>
y_valid_xgb = XGB.predict(X_test)
 y_valid_log = logmodel.predict(X_test)
 y_valid_rfc = RFC.predict(X_test)
 y_valid_lgb = lgb.predict(X_test )
%matplotlib inline
%precision 4
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 np.set_printoptions(suppress=True)
 pd.set_option("display.precision", 15 )<load_from_csv>
fpr_xgb, tpr_xgb, thresholds_xgb = roc_curve(y_test, y_valid_xgb)
 roc_auc_xgb = auc(fpr_xgb, tpr_xgb)
 fpr_log, tpr_log, thresholds_log = roc_curve(y_test, y_valid_log)
 roc_auc_log = auc(fpr_log, tpr_log)
 fpr_rfc, tpr_rfc, thresholds_rfc = roc_curve(y_test, y_valid_rfc)
 roc_auc_rfc = auc(fpr_rfc, tpr_rfc)
 fpr_lgb, tpr_lgb, thresholds_lgb = roc_curve(y_test, y_valid_lgb)
 roc_auc_lgb = auc(fpr_lgb, tpr_lgb )
df_train = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv', index_col=0)
 y_train = df_train.pop('target')
 len_train = len(df_train)
 df_test = pd.read_csv('./.. /input/santander-customer-transaction-prediction/test.csv', index_col=0)
 df_all = pd.concat(( df_train, df_test), sort=False)
 prev_cols = df_all.columns
scaler = StandardScaler()
df_all[prev_cols] = scaler.fit_transform(df_all[prev_cols])
 df_train = df_all[0:len_train]
df_test = df_all[len_train:] 
<randomize_order>
y_pred_final = y_pred
y_pred_final['Survived'] = round(0.25 * y_pred_lgb['Survived']
+ 0.25 * y_pred_rf['Survived']
+ 0.25 * y_pred_xgb['Survived']
+ 0.25 * y_pred_lr['Survived'])
 y_pred_final['PassengerId'] = df_test['PassengerId']
y_pred_final['Survived'] = y_pred_final['Survived'].astype(int)
 y_pred_final.to_csv('titanic_pred_final.csv',index=False )
def augment_train(df_train, y_train):
t0 = df_train[y_train == 0].copy()
t1 = df_train[y_train == 1].copy()
i = 0
N = 3
for I in range(0):
for col in df_train.columns:
i = i + 1000
np.random.seed(i)
 np.random.shuffle(t0[col].values)
 np.random.shuffle(t1[col].values)
 df_train = pd.concat([df_train, t0.copy() ])
 df_train = pd.concat([df_train, t1.copy() ])
 y_train = pd.concat([y_train, pd.Series([0] * t0.shape[0]), pd.Series([1] * t1.shape[0])])
 return df_train, y_train<count_unique_values>
y_pred_final['Survived'].value_counts()
features = [c for c in df_train.columns if c not in ["ID_code","target"]]
def detect_test(test_df):
df_test=test_df.values
unique_count = np.zeros_like(df_test)
 for feature in tqdm(range(df_test.shape[1])) :
_, index_, count_ = np.unique(df_test[:, feature], return_counts=True, return_index=True)
 unique_count[index_[count_ == 1], feature] += 1
real_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synthetic_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
return real_samples_indexes,synthetic_samples_indexes
def generate_fe(trn, tst):
real,syn = detect_test(df_test[features])
 al = pd.concat([trn,tst,df_test.iloc[real]],axis=0)
 trn_fe = pd.DataFrame()
tst_fe = pd.DataFrame()
for c in features:
trn[c+"_test"]=trn[c].map(al[c].value_counts())
 trn[c+"_multi"] = trn[c+"_test"]*trn[c]
trn_fe[c] = trn[c]
trn_fe[c+"_test"] = trn[c+"_test"]
trn_fe[c+"_muti"] = trn[c+"_multi"]
tst[c+"_test"]=tst[c].map(al[c].value_counts())
 tst_fe[c] = tst[c]
tst[c+"_multi"] = tst[c+"_test"]*tst[c]
tst_fe[c+"_test"] = tst[c+"_test"]
tst_fe[c+"_muti"] = tst[c+"_multi"]
return trn_fe, tst_fe 
<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
def generate_fe_test(tst):
re,sy = detect_test(tst[features])
 al = pd.concat([df_train,df_test.iloc[re]],axis=0)
 tst_fe = pd.DataFrame()
for c in features:
tst[c+"_test"]=tst[c].map(al[c].value_counts())
 tst_fe[c] = tst[c]
tst[c+"_multi"] = tst[c+"_test"]*tst[c]
tst_fe[c+"_test"] = tst[c+"_test"]
tst_fe[c+"_muti"] = tst[c+"_multi"]
return tst_fe
test_fe = generate_fe_test(df_test[features] )<train_model>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_sample= pd.read_csv('.. /input/gender_submission.csv' )
class Logger(callbacks.Callback):
def __init__(self, out_path='./', patience=30, lr_patience=3, out_fn='', log_fn=''):
self.auc = 0
self.path = out_path
self.fn = out_fn
self.patience = patience
self.lr_patience = lr_patience
self.no_improve = 0
self.no_improve_lr = 0
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
cv_pred = self.model.predict(self.validation_data[0], batch_size=1024)
 cv_true = self.validation_data[1]
auc_val = roc_auc_score(cv_true, cv_pred)
 if self.auc < auc_val:
self.no_improve = 0
self.no_improve_lr = 0
print("Epoch %s - best AUC: %s" %(epoch, round(auc_val, 4)))
 self.auc = auc_val
self.model.save(self.path + self.fn, overwrite=True)
 else:
self.no_improve += 1
self.no_improve_lr += 1
print("Epoch %s - current AUC: %s" %(epoch, round(auc_val, 4)))
 if self.no_improve >= self.patience:
self.model.stop_training = True
if self.no_improve_lr >= self.lr_patience:
lr = float(K.get_value(self.model.optimizer.lr))
K.set_value(self.model.optimizer.lr, 0.75*lr)
 print("Setting lr to {}".format(0.75*lr))
self.no_improve_lr = 0
return<prepare_x_and_y>
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
preds = []
c = 0
oof_preds = np.zeros(( len(df_train), 1))
cv = StratifiedKFold(n_splits=5,shuffle=True, random_state=3263)
 for train, valid in cv.split(df_train, y_train):
print("VAL %s" % c)
 trn = df_train.iloc[train]
tst = df_train.iloc[valid]
trn, tst = generate_fe(trn, tst)
 X_train = np.reshape(trn.values,(-1,200,3))
y_train_ = y_train.iloc[train].values
X_valid = np.reshape(tst.values,(-1,200,3))
y_valid = y_train.iloc[valid].values
model = _Model()
logger = Logger(patience=30, out_path='./', out_fn='cv_{}.h5'.format(c))
model.fit(X_train, y_train_, validation_data=(X_valid, y_valid), epochs=150, verbose=2, batch_size=1024,
callbacks=[logger])
 model.load_weights('cv_{}.h5'.format(c))
fe = [c for c in test_fe.columns if c not in ["ID_code","target"]]
X_test = np.reshape(test_fe[fe].values,(200000, 200, 3))
curr_preds = model.predict(X_test, batch_size=2048)
 oof_preds[valid] = model.predict(X_valid)
 preds.append(curr_preds)
 c += 1
pd.DataFrame(oof_preds ).to_csv("NN_oof_preds.csv", index = False)
 auc = roc_auc_score(y_train, oof_preds)
 print("CV_AUC: {}".format(auc))
preds = np.asarray(preds)
 preds = preds.reshape(( 5, 200000))
preds_final = np.mean(preds.T, axis=1)
 submission = pd.read_csv('./.. /input/santander-customer-transaction-prediction/sample_submission.csv')
 submission['target'] = preds_final
submission.to_csv('submission.csv', index=False)
<set_options>
Outliers_to_drop = detect_outliers(df_train,2,["Age","SibSp","Parch","Fare"])
 df_train = df_train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )
pd.options.display.max_rows = 200
pd.options.display.max_columns = 200
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42<load_from_csv>
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in df_train["Name"]]
df_train["Title"] = pd.Series(dataset_title)
 df_train["Title"] = df_train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df_train["Title"] = df_train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_train["Title"] = df_train["Title"].astype(int)
 df_train.drop(labels = ["Name"], axis = 1, inplace = True )
df_train = pd.read_csv('.. /input/train.csv')
 df_train.name = 'Training Set'
df_test = pd.read_csv('.. /input/test.csv')
 df_test.name = 'Test Set'
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}'.format(df_train['target'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns )<merge>
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age
df_train_unique = df_train.agg(['nunique'] ).transpose().sort_values(by='nunique')
 df_test_unique = df_test.agg(['nunique'] ).transpose().sort_values(by='nunique')
 df_uniques = df_train_unique.drop('target' ).reset_index().merge(df_test_unique.reset_index() , how='left', right_index=True, left_index=True)
 df_uniques.drop(columns=['index_y'], inplace=True)
 df_uniques.columns = ['Feature', 'Training Set Unique Count', 'Test Set Unique Count']<create_dataframe>
def impute_fare(cols):
Fare = cols[0]
Pclass = cols[1]
if pd.isnull(Fare):
if Pclass == 1:
return 84
elif Pclass == 2:
return 20
else:
return 13
else:
return Fare
df_qdist = pd.DataFrame(np.zeros(( 200, 9)) , columns=['Quartile 1 Positives', 'Quartile 2 Positives', 'Quartile 3 Positives', 'Quartile 4 Positives',
'Quartile 1 Positive Percentage', 'Quartile 2 Positive Percentage', 'Quartile 3 Positive Percentage', 'Quartile 4 Positive Percentage',
'Quartile Order'])
 features = [col for col in df_train.columns.values.tolist() if col.startswith('var')]
quartiles = np.arange(0, 1, 0.25)
 df_qdist.index = features
for i, feature in enumerate(features):
for j, quartile in enumerate(quartiles):
target_counts = df_train[np.logical_and(df_train[feature] >= df_train[feature].quantile(q=quartile),
df_train[feature] < df_train[feature].quantile(q=quartile + 0.25)) ].target.value_counts()
ones_per = target_counts[1] /(target_counts[0] + target_counts[1])* 100
df_qdist.iloc[i, j] = target_counts[1]
df_qdist.iloc[i, j + 4] = ones_per
pers = df_qdist.columns.tolist() [4:-1]
for i, index in enumerate(df_qdist.index):
order = df_qdist[pers].iloc[[i]].sort_values(by=index, ascending=False, axis=1 ).columns
order_str = ''.join([col[9] for col in order])
 df_qdist.iloc[i, 8] = order_str
df_qdist = df_qdist.round(2)
 df_qdist.head(10 )<filter>
df_train['Age'] = df_train[['Age','Pclass']].apply(impute_age,axis=1 )
df_qdist[np.logical_or(df_qdist['Quartile Order'].str.startswith('2'), df_qdist['Quartile Order'].str.startswith('3')) ]<count_unique_values>
sex = pd.get_dummies(df_train['Sex'],drop_first=True)
 embark = pd.get_dummies(df_train['Embarked'],drop_first=True)
 df_train = pd.concat([df_train,sex,embark],axis=1 )
test = df_test.drop(['ID_code'], axis=1 ).values
unique_count = np.zeros_like(test)
 for feature in range(test.shape[1]):
_, index, count = np.unique(test[:, feature], return_counts=True, return_index=True)
 unique_count[index[count == 1], feature] += 1
real_samples = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synth_samples = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
print('Number of real samples in test set is {}'.format(len(real_samples)))
 print('Number of synthetic samples in test set is {}'.format(len(synth_samples)) )<data_type_conversions>
df_train["Family"] = df_train["SibSp"] + df_train["Parch"] + 1
df_train['Single'] = df_train['Family'].map(lambda s: 1 if s == 1 else 0)
 df_train['SmallF'] = df_train['Family'].map(lambda s: 1 if s == 2 else 0)
 df_train['MedF'] = df_train['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df_train['LargeF'] = df_train['Family'].map(lambda s: 1 if s >= 5 else 0)
 df_train['Senior'] = df_train['Age'].map(lambda s:1 if s>60 else 0 )
features = [col for col in df_train.columns if col.startswith('var')]
df_all = pd.concat([df_train, df_test.ix[real_samples]])
 for feature in features:
temp = df_all[feature].value_counts(dropna=True)
 df_train[feature + 'vc'] = df_train[feature].map(temp ).map(lambda x: min(10, x)).astype(np.uint8)
 df_test[feature + 'vc'] = df_test[feature].map(temp ).map(lambda x: min(10, x)).astype(np.uint8)
 df_train[feature + 'sum'] =(( df_train[feature] - df_all[feature].mean())* df_train[feature + 'vc'].map(lambda x: int(x > 1)) ).astype(np.float32)
 df_test[feature + 'sum'] =(( df_test[feature] - df_all[feature].mean())* df_test[feature + 'vc'].map(lambda x: int(x > 1)) ).astype(np.float32)
 df_train[feature + 'sum2'] =(( df_train[feature])* df_train[feature + 'vc'].map(lambda x: int(x > 2)) ).astype(np.float32)
 df_test[feature + 'sum2'] =(( df_test[feature])* df_test[feature + 'vc'].map(lambda x: int(x > 2)) ).astype(np.float32)
 df_train[feature + 'sum3'] =(( df_train[feature])* df_train[feature + 'vc'].map(lambda x: int(x > 4)) ).astype(np.float32)
 df_test[feature + 'sum3'] =(( df_test[feature])* df_test[feature + 'vc'].map(lambda x: int(x > 4)) ).astype(np.float32)
 print('Training set shape after creating magic features: {}'.format(df_train.shape))
print('Test set shape after creating magic features: {}'.format(df_test.shape))<categorify>
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in df_test["Name"]]
df_test["Title"] = pd.Series(dataset_title)
 df_test["Title"] = df_test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df_test["Title"] = df_test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_test["Title"] = df_test["Title"].astype(int)
 df_test.drop(labels = ["Name"], axis = 1, inplace = True )
def augment(x, y, t=2):
xs, xn = [], []
for i in range(t // 2):
mask = y == 0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 featnum = x1.shape[1] // 200 - 1
for c in range(200):
np.random.shuffle(ids)
 x1[:, [c] + [200 + featnum * c + idc for idc in range(featnum)]] = x1[ids][:, [c] + [200 + featnum * c + idc for idc in range(featnum)]]
xn.append(x1)
 for i in range(t):
mask = y > 0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 featnum = x1.shape[1] // 200 - 1
for c in range(200):
np.random.shuffle(ids)
 x1[:, [c] + [200 + featnum * c + idc for idc in range(1)]] = x1[ids][:, [c] + [200 + featnum * c + idc for idc in range(1)]]
xs.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x, xs, xn])
 y = np.concatenate([y, ys, yn])
 return x, y<feature_engineering>
df_test['Age'] = df_test[['Age','Pclass']].apply(impute_age,axis=1)
 sex = pd.get_dummies(df_test['Sex'],drop_first=True)
 embark = pd.get_dummies(df_test['Embarked'],drop_first=True)
 df_test = pd.concat([df_test,sex,embark],axis=1)
 df_test['Fare'].fillna(value=df_test['Fare'].median() ,inplace=True )
<categorify>
df_test['Fare'] = df_test[['Fare','Pclass']].apply(impute_fare,axis=1 )
class KMeansFeaturizer:
def __init__(self, k, target_scale=5.0, random_state=None):
self.k = k
self.target_scale = target_scale
self.random_state = random_state
self.encoder = OneHotEncoder(categories='auto' ).fit(np.array(range(k)).reshape(-1, 1))
def fit(self, X, y=None):
if y is None:
kmeans = KMeans(n_clusters=self.k, n_init=20, random_state=self.random_state)
 kmeans.fit(X)
 self.kmeans = kmeans
self.cluster_centers_ = kmeans.cluster_centers_
else:
Xy = np.hstack(( X, y[:, np.newaxis] * self.target_scale))
kmeans_pretrain = KMeans(n_clusters=self.k, n_init=20, random_state=self.random_state)
 kmeans_pretrain.fit(Xy)
 kmeans = KMeans(n_clusters=self.k, init=kmeans_pretrain.cluster_centers_[:, :2], n_init=1, max_iter=1)
 kmeans.fit(X)
 self.kmeans = kmeans
self.cluster_centers_ = km_model.cluster_centers_
return self
def transform(self, X, y=None):
clusters = self.kmeans.predict(X)
 return self.encoder.transform(clusters.reshape(-1, 1))
def fit_transform(self, X, y=None):
self.fit(X, y)
 return self.transform(X, y)
<feature_engineering>
df_test["Fare"] = df_test["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )
transform_feature(df=df_train, feature='var_108', transformation=np.round, decimals=2 )<init_hyperparams>
df_test["Family"] = df_test["SibSp"] + df_test["Parch"] + 1
gbdt_param = {
'objective': 'binary',
'boosting': 'gbdt',
'learning_rate': 0.01,
'num_leaves': 15,
'tree_learner': 'serial',
'num_threads': 8,
'seed': SEED,
'max_depth': -1,
'min_data_in_leaf': 50,
'min_sum_hessian_in_leaf': 10,
'bagging_fraction': 0.6,
'bagging_freq': 5,
'feature_fraction': 0.05,
'lambda_l1': 1.,
'bagging_seed': SEED,
'verbosity ': 1,
'boost_from_average': False,
'metric': 'auc',
}<prepare_x_and_y>
df_test['Single'] = df_test['Family'].map(lambda s: 1 if s == 1 else 0)
 df_test['SmallF'] = df_test['Family'].map(lambda s: 1 if s == 2 else 0)
 df_test['MedF'] = df_test['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df_test['LargeF'] = df_test['Family'].map(lambda s: 1 if s >= 5 else 0)
 df_test['Senior'] = df_test['Age'].map(lambda s:1 if s>60 else 0 )
predictors = df_train.columns.tolist() [2:]
X_test = df_test[predictors]
n_splits = 5
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = df_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()<split>
df_train['Person'] = df_train[['Age','Sex']].apply(get_person,axis=1)
 df_test['Person'] = df_test[['Age','Sex']].apply(get_person,axis=1)
 person_dummies_train = pd.get_dummies(df_train['Person'])
 person_dummies_train.columns = ['Child','Female','Male']
person_dummies_train.drop(['Male'], axis=1, inplace=True)
 person_dummies_test = pd.get_dummies(df_test['Person'])
 person_dummies_test.columns = ['Child','Female','Male']
person_dummies_test.drop(['Male'], axis=1, inplace=True)
 df_train = df_train.join(person_dummies_train)
 df_test = df_test.join(person_dummies_test)
 df_train.drop(['Person'],axis=1,inplace=True)
 df_test.drop(['Person'],axis=1,inplace=True )
for fold,(train_ind, val_ind)in enumerate(skf.split(df_train, df_train.target.values)) :
X_train, y_train = df_train.iloc[train_ind][predictors], df_train.iloc[train_ind]['target']
X_valid, y_valid = df_train.iloc[val_ind][predictors], df_train.iloc[val_ind]['target']
N = 1
p_valid, yp = 0, 0
for i in range(N):
print('
Fold {} - N {}'.format(fold + 1, i + 1))
X_t, y_t = augment(X_train.values, y_train.values)
 weights = np.array([0.8] * X_t.shape[0])
 weights[:X_train.shape[0]] = 1.0
print('Shape of X_train after augment: {}
Shape of y_train after augment: {}'.format(X_t.shape, y_t.shape))
X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t, weight=weights)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(gbdt_param, trn_data, 100000, valid_sets=[trn_data, val_data], early_stopping_rounds=5000, verbose_eval=1000, evals_result=evals_result)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = predictors
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_ind] = p_valid / N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold + 1)] = yp / N 
<compute_test_metric>
df_train.drop('male',axis=1,inplace=True)
 df_test.drop('male',axis=1,inplace=True )
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print('Mean AUC: {}, std: {}.
 All AUC: {}.'.format(mean_auc, std_auc, all_auc))<save_to_csv>
df_train.drop(['Cabin','Ticket'],axis = 1, inplace= True)
 df_test.drop(['Ticket','Cabin'],axis = 1, inplace= True )
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )<load_from_csv>
df_train.drop(['Sex','Embarked'],axis=1,inplace=True)
 df_test.drop(['Sex','Embarked'],axis=1,inplace=True )
warnings.simplefilter(action='ignore', category=FutureWarning)
 train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 synthetic_samples_indexes = np.load('.. /input/fakedata/synthetic_samples_indexes.npy')
 private_LB = np.load('.. /input/fakedata/private_LB.npy')
 public_LB = np.load('.. /input/fakedata/public_LB.npy')
 full = pd.concat([train_df, pd.concat([test_df.loc[private_LB], test_df.loc[public_LB]], sort = False)], sort = False)
 for feat in ['var_' + str(x)for x in range(200)]:
count_values = full.groupby(feat)[feat].count()
train_df['new_' + feat] = count_values.loc[train_df[feat]].values
test_df['new_' + feat] = count_values.loc[test_df[feat]].values
print('full_df has {} rows and {} columns'.format(full.shape[0], full.shape[1])) 
<init_hyperparams>
X_train, X_test, y_train, y_test = train_test_split(df_train.drop('Survived',axis=1),
df_train['Survived'], test_size=0.15,
random_state=101 )
seed = 2319
param = {
'num_leaves': 8,
'min_data_in_leaf': 17,
'learning_rate': 0.01,
'min_sum_hessian_in_leaf': 9.67,
'bagging_fraction': 0.8329,
'bagging_freq': 2,
'feature_fraction': 1,
'lambda_l1': 0.6426,
'lambda_l2': 0.3067,
'min_gain_to_split': 0.02832,
'max_depth': -1,
'seed': seed,
'feature_fraction_seed': seed,
'bagging_seed': seed,
'drop_seed': seed,
'data_random_seed': seed,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbosity': -1,
'metric': 'auc',
'is_unbalance': True,
'save_binary': True,
'boost_from_average': 'false',
'num_threads': 8
}
features = [c for c in train_df.columns if(c not in ['ID_code', 'target'])]
test_size = 0.3
X_train, X_test, y_train, y_test = train_test_split(train_df[features], train_df['target'], test_size = test_size, random_state=42)
 iterations = 110
y_hat = np.zeros([int(200000*test_size), 200])
 i = 0
for feature in ['var_' + str(x)for x in range(200)]:
feat_choices = [feature, 'new_' + feature]
lgb_train = lgb.Dataset(X_train[feat_choices], y_train)
 gbm = lgb.train(param, lgb_train, iterations, verbose_eval=-1)
 y_hat[:, i] = gbm.predict(X_test[feat_choices], num_iteration=gbm.best_iteration)
 i += 1
sub_preds =(y_hat ).sum(axis=1)
 score = roc_auc_score(y_test, sub_preds)
 print('Your CV score is', score )<init_hyperparams>
dt = DecisionTreeClassifier()
dt.fit(X_train,y_train);
plt.figure(figsize=(18,18))
plot_tree(dt,filled=True);
iterations = 126
param = {'bagging_fraction': 0.7693,
'bagging_freq': 2,
'lambda_l1': 0.7199,
'lambda_l2': 1.992,
'learning_rate': 0.009455,
'max_depth': 3,
'min_data_in_leaf': 22,
'min_gain_to_split': 0.06549,
'min_sum_hessian_in_leaf': 18.55,
'num_leaves': 20,
'feature_fraction': 1,
'save_binary': True,
'seed': 2319,
'feature_fraction_seed': 2319,
'bagging_seed': 2319,
'drop_seed': 2319,
'data_random_seed': 2319,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbosity': -1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': 'false',
'num_threads': 6}<init_hyperparams>
XGB = XGBClassifier(max_depth=4,learning_rate=0.005,n_estimators=500,n_jobs=-1,min_child_weight=2)
 XGB.fit(X_train,y_train )
iterations = 126
param = {'bagging_fraction': 0.7693,
'bagging_freq': 2,
'lambda_l1': 0.7199,
'lambda_l2': 1.992,
'learning_rate': 0.009455,
'max_depth': 3,
'min_data_in_leaf': 22,
'min_gain_to_split': 0.06549,
'min_sum_hessian_in_leaf': 18.55,
'num_leaves': 20,
'feature_fraction': 1,
'save_binary': True,
'seed': 2319,
'feature_fraction_seed': 2319,
'bagging_seed': 2319,
'drop_seed': 2319,
'data_random_seed': 2319,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbosity': -1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': 'false',
'num_threads': 6}
features = [c for c in train_df.columns if(c not in ['ID_code', 'target'])]
test_size = 0.3
X_train, X_test, y_train, y_test = train_test_split(train_df[features], train_df['target'], test_size = test_size, random_state=42)
 iterations = 110
y_hat = np.zeros([int(200000*test_size), 200])
 i = 0
for feature in ['var_' + str(x)for x in range(200)]:
feat_choices = [feature, 'new_' + feature]
lgb_train = lgb.Dataset(X_train[feat_choices], y_train)
 gbm = lgb.train(param, lgb_train, iterations, verbose_eval=-1)
 y_hat[:, i] = gbm.predict(X_test[feat_choices], num_iteration=gbm.best_iteration)
 i += 1 
<compute_test_metric>
y_pred = pd.DataFrame(XGB.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_xgb = y_pred
y_pred.to_csv('titanic_pred_xgb.csv',index=False )
weights = []
for col in range(200):
if roc_auc_score(y_test, y_hat[:,col])>= 0.5:
weights.append(roc_auc_score(y_test, y_hat[:,col]))
else:
weights.append(0)
 weights = np.array(weights)
 weights =(weights - weights.mean())/ weights.mean()
weights += 1
sub_preds_regular =(y_hat ).sum(axis=1)
 sub_preds_weighted =(y_hat*weights ).sum(axis=1)
 score_regular = roc_auc_score(y_test, sub_preds_regular)
 score_weighted = roc_auc_score(y_test, sub_preds_weighted)
 print('Your unweigthed score is:', score_regular)
 print('Your weigthed score is:', score_weighted )<compute_test_metric>
Scaler1 = StandardScaler()
Scaler2 = StandardScaler()
X_train_scaled = Scaler1.fit_transform(X_train)
 df_test_scaled = Scaler2.fit_transform(df_test )
test_data_length = len(y_test)
 validation_length = int(test_data_length/4)
 weights = []
for col in range(200):
if roc_auc_score(y_test[:validation_length], y_hat[:validation_length,col])>= 0.5:
weights.append(roc_auc_score(y_test[:validation_length], y_hat[:validation_length,col]))
else:
weights.append(0)
 weights = np.array(weights)
 weights =(weights - weights.mean())/ weights.mean()
weights += 1
sub_preds_regular =(y_hat[validation_length:] ).sum(axis=1)
 sub_preds_weighted =(y_hat[validation_length:]*weights ).sum(axis=1)
 score_regular = roc_auc_score(y_test[validation_length:], sub_preds_regular)
 score_weighted = roc_auc_score(y_test[validation_length:], sub_preds_weighted)
 print('Your unweigthed score is:', score_regular)
 print('Your weigthed score is:', score_weighted)
 if score_weighted > score_regular:
print('Your weights ARE NOT overfitting')
 else:
print('Your weights ARE overfitting' )<concatenate>
logmodel = LogisticRegression(C=10 ).fit(X_train,y_train)
 y_pred = pd.DataFrame(logmodel.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_lr = y_pred
y_pred.to_csv('titanic_pred_logistic.csv',index=False )
pd.options.mode.chained_assignment = None
min_n_unique_full = 2000
min_n_unique_train = int(min_n_unique_full*2/3)
 min_n_unique_test = int(min_n_unique_full*1/3)
 full = pd.concat([train_df, pd.concat([test_df.loc[private_LB], test_df.loc[public_LB]], sort = False)], sort = False)
 true_test_df = pd.concat([test_df.loc[private_LB], test_df.loc[public_LB]], sort = False)
 count = 1
for feat in ['var_' + str(x)for x in range(200)]:
if count%50 == 1:
print('Processing reached', feat)
 n_unique_list_full = list(set(full['new_' + feat]))
n_unique_list_full.sort()
for i in range(len(n_unique_list_full)) :
n_unique_full = n_unique_list_full[i]
len_n_unique_full = len(full[feat][full['new_' + feat] == n_unique_full])
 len_n_unique_train = len(train_df[feat][train_df['new_' + feat] == n_unique_full])
 len_n_unique_test = len(true_test_df[feat][true_test_df['new_' + feat] == n_unique_full])
 if len_n_unique_full < min_n_unique_full or len_n_unique_train < min_n_unique_train or len_n_unique_test < min_n_unique_test :
try:
full['new_' + feat][full['new_' + feat] == n_unique_full] = n_unique_list_full[i+1]
train_df['new_' + feat][train_df['new_' + feat] == n_unique_full] = n_unique_list_full[i+1]
true_test_df['new_' + feat][true_test_df['new_' + feat] == n_unique_full] = n_unique_list_full[i+1]
test_df['new_' + feat][test_df['new_' + feat] == n_unique_full] = n_unique_list_full[i+1]
except:
continue
for i in reversed(range(len(n_unique_list_full))):
n_unique_full = n_unique_list_full[i]
len_n_unique_full = len(full[feat][full['new_' + feat] == n_unique_full])
 len_n_unique_train = len(train_df[feat][train_df['new_' + feat] == n_unique_full])
 len_n_unique_test = len(true_test_df[feat][true_test_df['new_' + feat] == n_unique_full])
 if len_n_unique_full < min_n_unique_full or len_n_unique_train < min_n_unique_train or len_n_unique_test < min_n_unique_test :
try:
full['new_' + feat][full['new_' + feat] == n_unique_full] = n_unique_list_full[i-1]
train_df['new_' + feat][train_df['new_' + feat] == n_unique_full] = n_unique_list_full[i-1]
true_test_df['new_' + feat][true_test_df['new_' + feat] == n_unique_full] = n_unique_list_full[i-1]
test_df['new_' + feat][test_df['new_' + feat] == n_unique_full] = n_unique_list_full[i-1]
except:
continue
count+=1
print('Done!' )<define_variables>
RFC = RandomForestClassifier(n_estimators=500,max_depth=9,min_samples_split=3)
 RFC.fit(X_train,y_train )
for n in [2, 53, 81, 111, 121, 126, 130, 146]:
print('Variable', 'var_' + str(n))
plt.figure(figsize=(15,8))
count = 1
for n_unique in list(set(train_df['new_var_' + str(n)])) [:6]:
var_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
var_tar_0
samples_0 = len(var_tar_0)
 samples_1 = len(var_tar_1)
 if samples_0 < 20 or samples_1 < 20:
continue
samples_percentage = np.round(( samples_0 + samples_1)*100/ 200000)
 plt.subplot(2, 3, count)
 sns.kdeplot(var_tar_0, shade=False, color="red", label = 'target = 0')
 sns.kdeplot(var_tar_1, shade=False, color="blue", label = 'target = 1')
 plt.title('Count = {} represents {}% of the data'.format(n_unique, samples_percentage))
plt.xlabel('Feature Values')
 plt.ylabel('Probability')
 count += 1
plt.tight_layout()
plt.show() 
<define_variables>
y_pred = pd.DataFrame(RFC.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_rf = y_pred
y_pred.to_csv('titanic_pred_rfc.csv',index=False )
for n in [117, 120]:
print('Variable', 'var_' + str(n))
plt.figure(figsize=(15,8))
count = 1
for n_unique in list(set(train_df['new_var_' + str(n)])) [:6]:
var_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
var_tar_0
samples_0 = len(var_tar_0)
 samples_1 = len(var_tar_1)
 if samples_0 < 20 or samples_1 < 20:
continue
samples_percentage = np.round(( samples_0 + samples_1)*100/ 200000)
 plt.subplot(2, 3, count)
 sns.kdeplot(var_tar_0, shade=False, color="red", label = 'target = 0')
 sns.kdeplot(var_tar_1, shade=False, color="blue", label = 'target = 1')
 plt.title('Count = {} represents {}% of the data'.format(n_unique, samples_percentage))
plt.xlabel('Feature Values')
 plt.ylabel('Probability')
 count += 1
plt.tight_layout()
plt.show()<filter>
lgb = LGBMClassifier(learning_rate=0.01,max_depth=5,n_estimators=500,num_leaves=3 ).fit(X_train,y_train )
n_unique = 1
n = 126
var_126_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_126_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
n = 81
var_81_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_81_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
fig = plt.subplots()
sns.kdeplot(var_126_tar_0, shade=False, color="r", label = 'var_126_tar_0')
 sns.kdeplot(var_126_tar_1, shade=False, color="blue", label = 'var_126_tar_1')
 sns.kdeplot(var_81_tar_0, shade=False, color="orange", label = 'var_81_tar_0')
 sns.kdeplot(var_81_tar_1, shade=False, color="black", label = 'var_81_tar_1')
 plt.title('PDFs of VAR 126 and 81 BEFORE transformation')
 plt.xlabel('Feature Values')
 plt.ylabel('Probability')
 plt.show()
n_unique = 1
n = 126
var_126_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)] * 2.486 -18.5
var_126_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)] * 2.486 - 18.5
n = 81
var_81_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_81_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
fig = plt.subplots()
sns.kdeplot(var_126_tar_0, shade=False, color="r", label = 'var_126_tar_0')
 sns.kdeplot(var_126_tar_1, shade=False, color="blue", label = 'var_126_tar_1')
 sns.kdeplot(var_81_tar_0, shade=False, color="orange", label = 'var_81_tar_0')
 sns.kdeplot(var_81_tar_1, shade=False, color="black", label = 'var_81_tar_1')
 plt.title('PDFs of VAR 126 and 81 AFTER transformation')
 plt.xlabel('Feature Values')
 plt.ylabel('Probability')
 plt.show()<init_hyperparams>
y_pred = pd.DataFrame(lgb.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_lgb = y_pred
y_pred.to_csv('titanic_pred_lgb.csv',index=False )
param = {'bagging_fraction': 0.5166,
'bagging_freq': 3,
'lambda_l1': 3.968,
'lambda_l2': 1.263,
'learning_rate': 0.00141,
'max_depth': 3,
'min_data_in_leaf': 17,
'min_gain_to_split': 0.2525,
'min_sum_hessian_in_leaf': 19.55,
'num_leaves': 20,
'feature_fraction': 1,
'save_binary': True,
'seed': 2319,
'feature_fraction_seed': 2319,
'bagging_seed': 2319,
'drop_seed': 2319,
'data_random_seed': 2319,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbosity': -1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': 'false',
'num_threads': 6}
folds = StratifiedKFold(n_splits=4, shuffle=False, random_state=2319)
 target = train_df['target']
y_hat = np.zeros([200000, 200])
 test_hat = np.zeros([200000, 200])
 i = 0
for feature in ['var_' + str(x)for x in range(200)]:
print(feature)
 feat_choices = [feature, 'new_' + feature]
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df[feat_choices].values, target.values)) :
trn_data = lgb.Dataset(train_df.iloc[trn_idx][feat_choices], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][feat_choices], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, 126, valid_sets = [trn_data, val_data], verbose_eval=-1)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][feat_choices], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_df[feat_choices], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)))
 y_hat[:, i] = oof
test_hat[:, i] = predictions
i += 1
weights = []
for col in range(200):
if roc_auc_score(target, y_hat[:,col])>= 0.5:
weights.append(roc_auc_score(target, y_hat[:,col]))
else:
weights.append(0)
 weights = np.array(weights)
 weights =(weights - weights.mean())/ weights.mean()
weights += 1
sub_preds =(y_hat*weights ).sum(axis=1)/200
print('Your CV score is', roc_auc_score(target, sub_preds))
sub_preds_test =(test_hat*weights ).sum(axis=1)/200
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = sub_preds_test
sub.to_csv('submission.csv', index=False )<set_options>
print("XGB train score: ",round(XGB.score(X_train,y_train),2), " XGB test score: ",round(XGB.score(X_test,y_test),2))
print("Log-Reg.train score: ",round(logmodel.score(X_train,y_train),2)," Log-Reg.test score: ",round(logmodel.score(X_test,y_test),2))
print("Random Forest's train score: ",round(RFC.score(X_train,y_train),2), " Random Forest test score: ",round(RFC.score(X_test,y_test),2))
print("LGBM train score: ",round(lgb.score(X_train,y_train),2), " LGB Model test score: ",round(lgb.score(X_test,y_test),2))
warnings.filterwarnings('ignore')
 PATH=".. /input/"
N_SPLITS = 10
SEED_SKF = 4221<split>
y_valid_xgb = XGB.predict(X_test)
 y_valid_log = logmodel.predict(X_test)
 y_valid_rfc = RFC.predict(X_test)
 y_valid_lgb = lgb.predict(X_test )
def merge_train_test(df_train, df_test):
if "target" not in df_test.columns.values:
df_test["target"] = -1
res = pd.concat([df_train, df_test])
 res.reset_index(inplace=True, drop=True)
 return res
def split_train_test(df):
df_train = df[df["target"] >= 0]
df_test = df[df["target"] <= -1]
df_train.reset_index(inplace=True, drop=True)
 df_test.reset_index(inplace=True, drop=True)
 assert list(df_train["ID_code"].values)== [f"train_{i}" for i in range(200000)]
assert list(df_test["ID_code"].values)== [f"test_{i}" for i in range(200000)]
return df_train, df_test<load_from_csv>
fpr_xgb, tpr_xgb, thresholds_xgb = roc_curve(y_test, y_valid_xgb)
 roc_auc_xgb = auc(fpr_xgb, tpr_xgb)
 fpr_log, tpr_log, thresholds_log = roc_curve(y_test, y_valid_log)
 roc_auc_log = auc(fpr_log, tpr_log)
 fpr_rfc, tpr_rfc, thresholds_rfc = roc_curve(y_test, y_valid_rfc)
 roc_auc_rfc = auc(fpr_rfc, tpr_rfc)
 fpr_lgb, tpr_lgb, thresholds_lgb = roc_curve(y_test, y_valid_lgb)
 roc_auc_lgb = auc(fpr_lgb, tpr_lgb )
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )<categorify>
y_pred_final = y_pred
y_pred_final['Survived'] = round(0.25 * y_pred_lgb['Survived']
+ 0.25 * y_pred_rf['Survived']
+ 0.25 * y_pred_xgb['Survived']
+ 0.25 * y_pred_lr['Survived'])
 y_pred_final['PassengerId'] = df_test['PassengerId']
y_pred_final['Survived'] = y_pred_final['Survived'].astype(int)
 y_pred_final.to_csv('titanic_pred_final.csv',index=False )
<categorify><EOS> 
y_pred_final['Survived'].value_counts()