Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<create_dataframe>
%matplotlib inline
oof_preds = np.zeros(train.shape[0])
 sub_preds = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
feats = [f for f in train.columns if f not in ['target']]
for n_fold,(train_idx, valid_idx)in enumerate(kf.split(train[feats], train['target'])) :
print(n_fold)
 trn_data = lgb.Dataset(train.iloc[train_idx][feats], label=train['target'].iloc[train_idx])
 val_data = lgb.Dataset(train.iloc[valid_idx][feats], label=train['target'].iloc[valid_idx])
 clf = lgb.train(param, trn_data,5120, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)
 oof_preds[valid_idx] = clf.predict(train.iloc[valid_idx][feats], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = feats
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 initial_idx = 0
chunk_size = 100000
current_pred = np.zeros(len(test))
while initial_idx < test.shape[0]:
final_idx = min(initial_idx + chunk_size, test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = clf.predict(test.iloc[idx][feats], num_iteration=clf.best_iteration)
 initial_idx = final_idx
sub_preds += current_pred / 5
print('Full AUC score %.6f' % roc_auc_score(train['target'], oof_preds))<prepare_x_and_y>
warnings.filterwarnings('ignore' )
<set_options>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 IDtest = test["PassengerId"]
del train
gc.collect()<import_modules>
dataset = pd.concat(objs=[train, test], axis=0 ).reset_index(drop=True)
 len(dataset )
<save_to_csv>
train_len = len(train )
output_xgb=pd.DataFrame({'ID_code':test.index,'target':sub_preds})
 output_xgb.to_csv(r'predictions.csv',index=False )<set_options>
dataset.isnull().sum()
sns.set(font_scale=1)
 warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings('ignore' )<load_from_csv>
sex_survived = pd.DataFrame(columns=['Total', 'Survived', 'Survived Ratio'])
 sex_survived['Total'] = train['Sex'].value_counts()
sex_survived['Survived'] = train[train['Survived']==1].groupby(['Sex'] ).apply(lambda x: x.shape[0])
 sex_survived['Survived Ratio'] = sex_survived['Survived'] / sex_survived['Total']
sex_survived
random_state = 42
np.random.seed(random_state)
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<normalization>
dataset['Sex'] = dataset['Sex'].map(lambda s: 1 if s == 'male' else 0 )
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<init_hyperparams>
dataset['Age'][dataset['Age'].isnull() == True] = dataset[dataset['Age'].isnull() == False].median() [0]
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 31,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 150,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state}<create_dataframe>
dataset['AgeGroup'] = None
dataset.loc[(( dataset['Sex'] == 1)&(dataset['Age'] <= 15)) , 'AgeGroup'] = 'boy'
dataset.loc[(( dataset['Sex'] == 0)&(dataset['Age'] <= 15)) , 'AgeGroup'] = 'girl'
dataset.loc[(( dataset['Sex'] == 1)&(dataset['Age'] > 15)) , 'AgeGroup'] = 'adult male'
dataset.loc[(( dataset['Sex'] == 0)&(dataset['Age'] > 15)) , 'AgeGroup'] = 'adult female'
dataset['AgeGroup'].value_counts()
skf = StratifiedKFold(n_splits=7, shuffle=True, random_state=random_state)
 oof = train[['ID_code', 'target']]
oof['predict'] = 0
predictions = test[['ID_code']]
val_aucs = []
feature_importance = pd.DataFrame()<prepare_x_and_y>
pd.DataFrame(dataset[['AgeGroup', 'Survived']] 
.groupby(['AgeGroup', 'Survived'])
.apply(lambda x: x.shape[0]),
columns=['Count'] )
features = [col for col in train.columns if col not in ['target', 'ID_code']]
X_test = test[features].values<split>
dataset['AgeGroup'] = dataset['AgeGroup'].map({'boy': 0, 'girl': 1, 'adult male': 2, 'adult female': 3})
 dataset['AgeGroup'] = dataset['AgeGroup'].astype(int )
for fold,(trn_idx, val_idx)in enumerate(skf.split(train, train['target'])) :
X_train, y_train = train.iloc[trn_idx][features], train.iloc[trn_idx]['target']
X_valid, y_valid = train.iloc[val_idx][features], train.iloc[val_idx]['target']
N = 3
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=3000,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance = pd.DataFrame()
fold_importance["feature"] = features
fold_importance["importance"] = lgb_clf.feature_importance()
fold_importance["fold"] = fold + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp/N<save_to_csv>
pd.DataFrame(dataset[['Pclass', 'Survived', 'Sex']] 
.groupby(['Pclass', 'Sex', 'Survived'])
.apply(lambda x: x.shape[0]),
columns=['Count'] )
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub = pd.DataFrame({"ID_code":test["ID_code"].values})
 sub["target"] = predictions['target']
sub.to_csv("lgb_submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )<import_modules>
dataset[dataset['Embarked'].isnull() ]
print(os.listdir(".. /input")) 
<load_from_csv>
embarked_survived = pd.DataFrame(dataset[['Embarked', 'Survived']] 
.groupby(['Embarked', 'Survived'])
.apply(lambda x: x.shape[0]), columns=['count'] ).reset_index()
embarked_survived_pivot = embarked_survived.pivot(index='Embarked', columns='Survived', values='count')
 embarked_survived_pivot
test_data = pd.read_csv(".. /input/test.csv")
 train_data= pd.read_csv(".. /input/train.csv")
 train_data.head()<define_variables>
train['Embarked'].value_counts()
count_0 = len(train_data[train_data["target"] == 0])
 count_1 = len(train_data[train_data["target"] == 1])
 percentage_count_0 =(( count_0)/(count_0+count_1)) * 100
percentage_count_1 = 100-percentage_count_0
print("{}{}{}{}{}".format("Percentage of 0 class is ",percentage_count_0,"
","Percentage of 1 class is ",percentage_count_1)) 
<split>
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} )
labels = train_data["target"]
new_train_data = train_data.drop(["target","ID_code"],axis =1)
 new_train_data.head()
x_train, x_test, y_train, y_test = train_test_split(new_train_data, labels, test_size = 0.25, random_state = 0)
 print(x_train.shape,x_test.shape)
 print(y_train.shape,y_test.shape)
 x_train.head()<choose_model_class>
dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace=True )
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=2319)
 param = {
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.0405,
'learning_rate': 0.083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 4,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
oof = np.zeros(len(train_data))
predictions = np.zeros(len(test_data))
features = [c for c in train_data.columns if c not in ['ID_code', 'target']]
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data.values, labels.values)) :
trn_data = lgb.Dataset(train_data.iloc[trn_idx][features], label=labels.iloc[trn_idx])
 val_data = lgb.Dataset(train_data.iloc[val_idx][features], label=labels.iloc[val_idx])
 clf = lgb.train(param, trn_data, 1000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train_data.iloc[val_idx][features], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_data[features], num_iteration=clf.best_iteration)/ folds.n_splits 
<compute_test_metric>
dataset['Embarked'] = dataset['Embarked'].astype('int' )
target = train_data.iloc[val_idx]['target']
print("
>> CV score: {:<8.5f}".format(roc_auc_score(target, oof[val_idx])))
<save_to_csv>
dataset['Embarked'].isnull().sum()
ID_code = test_data["ID_code"]
submission = pd.DataFrame({'ID_code' : ID_code,
'target' : predictions})
 submission.to_csv('./version1.csv', index=False)
 sub = pd.read_csv('./version1.csv')
 sub.head()<install_modules>
dataset['Embarked'].value_counts()
!pip install target_encoding 
<categorify>
dataset[dataset['Fare'].isnull() ]
X, y = load_breast_cancer(return_X_y=True)
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 enc = TargetEncoder()
new_X_train = enc.transform_train(X=X_train, y=y_train)
 new_X_test = enc.transform_test(X_test)
 rf = RandomForestClassifier(n_estimators=100, random_state=42)
 rf.fit(X_train, y_train)
 pred = rf.predict_proba(X_test)[:,1]
print('without target encoding', roc_auc_score(y_test, pred))
rf.fit(new_X_train, y_train)
 pred = rf.predict_proba(new_X_test)[:,1]
print('with target encoding', roc_auc_score(y_test, pred))
enc = TargetEncoderClassifier()
enc.fit(X_train, y_train)
 pred = enc.predict_proba(X_test)[:,1]
print('target encoding classifier', roc_auc_score(y_test, pred))<load_from_csv>
old_man_nan_fare_idx = dataset[dataset['Fare'].isnull() ].index
old_man_nan_fare_idx
train=pd.read_csv(".. /input/train.csv" ).drop("ID_code",axis=1)
 test=pd.read_csv(".. /input/test.csv" ).drop("ID_code",axis=1)
 X = train.drop('target', axis=1)
 y = train.target
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42 )<compute_train_metric>
dataset[dataset['Pclass'] == 3][dataset['Cabin'].isnull() ][dataset['Age'] < 20]['Fare'].mean()
enc = TargetEncoderClassifier(alpha=100, max_unique=25, used_features=170)
 score = cross_val_score(enc, X, y, scoring='roc_auc', cv=cv)
 print(score.mean() , score.std() )<categorify>
mean_old_pclass3 = dataset[dataset['Pclass'] == 3][dataset['Cabin'].isnull() ][dataset['Age'] > 50]['Fare'].mean()
mean_old_pclass3
enc = TargetEncoderClassifier(alpha=100, max_unique=25, used_features=170)
 enc.fit(X, y)
 pred = enc.predict_proba(test)[:,1]<save_to_csv>
dataset.loc[old_man_nan_fare_idx, 'Fare'] = mean_old_pclass3
sample_submission['target'] = pred
sample_submission.to_csv('submission.csv', index=False )<load_from_csv>
dataset.iloc[old_man_nan_fare_idx]
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv")
 sample_data = pd.read_csv(".. /input/sample_submission.csv" )<prepare_x_and_y>
np.exp(2.3), np.exp(2.9), np.exp(3.5 )
oof = train_data[["ID_code","target"]]
oof['predict'] = 0
prediction = test_data['ID_code']
label_df = train_data['target']<choose_model_class>
dataset.loc[ dataset['Fare'] < 9.974, 'FareBin'] = 0
dataset.loc[(dataset['Fare'] >= 9.974)&(dataset['Fare'] <= 18.174), 'FareBin'] = 1
dataset.loc[(dataset['Fare'] > 18.174)&(dataset['Fare'] <= 33.115), 'FareBin'] = 2
dataset.loc[ dataset['Fare'] > 33.115, 'FareBin'] = 3
dataset['FareBin'] = dataset['FareBin'].astype(int )
skf_three= StratifiedKFold(n_splits=15, shuffle=True, random_state=2319 )<init_hyperparams>
dataset['Cabin'].value_counts()
random_state = 42
np.random.seed(random_state)
 params = {
"objective" : "binary", "metric" : "auc", "boosting": 'gbdt', "max_depth" : -1, "num_leaves" : 13,
"learning_rate" : 0.01, "bagging_freq": 0.5, "bagging_fraction" : 0.4, "feature_fraction" : 0.05,
"min_data_in_leaf": 80, "min_sum_heassian_in_leaf": 10, 'num_leaves': 13,
'num_threads': 8,"tree_learner": "serial", "boost_from_average": "false",
"bagging_seed" : random_state, "verbosity" : 1, "seed": random_state
}<init_hyperparams>
dataset['CabinType'] = dataset['Cabin'].apply(lambda x: str(x)[0].upper() if type(x)== str else 'None' )
random_state = 42
np.random.seed(random_state)
 lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 13,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 80,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state
}<categorify>
dataset[dataset['CabinType'] == 'T']
random_state = 42
np.random.seed(random_state)
 def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<count_duplicates>
t_idx = dataset[dataset['CabinType'] == 'T'].index
dataset.loc[t_idx, 'CabinType'] = 'None'
def augmen(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs); xn = np.vstack(xn)
 ys = np.ones(xs.shape[0]);yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn]); y = np.concatenate([y,ys,yn])
 return x,y<prepare_x_and_y>
dataset.iloc[t_idx]
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)
 oof = train[['ID_code', 'target']]
oof['predict'] = 0
predictions = test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()
features = [col for col in train.columns if col not in ['target', 'ID_code']]
X_test = test[features].values
for fold,(trn_idx, val_idx)in enumerate(skf.split(train, train['target'])) :
X_train, y_train = train.iloc[trn_idx][features], train.iloc[trn_idx]['target']
X_valid, y_valid = train.iloc[val_idx][features], train.iloc[val_idx]['target']
N = 5
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=3000,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp/N<save_to_csv>
survival_rate_with_cabin = 100 * train[train['Cabin'].isnull() == False]['Survived'].value_counts() [1] / \
train[train['Cabin'].isnull() == False]['Survived'].shape[0]
survival_rate_without_cabin = 100 * train[train['Cabin'].isnull() == True]['Survived'].value_counts() [1] / \
train[train['Cabin'].isnull() == True]['Survived'].shape[0]
print(f'Survival rate with cabin: {survival_rate_with_cabin:.1f}%')
 print(f'Survival rate without cabin: {survival_rate_without_cabin:.1f}%' )
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_submission.csv", index=False)
 submission3 = pd.DataFrame({"ID_code":ID_code,"target":predictions['fold4']*0.5+predictions['fold5']*0.5})
 submission3.to_csv("lgb_submission3.csv", index=False )<prepare_x_and_y>
cabin_mapping = {"None": 0, "A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7}
dataset['CabinType'] = dataset['CabinType'].map(cabin_mapping )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)
 val_aucs = []
features = [col for col in train_data.columns if col not in ['target', 'ID_code']]
X_test = test_data[features].values
for fold,(trn_idx, val_idx)in enumerate(skf.split(train_data, label_df)) :
X_train, y_train = train_data.iloc[trn_idx][features], label_df.iloc[trn_idx]
X_valid, y_valid = train_data.iloc[val_idx][features], label_df.iloc[val_idx]
N = 3
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augmen(X_train.values, y_train.values)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,trn_data,100000,valid_sets = [trn_data, val_data],early_stopping_rounds=1000,verbose_eval = 5000,evals_result=evals_result)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 prediction['fold{}'.format(fold+1)] = yp/N<save_to_csv>
dataset['CabinType'].value_counts()
submission = pd.DataFrame({"ID_code":ID_code,"target":yp/N})
 submission.to_csv("lgb_submission.csv", index=False)
 submission1 = pd.DataFrame({"ID_code":ID_code,"target":yp/N*0.4+predictions['target']*0.6})
 submission1.to_csv("lgb_submission1.csv", index=False)
 submission2 = pd.DataFrame({"ID_code":ID_code,"target":yp/N*0.25+predictions['target']*0.25+predictions['fold4']*0.25+predictions['fold5']*0.25})
 submission2.to_csv("lgb_submission2.csv", index=False )<load_from_csv>
def get_title(name):
title_search = re.search(r'([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ''
dataset['Title'] = dataset['Name'].apply(get_title )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<load_from_csv>
dataset['Title'].value_counts()
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<merge>
def replace_titles(title):
if title in ['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona']:
return 'Rare'
elif title in ['Countess', 'Mme']:
return 'Mrs'
elif title in ['Mlle', 'Ms']:
return 'Miss'
elif title =='Dr':
if x['Sex']=='Male':
return 'Mr'
else:
return 'Mrs'
else:
return title
dataset['Title'] = dataset['Title'].apply(replace_titles )
def transform(df, var='var_12'):
df['random_{}'.format(var)] = np.random.normal(df[var].mean() , df[var].std() , 200000 ).round(4)
 var_counts = pd.DataFrame(df.groupby(var)['ID_code'].count() ).reset_index()
var_counts_random = pd.DataFrame(df.groupby('random_{}'.format(var)) ['ID_code'].count() ).reset_index()
merged_counts = pd.merge(var_counts, var_counts_random, left_on=var, right_on='random_{}'.format(var))
merged_counts['diff'] = merged_counts['ID_code_x'] - merged_counts['ID_code_y']
df['{}_diff_normal_dist'.format(var)] = df.merge(merged_counts[[var,'diff']], how='left')['diff']
df = df.drop('random_{}'.format(var), axis=1)
 return df<categorify>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
dataset['Title'] = dataset['Title'].map(title_mapping )
for var in tqdm(['var_{}'.format(x)for x in range(0, 200)]):
train_df = transform(train_df, var=var)
 test_df = transform(test_df, var=var )<choose_model_class>
dataset['Noble'] = dataset['Name'].apply(lambda x: 1 if re.search(r'\ (.*?\)', x)else 0 )
random_state = 42
params = {
"objective" : "binary", "metric" : "auc", "boosting": 'gbdt', "max_depth" : -1, "num_leaves" : 13,
"learning_rate" : 0.01, "bagging_freq": 5, "bagging_fraction" : 0.4, "feature_fraction" : 0.05,
"min_data_in_leaf": 80, "min_sum_heassian_in_leaf": 10, "tree_learner": "serial", "boost_from_average": "false",
"bagging_seed" : random_state, "verbosity" : 1, "seed": random_state
}
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)
 oof = train_df[['ID_code', 'target']].copy()
oof['predict'] = 0
predictions = test_df[['ID_code']].copy()
val_aucs = []
features = [col for col in train_df.columns if col not in ['target', 'ID_code']]
X_test = test_df[features].values
for fold,(trn_idx, val_idx)in enumerate(skf.split(train_df, train_df['target'])) :
X_train, y_train = train_df.iloc[trn_idx][features], train_df.iloc[trn_idx]['target']
X_valid, y_valid = train_df.iloc[val_idx][features], train_df.iloc[val_idx]['target']
N = 3
p_valid,yp = 0,0
trn_data = lgb.Dataset(X_train, label=y_train)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=1000,
verbose_eval = 500,
evals_result=evals_result)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 oof['predict'][val_idx] = p_valid
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.9f, std: %.9f.All auc: %.9f." %(mean_auc, std_auc, all_auc))
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 submission = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 submission["target"] = predictions['target']
submission.to_csv("lgb_submission2.csv", index=False )<load_from_csv>
dataset[:train_len][['Noble', 'Survived']].groupby(['Noble'] ).sum()
df=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv" )<filter>
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
df.skew().index[df.skew().values>1]<filter>
dataset.loc[ dataset['FamilySize'] == 1, 'FamilySizeBin'] = 0
dataset.loc[(dataset['FamilySize'] >= 2)&(dataset['FamilySize'] <= 3), 'FamilySizeBin'] = 1
dataset.loc[(dataset['FamilySize'] == 4), 'FamilySizeBin'] = 2
dataset.loc[(dataset['FamilySize'] >= 5)&(dataset['FamilySize'] <= 7), 'FamilySizeBin'] = 3
dataset.loc[(dataset['FamilySize'] >=8), 'FamilySizeBin'] = 4
outlier_cols=df.skew().index[df.skew().values>1]<count_missing_values>
dataset['FamilySizeBin'] = dataset['FamilySizeBin'].astype('int' )
df.isnull().sum().index[df.isnull().sum().values>0]<create_dataframe>
dataset['IsAlone'] = dataset['FamilySize'].map(lambda s: 1 if s == 1 else 0 )
df_copy=df.copy()<data_type_conversions>
dataset['LastName'] = dataset.Name.str.split(',' ).str[0]
cat_columns=df_copy.select_dtypes(include=['O','object'] ).columns
for cols in cat_columns:
df_copy[cols].fillna(df_copy[cols].mode() [0],inplace=True )<data_type_conversions>
le = LabelEncoder()
dataset['LastName'] = le.fit_transform(dataset['LastName'])
 dataset['LastName'] = dataset['LastName'].astype(int )
num_columns=df_copy.select_dtypes(exclude=['O','object'] ).columns
for cols in num_columns:
if cols in outlier_cols:
df_copy[cols].fillna(df_copy[cols].median() ,inplace=True)
 else:
df_copy[cols].fillna(df_copy[cols].mean() ,inplace=True )<count_missing_values>
dataset.drop(labels=['Fare', 'Cabin', 'FamilySize', 'PassengerId', 'Ticket', 'Age', 'Name'],
axis=1, inplace=True )
df_copy.isnull().sum().index[df_copy.isnull().sum().values>0]<set_options>
dataset = pd.get_dummies(dataset, columns = ["Embarked"])
 dataset = pd.get_dummies(dataset, columns = ["Parch"])
 dataset = pd.get_dummies(dataset, columns = ["Pclass"])
 dataset = pd.get_dummies(dataset, columns = ["CabinType"])
 dataset = pd.get_dummies(dataset, columns = ["Title"])
 dataset = pd.get_dummies(dataset, columns = ["FamilySizeBin"])
 dataset = pd.get_dummies(dataset, columns = ["FareBin"])
 dataset = pd.get_dummies(dataset, columns = ["AgeGroup"] )
fig=px.box(df_copy['LotArea'])
 fig.show("notebook" )<sort_values>
train = dataset[:len(train)]
test = dataset[len(train):]
test.drop(labels=["Survived"],axis = 1,inplace=True )
df_copy.corr() ['SalePrice'].sort_values()<prepare_x_and_y>
X_train = train.drop(labels = ["Survived"],axis = 1)
 y_train = train["Survived"].astype(int )
X=df_copy.copy()
X.drop(["SalePrice","Id"],axis=1,inplace=True)
 y=df_copy['SalePrice']<compute_test_metric>
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import accuracy_score
y = np.log1p(y )<define_variables>
gbdt = GradientBoostingClassifier(learning_rate=0.02, min_samples_split=6, min_samples_leaf=4)
 cross_val_score(gbdt, X_train, y_train, cv=10 ).mean()
num_train_columns=['MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual',
'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd',
'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
'MiscVal', 'MoSold', 'YrSold']<normalization>
model = GradientBoostingClassifier(learning_rate=0.02, min_samples_split=6, min_samples_leaf=4 )
qt=QuantileTransformer(output_distribution='normal',random_state=0)
 X_num_transformed=qt.fit_transform(X[num_train_columns])
<create_dataframe>
model.fit(X_train, y_train )
df_num_transformed=pd.DataFrame(X_num_transformed.reshape(-1,36),columns=X[num_train_columns].columns )<normalization>
model.fit(X_train, y_train )
rs=RobustScaler()
X_num_transformed=rs.fit_transform(df_num_transformed )<create_dataframe>
test_Survived = pd.Series(model.predict(test), name="Survived")
 results = pd.concat([IDtest, test_Survived], axis=1)
 results.to_csv("titanic_with_ensemble.csv",index=False )
df_num_transformed=pd.DataFrame(X_num_transformed.reshape(-1,36),columns=X[num_train_columns].columns )<concatenate>
dataset.iloc[old_man_nan_fare_idx]
X_transformed= pd.concat([df_num_transformed,X[cat_columns]],axis=1 )<categorify>
model.predict(dataset.iloc[old_man_nan_fare_idx].drop(['Survived'], axis=1))
<set_options>
dataset.loc[train_len:, 'Survived'] = model.predict(dataset[train_len:].drop(['Survived'], axis=1))
<import_modules><EOS> 
survived_corr = pd.DataFrame(dataset.corr() ['Survived'].drop('Survived'))
survived_corr_most = survived_corr[ 
(survived_corr['Survived'] > 0.1)|(survived_corr['Survived'] < -0.1)]\ 
.sort_values(['Survived'], ascending=True )