Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
np.random.seed(42 )
nrows=184903891-1
nchunk=12000000
val_size=1000000
frm=nrows-84903891
if debug:
frm=0
nchunk=100000
val_size=10000
to=frm+nchunk
sub=DO(frm,to,FILENO )<define_variables>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv', index_col=0)
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv',index_col=0)
 train_data.head()
FILENO= 24
debug=0 
<merge>
X = train_data.drop(columns=['Survived'])
 y = train_data.Survived
def do_count(df, group_cols, agg_type='uint32', show_max=False, show_agg=True):
agg_name='{}count'.format('_'.join(group_cols))
if show_agg:
print("
Aggregating by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols][group_cols].groupby(group_cols ).size().rename(agg_name ).to_frame().reset_index()
df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<count_unique_values>
train_data[train_data.Age <= 15].Survived.value_counts(normalize=True )
def do_countuniq(df, group_cols, counted, agg_type='uint32', show_max=False, show_agg=True):
agg_name= '{}_by_{}_countuniq'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Counting unqiue ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<groupby>
print(train_data[train_data.Sex == 'male'].Survived.value_counts(normalize=True))
print('-'*40)
 print(train_data[train_data.Sex == 'female'].Survived.value_counts(normalize=True))
def do_cumcount(df, group_cols, counted,agg_type='uint16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_cumcount'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Cumulative count by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()
df[agg_name]=gp.values
del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<categorify>
train_data.groupby('Pclass' ).Survived.value_counts(normalize=True, sort=False )
def do_mean(df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_mean'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Calculating mean of ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<categorify>
ranks = X['Name'].str.extract(r'\b(\w+)\.')
 ranks[0].value_counts()
def do_var(df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_var'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Calculating variance of ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<train_on_grid>
ranks[train_data.Age.isna() ][0].value_counts()
if debug:
print('*** debug parameter set: this is a test run for debugging purposes ***')
 def lgb_modelfit_nocv(params, dtrain, dvalid, predictors, target='target', objective='binary', metrics='auc',
feval=None, early_stopping_rounds=50, num_boost_round=3000, verbose_eval=10, categorical_features=None):
lgb_params = {
'boosting_type': 'gbdt',
'objective': objective,
'metric':metrics,
'learning_rate': 0.05,
'num_leaves': 31,
'max_depth': -1,
'min_child_samples': 20,
'max_bin': 255,
'subsample': 0.6,
'subsample_freq': 0,
'colsample_bytree': 0.3,
'min_child_weight': 5,
'subsample_for_bin': 200000,
'min_split_gain': 0,
'reg_alpha': 0,
'reg_lambda': 0,
'nthread': 8,
'verbose': 0,
}
lgb_params.update(params)
 print("preparing validation datasets")
 xgtrain = lgb.Dataset(dtrain[predictors].values, label=dtrain[target].values,
feature_name=predictors,
categorical_feature=categorical_features 
)
 xgvalid = lgb.Dataset(dvalid[predictors].values, label=dvalid[target].values,
feature_name=predictors,
categorical_feature=categorical_features 
)
 del dtrain
gc.collect()
del dvalid
gc.collect()
evals_results = {}
bst1 = lgb.train(lgb_params,
xgtrain,
valid_sets=[ xgvalid],
valid_names=['valid'],
evals_result=evals_results,
num_boost_round=num_boost_round,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=10,
feval=feval)
 print("
Model Report")
 print("bst1.best_iteration: ", bst1.best_iteration)
 print(metrics+":", evals_results['valid'][metrics][bst1.best_iteration-1])
 return(bst1,bst1.best_iteration )<categorify>
missing_age_ranks = ranks[train_data.Age.isna() ][0].value_counts().index.values
missing_age_ranks
def DO(frm,to,fileno):
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint8',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32',
}
print('loading train data...',frm,to)
 train_df = pd.read_csv(".. /input/train.csv", parse_dates=['click_time'], skiprows=range(1,frm), nrows=to-frm, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])
 print('loading test data...')
 if debug:
test_df = pd.read_csv(".. /input/test.csv", nrows=100000, parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])
 else:
test_df = pd.read_csv(".. /input/test.csv", parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])
 len_train = len(train_df)
 train_df=train_df.append(test_df)
 del test_df
gc.collect()
train_df['hour'] = pd.to_datetime(train_df.click_time ).dt.hour.astype('int8')
 train_df['day'] = pd.to_datetime(train_df.click_time ).dt.day.astype('int8')
 train_df = do_next_Click(train_df,agg_suffix='nextClick', agg_type='float32'); gc.collect()
train_df = do_prev_Click(train_df,agg_suffix='prevClick', agg_type='float32'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'channel'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'device', 'os'], 'app'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'day'], 'hour'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'app'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'app'], 'os'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'device'); gc.collect()
train_df = do_countuniq(train_df, ['app'], 'channel'); gc.collect()
train_df = do_cumcount(train_df, ['ip'], 'os'); gc.collect()
train_df = do_cumcount(train_df, ['ip', 'device', 'os'], 'app'); gc.collect()
train_df = do_count(train_df, ['ip', 'day', 'hour']); gc.collect()
train_df = do_count(train_df, ['ip', 'app']); gc.collect()
train_df = do_count(train_df, ['ip', 'app', 'os']); gc.collect()
train_df = do_var(train_df, ['ip', 'app', 'os'], 'hour'); gc.collect()
del train_df['day']
gc.collect()
gc.collect()
print('
Before appending predictors...
 ',sorted(predictors))
target = 'is_attributed'
word= ['app','device','os', 'channel', 'hour']
for feature in word:
if feature not in predictors:
predictors.append(feature)
 categorical = ['app', 'device', 'os', 'channel', 'hour']
print('
After appending predictors...
 ',sorted(predictors))
test_df = train_df[len_train:]
val_df = train_df[(len_train-val_size):len_train]
train_df = train_df[:(len_train-val_size)]
print("
train size: ", len(train_df))
print("
valid size: ", len(val_df))
print("
test size : ", len(test_df))
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id'].astype('int')
 gc.collect()
print("Training...")
 start_time = time.time()
params = {
'learning_rate': 0.10,
'num_leaves': 7,
'max_depth': 3,
'min_child_samples': 100,
'max_bin': 100,
'subsample': 0.7,
'subsample_freq': 1,
'colsample_bytree': 0.9,
'min_child_weight': 0,
'scale_pos_weight':200
} 
(bst,best_iteration)= lgb_modelfit_nocv(params,
train_df,
val_df,
predictors,
target,
objective='binary',
metrics='auc',
early_stopping_rounds=30,
verbose_eval=True,
num_boost_round=1000,
categorical_features=categorical)
 print('[{}]: model training time'.format(time.time() - start_time))
del train_df
del val_df
gc.collect()
ax = lgb.plot_importance(bst, max_num_features=300)
 plt.savefig('test%d.png'%(fileno), dpi=600, bbox_inches='tight')
 plt.show()
print("Predicting...")
 sub['is_attributed'] = bst.predict(test_df[predictors],num_iteration=best_iteration)
 sub.to_csv('sub_it%d.csv'%(fileno),index=False,float_format='%.9f')
 print("done...")
 return sub<define_variables>
age_na_fills = {}
for rank in missing_age_ranks:
age_na_fills[rank] = round(train_data[(ranks == rank ).values].Age.mean())
 age_na_fills
nrows=184903891-1
nchunk=25000000
val_size=2500000
frm=nrows-85000000
if debug:
frm=0
nchunk=100000
val_size=10000
to=frm+nchunk
sub=DO(frm,to,FILENO )<set_options>
def fill_age_na(data, age_na_fills, ranks):
data['Age'] = data.apply(lambda row: age_na_fills.get(ranks[row.name], 29)if np.isnan(row['Age'])else row['Age'], axis=1 )
%matplotlib inline
pd.pandas.set_option('display.max_columns',None)
<import_modules>
class AgeFillNa(BaseEstimator, TransformerMixin):
def __init__(self, age_na_fills=None):
self.age_na_fills = age_na_fills
def fit(self, X, y=None):
return self
def transform(self, X):
output = X.copy()
ranks = X['Name'].str.extract(r'\b(\w+)\.')
 if self.age_na_fills is not None:
fill_age_na(output, self.age_na_fills, ranks[0])
 else:
output.Age = output.Age.fillna(output.Age.mean())
 return output
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, columns_drop):
self.columns_drop = columns_drop
def fit(self, X, y=None):
return self
def transform(self, X):
return X.drop(columns=self.columns_drop)
 class MultiColumnLabelEncoder(BaseEstimator, TransformerMixin):
def __init__(self, columns=None):
self.columns = columns
def fit(self, X, y=None):
return self
def transform(self, X):
output = X.copy()
if self.columns is not None:
for col in self.columns:
output[col] = LabelEncoder().fit_transform(output[col].astype(str))
else:
for colname, col in output.iteritems() :
output[colname] = LabelEncoder().fit_transform(col)
 return output
columns_to_drop = ['Ticket', 'Cabin', 'Name']
columns_to_encode = ['Sex', 'Embarked']
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error, mean_squared_log_error, mean_absolute_error
from mlxtend.regressor import StackingCVRegressor
from xgboost import XGBRegressor 
<define_search_model>
prepare_pipeline = Pipeline([ 
('age_fill_na', AgeFillNa(age_na_fills)) , 
('selector', DataFrameSelector(columns_to_drop)) , 
('label_enc', MultiColumnLabelEncoder(columns=columns_to_encode)) , 
('imputer', SimpleImputer(strategy="mean")) , 
('std_scaler', StandardScaler()),
] )
elasticnet_alphas = [5e-5, 1e-4, 5e-4, 1e-3]
elasticnet_l1ratios = [0.8, 0.85, 0.9, 0.95, 1]
lasso_alphas = [5e-5, 1e-4, 5e-4, 1e-3]
ridge_alphas = [13.5, 14, 14.5, 15, 15.5]
MODELS = {
"elasticnet" : make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=elasticnet_alphas, l1_ratio=elasticnet_l1ratios)) ,
"lasso" : make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=lasso_alphas, random_state=42)) ,
"ridge" : make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas)) ,
"gradb" : GradientBoostingRegressor(n_estimators=6000, learning_rate=0.01,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=42),
"svr" : make_pipeline(RobustScaler() ,
SVR(C=20, epsilon=0.008, gamma=0.0003)) ,
"xgboost" : XGBRegressor(learning_rate=0.01, n_estimators=6000,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:squarederror', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006, random_state=42)}
MODELS_stack = StackingCVRegressor(regressors=(MODELS['elasticnet'], MODELS['gradb'], MODELS['lasso'],
MODELS['ridge'], MODELS['svr'], MODELS['xgboost']),
meta_regressor=MODELS['xgboost'],
use_features_in_secondary=True )<feature_engineering>
X = prepare_pipeline.fit_transform(train_data.drop(columns=['Survived']))
def feature_engineering(df):
temporal_features = [feature for feature in df.columns if 'Yr' in feature or 'Year' in feature or 'Mo' in feature]
numeric_features = [feature for feature in df.columns if df[feature].dtype != 'O' and feature not in temporal_features and feature not in("Id", "kfold","SalePrice")]
categorical_features = [feature for feature in df.columns if df[feature].dtype == 'O' and feature not in temporal_features]
for feature in temporal_features:
if feature == 'YrSold' or feature == 'MoSold':
pass
else:
df[feature] = df['YrSold'] - df[feature]
df['YrSold'] = df['YrSold'].astype(str)
 df['MoSold'] = df['MoSold'].astype(str)
 df['MSSubClass'] = df['MSSubClass'].apply(str)
 for feature in numeric_features:
df[feature] = df.groupby("Neighborhood")[feature].transform(lambda x: x.fillna(x.median()))
for feature in categorical_features:
df[feature] = df[feature].fillna("Missing")
 for feature in temporal_features:
if feature == 'YrSold' or feature == 'MoSold':
df[feature] = df[feature].fillna("Missing")
 else:
df[feature] = df[feature].fillna(0)
 df['TotalHouseSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalLot'] = df['LotFrontage'] + df['LotArea']
df['TotalBsmtFin'] = df['BsmtFinSF1'] + df['BsmtFinSF2']
df['TotalBath'] = df['FullBath'] + df['HalfBath']
df['TotalPorch'] = df['OpenPorchSF'] + df['EnclosedPorch'] + df['ScreenPorch']
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(df[c].values))
df[c] = lbl.transform(list(df[c].values))
df["TotalGarageQual"] = df["GarageQual"] * df["GarageCond"]
df["TotalExteriorQual"] = df["ExterQual"] * df["ExterCond"]
numeric_feats = [feature for feature in df.columns if df[feature].dtype != "object" and feature not in("Id", "kfold","SalePrice")]
skewed_feats = df[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness = skewness[abs(skewness)> 0.75]
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
df[feat] = boxcox1p(df[feat], lam)
 features = [feature for feature in df.columns if df[feature].dtype == 'O']
for feature in features:
abc = df[feature].value_counts().to_dict()
for key, value in abc.items() :
if value/len(df[feature])< 0.01:
df.loc[:,feature][df[feature]==key]="rare"
return df<load_from_csv>
empt_models = [GradientBoostingClassifier() , LogisticRegression(solver='lbfgs'), SGDClassifier() , GaussianNB() ,
KNeighborsClassifier() , DecisionTreeClassifier() , RandomForestClassifier(n_estimators=100),
SVC(gamma='auto')]
for model in empt_models:
results = cross_val_predict(model, X, y, cv=5)
 print("model: {}, result: {:.3f}".format(model.__class__, f1_score(y, results)) )
def run(fold,model):
df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 df["kfold"] = -1
kf = model_selection.StratifiedKFold(n_splits=4, shuffle=True, random_state=42)
 for f,(train_idx, val_idx)in enumerate(kf.split(X=df, y=df.SalePrice.values)) :
df.loc[val_idx, 'kfold'] = f
df = feature_engineering(df)
 df_test = feature_engineering(df_test)
 df_test.fillna(0,inplace=True)
 df.SalePrice = np.log1p(df.SalePrice)
 df = pd.concat([df,df_test],axis=0)
 df = pd.get_dummies(df)
 for feature in df.columns:
all_value_counts = df[feature].value_counts()
zero_value_counts = all_value_counts.iloc[0]
if zero_value_counts / len(df)> 0.99:
df.drop(feature,axis=1,inplace=True)
 df_test = df.loc[df["Id"].between(1461,2919)]
df = df.loc[df["Id"].between(1,1460)]
df_train = df[df.kfold != fold].reset_index(drop=True)
 df_valid = df[df.kfold == fold].reset_index(drop=True)
 numeric_features = [feature for feature in df_train.columns if feature not in("Id", "kfold","SalePrice")]
x_train = df_train[numeric_features].values
x_valid = df_valid[numeric_features].values
reg = MODELS[model]
reg.fit(x_train,df_train.SalePrice.values)
 valid_preds = reg.predict(x_valid)
 test_preds = reg.predict(df_test[numeric_features].values)
 rmse = np.sqrt(mean_squared_error(df_valid.SalePrice.values, valid_preds))
mae = mean_absolute_error(df_valid.SalePrice.values, valid_preds)
 print(f"FOLD={fold}, MODEL = {model}, RMSE = {rmse}, MAE ={mae} ")
 return(test_preds)
<create_dataframe>
from sklearn.model_selection import GridSearchCV
preds_df = pd.DataFrame()
for fold in range(3):
for keys,items in MODELS.items() :
preds_df["fold"+str(fold)+keys] = run(fold,keys)
<load_from_csv>
from sklearn.model_selection import GridSearchCV
def run(fold,model):
df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 df["kfold"] = -1
kf = model_selection.StratifiedKFold(n_splits=4, shuffle=True, random_state=42)
 for f,(train_idx, val_idx)in enumerate(kf.split(X=df, y=df.SalePrice.values)) :
df.loc[val_idx, 'kfold'] = f
df = feature_engineering(df)
 df_test = feature_engineering(df_test)
 df_test.fillna(0,inplace=True)
 df.SalePrice = np.log1p(df.SalePrice)
 df = pd.concat([df,df_test],axis=0)
 df = pd.get_dummies(df)
 for feature in df.columns:
all_value_counts = df[feature].value_counts()
zero_value_counts = all_value_counts.iloc[0]
if zero_value_counts / len(df)> 0.99:
df.drop(feature,axis=1,inplace=True)
 df_test = df.loc[df["Id"].between(1461,2919)]
df = df.loc[df["Id"].between(1,1460)]
df_train = df[df.kfold != fold].reset_index(drop=True)
 df_valid = df[df.kfold == fold].reset_index(drop=True)
 numeric_features = [feature for feature in df_train.columns if feature not in("Id", "kfold","SalePrice")]
x_train = df_train[numeric_features].values
x_valid = df_valid[numeric_features].values
reg = MODELS_stack
reg.fit(x_train,df_train.SalePrice.values)
 valid_preds = reg.predict(x_valid)
 test_preds = reg.predict(df_test[numeric_features].values)
 rmse = np.sqrt(mean_squared_error(df_valid.SalePrice.values, valid_preds))
mae = mean_absolute_error(df_valid.SalePrice.values, valid_preds)
 print(f"FOLD={fold}, MODEL = {model}, RMSE = {rmse}, MAE ={mae} ")
 return(test_preds)
 for fold in range(3):
preds_df["fold"+str(fold)+"stack"] = run(fold,keys )<feature_engineering>
params = [{'kernel':['linear'], 'gamma':[i / 10 for i in range(1, 100, 5)], 'C':[i / 10 for i in range(5, 30, 5)]},
{'kernel':['rbf'], 'C':[i / 10 for i in range(5, 30, 5)]},
{'kernel':['poly'], 'degree':np.arange(2, 10), 'C':[i / 10 for i in range(5, 30, 5)]}]
grid_search_svc = GridSearchCV(SVC(gamma='auto'), params, cv=5, scoring='neg_mean_squared_error')
 grid_search_svc.fit(X, y)
 grid_search_svc.best_params_
for cols in preds_df.columns:
preds_df[cols] = np.expm1(preds_df[cols])
<feature_engineering>
svc_model = SVC(gamma='auto', **grid_search_svc.best_params_)
 svc_score = cross_val_predict(svc_model, X, y, cv=5)
 f1_score(y, svc_score )
for e,col in enumerate(preds_df.columns):
if e == 0:
preds_df['SalePrice'] = preds_df[col]
elif col in ['fold0stack','fold1stack','fold2stack']:
preds_df['SalePrice'] += preds_df[col]*4
else:
preds_df['SalePrice'] += preds_df[col] 
<feature_engineering>
params = {'learning_rate':[i/10 for i in range(3, 6)], 'n_estimators':np.arange(128, 132, 1),
'min_samples_split':np.arange(2, 3), 'min_samples_leaf':np.arange(6, 8),
'max_depth':np.arange(1, 3), 'max_features':np.arange(1, 8)}
grid_search_gbc = GridSearchCV(GradientBoostingClassifier() , params, cv=5, scoring='neg_mean_squared_error')
 grid_search_gbc.fit(X, y)
 grid_search_gbc.best_params_
preds_df['SalePrice'] = preds_df['SalePrice']/30<load_from_csv>
gbc_model = GradientBoostingClassifier(**grid_search_gbc.best_params_)
 gbc_score = cross_val_predict(gbc_model, X, y, cv=5)
 f1_score(y, gbc_score )
df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv',usecols=["Id"] )<concatenate>
params = {'n_neighbors':np.arange(10, 25), 'p':np.arange(1, 6)}
grid_search_knn = GridSearchCV(KNeighborsClassifier() , params, cv=5, scoring='neg_mean_squared_error')
 grid_search_knn.fit(X, y)
 grid_search_knn.best_params_
preds_df["id"] = df_test.values.flatten()<prepare_output>
knn_model = KNeighborsClassifier(**grid_search_knn.best_params_)
 knn_score = cross_val_predict(knn_model, X, y, cv=5)
 f1_score(y, knn_score )
submission = preds_df[['id','SalePrice']]<save_to_csv>
params = {'n_estimators':np.arange(50, 1000, 100), 'max_depth':np.arange(3, 8)}
grid_search_rfc = GridSearchCV(RandomForestClassifier(random_state=42), params, cv=5, scoring='neg_mean_squared_error')
 grid_search_rfc.fit(X, y)
 grid_search_rfc.best_params_
submission.to_csv(".. /.. /kaggle/working/submission.csv", index=False )<set_options>
rfc_model = RandomForestClassifier(random_state=42,**grid_search_rfc.best_params_)
 rfc_score = cross_val_predict(rfc_model, X, y, cv=5)
 f1_score(y, rfc_score )
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.simplefilter("ignore", category=ConvergenceWarning)
 pd.pandas.set_option('display.max_columns', None)
 pd.set_option('display.float_format', lambda x: '%.3f' % x )<load_from_csv>
X_test = prepare_pipeline.fit_transform(test_data )
def data() :
train_ = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test_ = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 dataframe = pd.concat([train_, test_], ignore_index=True)
 return dataframe, train_, test_
df, train, test = data()
df.head()<count_missing_values>
svc_model.fit(X, y)
 predictions = svc_model.predict(X_test )
<groupby><EOS> 
my_submission = pd.DataFrame({'PassengerId': test_data.index.values, 'Survived': predictions})
 my_submission.to_csv('submission.csv', index=False)