Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<create_dataframe>
train=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train
def QDA_train(train_qda, test_qda):
oof = np.zeros(len(train_qda))
preds = np.zeros(len(test_qda))
for i in tqdm_notebook(range(512)) :
train2 = train_qda[train_qda['wheezy-copper-turtle-magic']==i]
test2 = test_qda[test_qda['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=31, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train_qda['target'], oof)
 print(f'AUC: {auc:.5}')
 return oof, preds<train_model>
k=train.isnull().sum()
k[k>0].sort_values(ascending=False)
oof_qda, preds_qda = QDA_train(train, test )<define_variables>
condition=condition.rename(columns={"SalePrice": "avg_sale_price_cond","Utilities": "total_records"})
 condition.sort_values(by='total_records',ascending=False,inplace=True)
 condition
LOW = 0.45
HIGH = 1 - LOW<filter>
def get_denoise_data(train_off, oof, low, high):
print(len(train_off[(oof > low)&(oof < high)]))
train_off_filter = train_off[(oof <= low)|(oof >= high)]
train_off_filter.reset_index(drop=True, inplace=True)
 return train_off_filter<create_dataframe>
train_outlier = train.copy()
oof_outlier = oof_qda.copy()<train_model>
NUM_DENOISE = 2
for i in range(NUM_DENOISE):
train_outlier = get_denoise_data(train_outlier, oof_outlier, LOW, HIGH)
 oof_outlier, preds_outlier = QDA_train(train_outlier, test )<categorify>
neighborhood_md.loc[neighborhood_md['SalePrice']>=5000,'neighborhood_flag']=0
neighborhood_md.loc[neighborhood_md['SalePrice']>=150000,'neighborhood_flag']=1
neighborhood_md.loc[neighborhood_md['SalePrice']>=200000,'neighborhood_flag']=2
neighborhood_md.loc[neighborhood_md['SalePrice']>=250000,'neighborhood_flag']=3
del neighborhood_md['SalePrice']
def Pseudo_train(preds, oof, train_pse, test_pse):
for itr in range(2):
test_pse['target'] = preds
test_pse.loc[test_pse['target'] > 0.955, 'target'] = 1
test_pse.loc[test_pse['target'] < 0.045, 'target'] = 0
usefull_test = test_pse[(test_pse['target'] == 1)|(test_pse['target'] == 0)]
new_train = pd.concat([train_pse, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.995, 'target'] = 1
new_train.loc[oof < 0.005, 'target'] = 0
oof = np.zeros(len(train_pse))
preds = np.zeros(len(test_pse))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train_pse[train_pse['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=31, random_state=time.time)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train_pse['target'], oof)
 print(f'AUC: {auc:.5}')
 return oof, preds<train_model>
neighborhood_md.columns=['Neighborhood','neighborhood_flag']
train=pd.merge(train,neighborhood_md,left_on='Neighborhood',right_on='Neighborhood')
 test=pd.merge(test,neighborhood_md,left_on='Neighborhood',right_on='Neighborhood')
oof_pse, preds_pse = Pseudo_train(preds_outlier, oof_outlier, train_outlier, test )<data_type_conversions>
train['tot_bath']=train['BsmtFullBath']+0.5*train['BsmtHalfBath']+train['FullBath']+0.5*train['HalfBath']
train['bsmt_bath']=train['BsmtFullBath']+0.5*train['BsmtHalfBath']
train['bed_bath_kitch']=train['tot_bath']+train['BedroomAbvGr']+train['KitchenAbvGr']
train['area_floors']=train['1stFlrSF']+train['2ndFlrSF']+train['BsmtFinSF1']+train['BsmtFinSF2']
train['bsmt_by_total']=(train['BsmtFinSF1']+train['BsmtFinSF2'])/(train['area_floors'])
 train['unf_bsmt']=(train['BsmtUnfSF']/train['TotalBsmtSF'] ).fillna(0)
 train["unf_bsmt"].replace([np.inf, -np.inf], 0)
 train['porch_area_tot']=train['OpenPorchSF']+train['EnclosedPorch']+train['3SsnPorch']+train['ScreenPorch']
train['wood_deck_porch']=(train['WoodDeckSF']/train['porch_area_tot'] ).replace(np.inf, 0)
 train['sale_built_yr']=train['YrSold']-train['YearBuilt']
train['remod_built_yr']=train['YearRemodAdd']-train['YearBuilt']
train['new_flag']=0
train.loc[train['YrSold']-train['YearBuilt'],'new_flag']=1
train['remod_flag']=0
train.loc[train['remod_built_yr']>=2,'remod_flag']=1
train['floor_by_lot']=train['area_floors']/train['LotArea']
train['garage_area_per_car']=(train['GarageArea']/train['GarageCars'] ).fillna(0)
 train["garage_area_per_car"]=train["garage_area_per_car"].replace([np.inf, -np.inf], 0)
 test['tot_bath']=test['BsmtFullBath']+0.5*test['BsmtHalfBath']+test['FullBath']+0.5*test['HalfBath']
test['bsmt_bath']=test['BsmtFullBath']+0.5*test['BsmtHalfBath']
test['bed_bath_kitch']=test['tot_bath']+test['BedroomAbvGr']+test['KitchenAbvGr']
test['area_floors']=test['1stFlrSF']+test['2ndFlrSF']+test['BsmtFinSF1']+test['BsmtFinSF2']
test['bsmt_by_total']=(test['BsmtFinSF1']+test['BsmtFinSF2'])/(test['area_floors'])
 test['unf_bsmt']=(test['BsmtUnfSF']/test['TotalBsmtSF'] ).fillna(0)
 test["unf_bsmt"].replace([np.inf, -np.inf], 0)
 test['porch_area_tot']=test['OpenPorchSF']+test['EnclosedPorch']+test['3SsnPorch']+test['ScreenPorch']
test['wood_deck_porch']=(test['WoodDeckSF']/test['porch_area_tot'] ).replace(np.inf, 0)
 test['sale_built_yr']=test['YrSold']-test['YearBuilt']
test['remod_built_yr']=test['YearRemodAdd']-test['YearBuilt']
test['new_flag']=0
test.loc[test['YrSold']-test['YearBuilt']==0,'new_flag']=1
test['remod_flag']=0
test.loc[test['remod_built_yr']>=2,'remod_flag']=1
test['floor_by_lot']=test['area_floors']/test['LotArea']
test['garage_area_per_car']=(test['GarageArea']/test['GarageCars'] ).fillna(0)
 test["garage_area_per_car"]=test["garage_area_per_car"].replace([np.inf, -np.inf], 0 )
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps<train_model>
mapping = { "NA" : 0,"Po" : 1,"Fa":2,"TA" : 3,"Gd":4,"Ex":5 }
mapping_shape = { "Reg" : 0,"IR1" : 1,"IR2":2,"IR3" : 3}
mapping_contour = { "Bnk" : 0,"Lvl" : 1,"Low":2,"HLS" : 3}
mapping_ms_zoning={"FV":4,"RL":3,"RH":2,"RM":1,"C(all)":0,"NA":-1}
mapping_paved_Drive={"Y":2,"P":1,"N":0}
mapping_utilities={"AllPub":1,"NoSeWa":0,"NoSeWr":0,"ELO":0,"NA":0}
mapping_functional={"Typ":4,"Min1":3,"Min2":3,"Mod":2,"Maj1":1,"Maj2":1,"Sev":0,"Sal":0,"NA":0}
l_col=['BsmtCond','BsmtQual','ExterQual','ExterCond','HeatingQC','GarageQual','GarageCond','FireplaceQu','KitchenQual']
l_col_zon=['MSZoning']
l_col_pav=['PavedDrive']
l_col_shape=['LotShape']
l_col_util=['Utilities']
l_col_con=['LandContour']
l_col_fun=['Functional']
def mapping_var(mapping,df,varlist):
for i in range(len(varlist)) :
varname=varlist[i]
varname_o=varname+'_o'
df[varname]=df[varname].fillna('NA')
 df[varname_o] =df[varname].apply(lambda x : mapping[x])
 return df
train=mapping_var(mapping,train,l_col)
 train=mapping_var(mapping_ms_zoning,train,l_col_zon)
 train=mapping_var(mapping_paved_Drive,train,l_col_pav)
 train=mapping_var(mapping_shape,train,l_col_shape)
 train=mapping_var(mapping_utilities,train,l_col_util)
 train=mapping_var(mapping_contour,train,l_col_con)
 train=mapping_var(mapping_functional,train,l_col_fun)
 test=mapping_var(mapping,test,l_col)
 test=mapping_var(mapping_ms_zoning,test,l_col_zon)
 test=mapping_var(mapping_paved_Drive,test,l_col_pav)
 test=mapping_var(mapping_shape,test,l_col_shape)
 test=mapping_var(mapping_utilities,test,l_col_util)
 test=mapping_var(mapping_contour,test,l_col_con)
 test=mapping_var(mapping_functional,test,l_col_fun)
 labelencoder = LabelEncoder()
train.loc[:, 'Street'] = labelencoder.fit_transform(train.loc[:, 'Street'])
 train.loc[:, 'CentralAir'] = labelencoder.fit_transform(train.loc[:, 'CentralAir'])
 test.loc[:, 'Street'] = labelencoder.fit_transform(test.loc[:, 'Street'])
 test.loc[:, 'CentralAir'] = labelencoder.fit_transform(test.loc[:, 'CentralAir'] )
def GMM_train(train_GMM, test_GMM):
cols = [c for c in train_GMM.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof_GMM = np.zeros(len(train_GMM))
preds_GMM = np.zeros(len(test_GMM))
for i in tqdm_notebook(range(512)) :
train2 = train_GMM[train_GMM['wheezy-copper-turtle-magic']==i]
test2 = test_GMM[test_GMM['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gm = GaussianMixture(n_components=2, init_params='random', covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([train3,test3],axis = 0))
oof_GMM[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:,0]
preds_GMM[idx2] += gm.predict_proba(test3)[:,0] / skf.n_splits
auc = roc_auc_score(train_GMM['target'],oof_GMM)
 print('QDA scores CV =',round(auc,5))
return oof_GMM, preds_GMM<train_on_grid>
train['tot_quality']=train['BsmtQual_o']+train['ExterQual_o']+train['KitchenQual_o']+train['GarageQual_o']+train['FireplaceQu_o']
test['tot_quality']=test['BsmtQual_o']+test['ExterQual_o']+test['KitchenQual_o']+test['GarageQual_o']+test['FireplaceQu_o']
oof_GMM, preds_GMM = GMM_train(train_outlier, test )<prepare_output>
k=corr.unstack().sort_values().drop_duplicates()
k[(k>0.8)|(k<-0.8)]
preds = preds_pse * 0.8 + preds_GMM * 0.2<load_from_csv>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds_pse<save_to_csv>
test.sort_values(by='Id',inplace=True)
 test
train.sort_values(by='Id',inplace=True)
 salePrice=train['SalePrice']
del train['logSalePrice']
del train['SalePrice']
sub.to_csv('submission.csv',index=False )<import_modules>
train['flag']='train'
test['flag']='test'
all_data = pd.concat([train, test], ignore_index=True, sort=False)
np.random.seed(2018)
<define_variables>
numeric_feats = all_data.dtypes[train.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.5]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats])
embed_size = 300
max_features = 95000
maxlen = 70<define_variables>
numcols=all_data._get_numeric_data().columns
all_data_num=all_data.loc[:,numcols]
k=all_data_num.columns.to_series() [np.isinf(all_data_num ).any() ]
all_data[k] = all_data[k].replace(-np.inf, np.nan )
def show_timer() :
time_steps.append(time.time())
 last_step = time_steps[-1] - time_steps[-2]
total = time_steps[-1] - time_steps[0]
print('Last step time: {:.1f}(s)- Total time: {:.1f}(s)'.format(last_step, total))<categorify>
all_data_mod=all_data[['LotArea','GrLivArea','TotRmsAbvGrd','OverallQual','OverallCond','YearBuilt','YearRemodAdd','TotalBsmtSF','BsmtQual_o','ExterQual_o','GarageCars','1stFlrSF','2ndFlrSF','FullBath','FireplaceQu_o','KitchenQual_o','MSZoning_o','PavedDrive_o','LotShape_o', 'Utilities_o', 'LandContour_o', 'Functional_o','tot_bath','bed_bath_kitch','area_floors','GarageArea','Fireplaces','BedroomAbvGr','tot_quality','sale_built_yr','remod_built_yr','MSSubClass','neighborhood_flag','MasVnrArea','garage_area_per_car','floor_by_lot','remod_flag','new_flag','unf_bsmt','bsmt_by_total','bsmt_bath','flag','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','BsmtUnfSF']]
all_data_mod
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<categorify>
all_data_mod=all_data[['LotArea','GrLivArea','TotRmsAbvGrd','OverallQual','OverallCond','YearBuilt','YearRemodAdd','TotalBsmtSF','BsmtQual_o','ExterQual_o','GarageCars','1stFlrSF','2ndFlrSF','FullBath','FireplaceQu_o','KitchenQual_o','MSZoning_o','PavedDrive_o','LotShape_o', 'Utilities_o', 'LandContour_o', 'Functional_o','tot_bath','bed_bath_kitch','area_floors','GarageArea','Fireplaces','BedroomAbvGr','tot_quality','sale_built_yr','remod_built_yr','MSSubClass','neighborhood_flag','MasVnrArea','floor_by_lot','remod_flag','new_flag','unf_bsmt','bsmt_by_total','bsmt_bath','flag','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF']]
train_mod=all_data_mod.loc[all_data_mod['flag']=='train',:]
test_mod=all_data_mod.loc[all_data_mod['flag']=='test',:]
del train_mod['flag']
del test_mod['flag']
my_imputer = SimpleImputer(missing_values=np.nan,strategy='mean')
 train_imp = my_imputer.fit_transform(train_mod)
 test_imp = my_imputer.fit_transform(test_mod )
def pre_process(x):
x = clean_text(x)
 x = clean_numbers(x)
 x = replace_typical_misspell(x)
 return x<feature_engineering>
X=train_imp
y=np.log1p(salePrice)
 X_train,X_test,Y_train,Y_Test=train_test_split(X, y,test_size=0.3,random_state=1)
 regressor = RandomForestRegressor(n_estimators=100, random_state=1,max_depth=10,max_features=10,min_samples_leaf=5)
 feature_list=train_mod.columns
model=regressor.fit(X_train,Y_train)
 pred_rf=model.predict(X_test)
 cv_results = cross_validate(regressor, X_train, Y_train, cv=5,scoring='r2')
 sorted(cv_results.keys())
 scoreOfModel = model.score(X_test, Y_Test)
 print("RSquared value for Model",scoreOfModel)
 feat_importances = pd.Series(model.feature_importances_, index=feature_list)
 feat_importances=feat_importances.sort_values()
feat_importances.plot(kind='barh',figsize=(12,9))
print("RMSLE for Model",np.sqrt(mean_squared_log_error(np.exp(Y_Test),np.exp(pred_rf))))
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: pre_process(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: pre_process(x))
train_X_all = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X_all))
train_X_all = tokenizer.texts_to_sequences(train_X_all)
 test_X = tokenizer.texts_to_sequences(test_X)
 word_index = tokenizer.word_index
train_X_all = pad_sequences(train_X_all, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y_all = train_df['target'].values
show_timer()<categorify>
pred=model.predict(test_imp)
 print(len(pred))
Id=pd.Series(range(1461,2920))
d = {'Id': Id, 'SalePrice': pred}
df = pd.DataFrame(data=d)
 df['SalePrice']=np.exp(df['SalePrice'])
 df
df.to_csv('submission.csv',index=False )
def load_and_prec(trn_idx, val_idx):
trn_idx = np.random.permutation(trn_idx)
 val_idx = np.random.permutation(val_idx)
 train_X = train_X_all[trn_idx]
val_X = train_X_all[val_idx]
train_y = train_y_all[trn_idx]
val_y = train_y_all[val_idx]
return train_X, val_X, test_X, train_y, val_y<statistical_test>
param_grid = {
'bootstrap': [True],
'max_depth': [5, 3,7,10,15],
'max_features': [5, 12,10,8],
'min_samples_split': [5, 10, 8,3],
'n_estimators': [100, 200,150,300]
}
rf = RandomForestRegressor()
grid_search_rf = GridSearchCV(estimator = rf, param_grid = param_grid,
cv = 5, n_jobs = -1, verbose = 2)
 grid_search_rf.fit(X_train,Y_train)
 grid_search_rf.best_params_
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<choose_model_class>
def evaluate(model, test_features, test_labels):
predictions = model.predict(test_features)
 errors = abs(predictions - test_labels)
 mape = 100 * np.mean(errors / test_labels)
 accuracy = 100 - mape
print('Model Performance')
 print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
 print('Accuracy = {:0.2f}%.'.format(accuracy))
return accuracy
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
best_grid_rf = grid_search_rf.best_estimator_
pred_gs_rf=best_grid_rf.predict(X_test)
 scoreOfModel = best_grid_rf.score(X_test, Y_Test)
 print("Rqusared for the Model",scoreOfModel)
 pred_gs_rf=best_grid_rf.predict(X_test)
 print("RMSLE for the Model",np.sqrt(mean_squared_log_error(np.exp(Y_Test),np.exp(pred_gs_rf))))
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<find_best_params>
xgb = xgboost.XGBRegressor(n_estimators=100, learning_rate=0.08, gamma=0, subsample=0.75,
colsample_bytree=1, max_depth=7)
 xgb.fit(X_train,Y_train )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score<feature_engineering>
param_grid = {
'max_depth': [8,10,5],
'learning_rate': [0.05, 0.1, 0.3],
'colsample_bytree': [0.5, 0.7],
'lambda':[0.75,1],
'min_child_weight':[2,4,6],
'subsample': [0.5, 0.7, 0.8],
'n_estimators': [150, 200,250,300]
}
xg = xgboost.XGBRegressor()
grid_search_xg = GridSearchCV(estimator = xg, param_grid = param_grid,
cv = 3, n_jobs = -1, verbose = 2)
 grid_search_xg.fit(X_train,Y_train)
 best_grid_xg=grid_search_xg.best_params_
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix)
 show_timer()<define_variables>
best_grid_xg=grid_search_xg.best_params_
outputs = []<split>
best_grid_xg = grid_search_xg.best_estimator_
pred_gs_xg = best_grid_xg.predict(X_test)
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_gs_xg))))
print(explained_variance_score(pred_gs_xg,Y_Test))
nsplits = 5
nfiles = np.arange(len(train_df))
kfold = StratifiedKFold(nsplits, shuffle=True ).split(train_X_all, train_y_all )<predict_on_test>
lasso=Lasso(max_iter=10000)
 parameters={'alpha': [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5]}
lasso_regressor=GridSearchCV(lasso,parameters,scoring='r2',cv=5)
 lasso_regressor.fit(X_train,Y_train)
 best_grid_lasso=lasso_regressor.best_params_
print(lasso_regressor.best_params_)
 print("R Squared value for the model",lasso_regressor.best_score_ )
idx = 0
for train_idx, val_idx in kfold:
train_X, val_X, test_X, train_y, val_y = load_and_prec(train_idx, val_idx)
 pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, f'2 LSTM w/ attention fold id {idx}'])
 idx += 1<save_to_csv>
lasso_model=Lasso(alpha=0.0001,max_iter=10000 ).fit(X_train,Y_train)
 pred_gs_lasso=lasso_model.predict(X_test)
 print("RMSLE for the model",np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_gs_lasso))))
best_thresh = 0.32
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y2 =(pred_test_y > best_thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y2
out_df.to_csv("submission.csv", index=False)
 show_timer()<define_variables>
ridge=Ridge(max_iter=10000)
 parameters={'alpha': [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5]}
ridge_regressor=GridSearchCV(ridge,parameters,scoring='r2',cv=5)
 ridge_regressor.fit(X_train,Y_train)
 best_grid_ridge=ridge_regressor.best_params_
print(ridge_regressor.best_params_)
 print("R Squared value for the model",ridge_regressor.best_score_)
 ridge_model=Ridge(alpha=0.3,max_iter=10000 ).fit(X_train,Y_train)
 pred_gs_ridge=ridge_model.predict(X_test)
 print("RMSLE for the model",np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_gs_lasso))))
coef = pd.Series(ridge_model.coef_, index = train_mod.columns)
 print(coef )
embed_size = 300
max_features = 95000
maxlen = 70<import_modules>
pred_rf=best_grid_rf.predict(X_test)
 pred_xg=best_grid_xg.predict(X_test)
 pred_ls=lasso_model.predict(X_test)
 pred_rg=ridge_model.predict(X_test)
 pred_xg_ls_rg=(pred_xg+pred_ls+pred_rg)/3
print('RMSLE for Random Forest')
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_rf))))
print('RMSLE for XG Boost')
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_xg))))
print('RMSLE for Lasso')
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_ls))))
print('RMSLE for Xgboost & Lasso & Ridge Blended')
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_xg_ls_rg))))
<load_from_csv><EOS> 
pred_test_rf=best_grid_rf.predict(test_imp)
 pred_test_xg=best_grid_xg.predict(test_imp)
 pred_test_ls=lasso_model.predict(test_imp)
 pred_test_rg=lasso_model.predict(test_imp)
 pred=(pred_test_ls+pred_test_xg+pred_test_rg)/3
print(len(pred))
Id=pd.Series(range(1461,2920))
d = {'Id': Id, 'SalePrice': pred}
df = pd.DataFrame(data=d)
 df['SalePrice']=np.exp(df['SalePrice'])
 df
df.to_csv('submission.csv',index=False )