Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<save_to_csv>
warnings.filterwarnings('ignore')
 %matplotlib inline
plt.style.use('ggplot' )
submission = pd.DataFrame(test_pred, columns=label_mapper.keys())
 submission['id'] = testdex
submission = submission[sub_cols]
submission.to_csv('submission_bert.csv', index=False)
 print(submission.shape )<train_model>
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import RobustScaler, StandardScaler
from sklearn.metrics import mean_squared_error
from sklearn.pipeline import Pipeline, make_pipeline
from scipy.stats import skew
from sklearn.decomposition import PCA, KernelPCA
from sklearn.preprocessing import Imputer
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))<set_options>
from sklearn.model_selection import cross_val_score, GridSearchCV, KFold
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor
from sklearn.svm import SVR, LinearSVR
from sklearn.linear_model import ElasticNet, SGDRegressor, BayesianRidge
from sklearn.kernel_ridge import KernelRidge
from xgboost import XGBRegressor
%matplotlib inline 
<load_from_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print(train_df.shape, test_df.shape )<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def clean_text(text):

 text = text.lower()
text = re.sub(r'[^\w\s]', '', text)
 text = re.sub(r'\_', '', text)
 return text<feature_engineering>
train.drop(train[(train["GrLivArea"]>4000)&(train["SalePrice"]<300000)].index,inplace=True )
train_df['text'] = train_df['text'].map(lambda x: clean_text(x))
train_df['text'] = train_df['text'].map(lambda x: x.strip().split())
 train_df.head()<feature_engineering>
full=pd.concat([train,test], ignore_index=True )
test_df['text'] = test_df['text'].map(lambda x: clean_text(x))
test_df['text'] = test_df['text'].map(lambda x: x.strip().split())
 test_df.head()<define_variables>
aa = full.isnull().sum()
aa[aa>0].sort_values(ascending=False )
data = []
for i in range(len(train_df)) :
data.append(train_df['text'][i])
 for j in range(len(test_df)) :
data.append(test_df['text'][j] )<choose_model_class>
full.groupby(['Neighborhood'])[['LotFrontage']].agg(['mean','median','count'] )
embedding = gensim.models.Word2Vec(data, size=50, window=10, min_count=1, sg=0 )<train_model>
full["LotAreaCut"] = pd.qcut(full.LotArea,10 )
embedding.train(data,total_examples=len(data),epochs=30 )<define_variables>
full.groupby(['LotAreaCut'])[['LotFrontage']].agg(['mean','median','count'] )
words = list(embedding.wv.vocab)
 print(len(words))<load_pretrained>
full['LotFrontage']=full.groupby(['LotAreaCut','Neighborhood'])['LotFrontage'].transform(lambda x: x.fillna(x.median()))
print(embedding['capered'] )<categorify>
full['LotFrontage']=full.groupby(['LotAreaCut'])['LotFrontage'].transform(lambda x: x.fillna(x.median()))
train_df['author'] = pd.Categorical(train_df['author'])
 df_Dummies = pd.get_dummies(train_df['author'], prefix='author')
 train_df = pd.concat([train_df, df_Dummies], axis=1)
 train_df.head()<prepare_x_and_y>
cols=["MasVnrArea", "BsmtUnfSF", "TotalBsmtSF", "GarageCars", "BsmtFinSF2", "BsmtFinSF1", "GarageArea"]
for col in cols:
full[col].fillna(0, inplace=True )
X = train_df['text'].str[:100]
Y = train_df[['author_EAP', 'author_HPL', 'author_MWS']].values
print(X.shape, X[0], Y.shape, Y[0] )<categorify>
cols1 = ["PoolQC" , "MiscFeature", "Alley", "Fence", "FireplaceQu", "GarageQual", "GarageCond", "GarageFinish", "GarageYrBlt", "GarageType", "BsmtExposure", "BsmtCond", "BsmtQual", "BsmtFinType2", "BsmtFinType1", "MasVnrType"]
for col in cols1:
full[col].fillna("None", inplace=True )
def text_to_avg(text):

 avg = np.zeros(( 50,))
for w in text:
avg += embedding[w]
avg = avg/len(text)
 return avg<normalization>
cols2 = ["MSZoning", "BsmtFullBath", "BsmtHalfBath", "Utilities", "Functional", "Electrical", "KitchenQual", "SaleType","Exterior1st", "Exterior2nd"]
for col in cols2:
full[col].fillna(full[col].mode() [0], inplace=True )
X_avg = np.zeros(( X.shape[0], 50))
for i in range(X.shape[0]):
X_avg[i] = text_to_avg(X[i] )<normalization>
full.isnull().sum() [full.isnull().sum() >0]
X_test_avg = np.zeros(( X_test.shape[0], 50))
for i in range(X_test.shape[0]):
X_test_avg[i] = text_to_avg(X_test[i] )<split>
NumStr = ["MSSubClass","BsmtFullBath","BsmtHalfBath","HalfBath","BedroomAbvGr","KitchenAbvGr","MoSold","YrSold","YearBuilt","YearRemodAdd","LowQualFinSF","GarageYrBlt"]
for col in NumStr:
full[col]=full[col].astype(str )
X_train, X_dev, Y_train, Y_dev = train_test_split(X_avg, Y, test_size=0.2, random_state=123)
 print(X_train.shape, Y_train.shape, X_dev.shape, Y_dev.shape )<choose_model_class>
full.groupby(['MSSubClass'])[['SalePrice']].agg(['mean','median','count'] )
model = models.Sequential()
model.add(layers.Dense(512, activation='relu', input_shape=(50,)))
 model.add(layers.Dense(3, activation='softmax'))
model.summary()<choose_model_class>
def map_values() :
full["oMSSubClass"] = full.MSSubClass.map({'180':1,
'30':2, '45':2,
'190':3, '50':3, '90':3,
'85':4, '40':4, '160':4,
'70':5, '20':5, '75':5, '80':5, '150':5,
'120': 6, '60':6})
 full["oMSZoning"] = full.MSZoning.map({'C(all)':1, 'RH':2, 'RM':2, 'RL':3, 'FV':4})
 full["oNeighborhood"] = full.Neighborhood.map({'MeadowV':1,
'IDOTRR':2, 'BrDale':2,
'OldTown':3, 'Edwards':3, 'BrkSide':3,
'Sawyer':4, 'Blueste':4, 'SWISU':4, 'NAmes':4,
'NPkVill':5, 'Mitchel':5,
'SawyerW':6, 'Gilbert':6, 'NWAmes':6,
'Blmngtn':7, 'CollgCr':7, 'ClearCr':7, 'Crawfor':7,
'Veenker':8, 'Somerst':8, 'Timber':8,
'StoneBr':9,
'NoRidge':10, 'NridgHt':10})
 full["oCondition1"] = full.Condition1.map({'Artery':1,
'Feedr':2, 'RRAe':2,
'Norm':3, 'RRAn':3,
'PosN':4, 'RRNe':4,
'PosA':5 ,'RRNn':5})
 full["oBldgType"] = full.BldgType.map({'2fmCon':1, 'Duplex':1, 'Twnhs':1, '1Fam':2, 'TwnhsE':2})
 full["oHouseStyle"] = full.HouseStyle.map({'1.5Unf':1,
'1.5Fin':2, '2.5Unf':2, 'SFoyer':2,
'1Story':3, 'SLvl':3,
'2Story':4, '2.5Fin':4})
 full["oExterior1st"] = full.Exterior1st.map({'BrkComm':1,
'AsphShn':2, 'CBlock':2, 'AsbShng':2,
'WdShing':3, 'Wd Sdng':3, 'MetalSd':3, 'Stucco':3, 'HdBoard':3,
'BrkFace':4, 'Plywood':4,
'VinylSd':5,
'CemntBd':6,
'Stone':7, 'ImStucc':7})
 full["oMasVnrType"] = full.MasVnrType.map({'BrkCmn':1, 'None':1, 'BrkFace':2, 'Stone':3})
 full["oExterQual"] = full.ExterQual.map({'Fa':1, 'TA':2, 'Gd':3, 'Ex':4})
 full["oFoundation"] = full.Foundation.map({'Slab':1,
'BrkTil':2, 'CBlock':2, 'Stone':2,
'Wood':3, 'PConc':4})
 full["oBsmtQual"] = full.BsmtQual.map({'Fa':2, 'None':1, 'TA':3, 'Gd':4, 'Ex':5})
 full["oBsmtExposure"] = full.BsmtExposure.map({'None':1, 'No':2, 'Av':3, 'Mn':3, 'Gd':4})
 full["oHeating"] = full.Heating.map({'Floor':1, 'Grav':1, 'Wall':2, 'OthW':3, 'GasW':4, 'GasA':5})
 full["oHeatingQC"] = full.HeatingQC.map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 full["oKitchenQual"] = full.KitchenQual.map({'Fa':1, 'TA':2, 'Gd':3, 'Ex':4})
 full["oFunctional"] = full.Functional.map({'Maj2':1, 'Maj1':2, 'Min1':2, 'Min2':2, 'Mod':2, 'Sev':2, 'Typ':3})
 full["oFireplaceQu"] = full.FireplaceQu.map({'None':1, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 full["oGarageType"] = full.GarageType.map({'CarPort':1, 'None':1,
'Detchd':2,
'2Types':3, 'Basment':3,
'Attchd':4, 'BuiltIn':5})
 full["oGarageFinish"] = full.GarageFinish.map({'None':1, 'Unf':2, 'RFn':3, 'Fin':4})
 full["oPavedDrive"] = full.PavedDrive.map({'N':1, 'P':2, 'Y':3})
 full["oSaleType"] = full.SaleType.map({'COD':1, 'ConLD':1, 'ConLI':1, 'ConLw':1, 'Oth':1, 'WD':1,
'CWD':2, 'Con':3, 'New':3})
 full["oSaleCondition"] = full.SaleCondition.map({'AdjLand':1, 'Abnorml':2, 'Alloca':2, 'Family':2, 'Normal':3, 'Partial':4})
 return "Done!"
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )<train_model>
map_values()
epochs = 50
history = model.fit(X_train, Y_train, epochs=epochs, batch_size=128, validation_data=(X_dev, Y_dev))<choose_model_class>
full.drop("LotAreaCut",axis=1,inplace=True)
 full.drop(['SalePrice'],axis=1,inplace=True )
model = models.Sequential()
model.add(layers.Dense(512, activation='relu', input_shape=(50,)))
 model.add(layers.Dense(3, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )<train_model>
class labelenc(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self,X,y=None):
return self
def transform(self,X):
lab=LabelEncoder()
X["YearBuilt"] = lab.fit_transform(X["YearBuilt"])
 X["YearRemodAdd"] = lab.fit_transform(X["YearRemodAdd"])
 X["GarageYrBlt"] = lab.fit_transform(X["GarageYrBlt"])
 return X
epochs = 10
model.fit(X_avg, Y, epochs=epochs, batch_size=128 )<predict_on_test>
class skew_dummies(BaseEstimator, TransformerMixin):
def __init__(self,skew=0.5):
self.skew = skew
def fit(self,X,y=None):
return self
def transform(self,X):
X_numeric=X.select_dtypes(exclude=["object"])
 skewness = X_numeric.apply(lambda x: skew(x))
skewness_features = skewness[abs(skewness)>= self.skew].index
X[skewness_features] = np.log1p(X[skewness_features])
 X = pd.get_dummies(X)
 return X
preds = model.predict(X_test_avg)
 print(preds.shape)
 print(preds[7] )<define_variables>
pipe = Pipeline([ 
('labenc', labelenc()), 
('skew_dummies', skew_dummies(skew=1)) ,
] )
pred_labels = []
for i in range(len(X_test_avg)) :
pred_label = np.argmax(preds[i])
 pred_labels.append(pred_label )<prepare_output>
full2 = full.copy()
result = pd.DataFrame(preds, columns=['EAP','HPL','MWS'])
 result.insert(0, 'id', test_df['id'])
 result.head()<save_to_csv>
data_pipe = pipe.fit_transform(full2 )
result.to_csv('submission.csv', index=False, float_format='%.20f' )<feature_engineering>
scaler = RobustScaler()
warnings.filterwarnings("ignore")
 def clean_ColText(data, col, stem=True):

 df = data.copy()
table = str.maketrans('', '', string.punctuation)
 df[col] = df[col].map(lambda x: x.translate(table))
df[col] = df[col].map(lambda x: x.lower())
 df[col] = df[col].apply(word_tokenize)
 stop_words = set(stopwords.words('english'))
df[col] = df[col].map(lambda x: [y for y in x if not y in stop_words])
 df[col] = df[col].map(lambda x: [y for y in x if y not in ["’","’","”","“","‘","—"]])
 if stem:
porter = PorterStemmer()
df[col] = df[col].map(lambda x: [porter.stem(y)for y in x])
 return df
return df
pd.read_csv('.. /input/train.csv' ).head(10 )<load_from_csv>
n_train=train.shape[0]
X = data_pipe[:n_train]
test_X = data_pipe[n_train:]
y= train.SalePrice
X_scaled = scaler.fit(X ).transform(X)
 y_log = np.log(train.SalePrice)
 test_X_scaled = scaler.transform(test_X )
sid = SentimentIntensityAnalyzer()
df = pd.read_csv('.. /input/train.csv')
 df.id = df.id.map(lambda x: int(x.replace('id','')))
 df['sent'] = df['text'].map(lambda x: sid.polarity_scores(x)['compound'])
 df = clean_ColText(df, 'text', stem=True)
 df = df.drop('id', axis=1)
 df.head(10 )<prepare_x_and_y>
lasso=Lasso(alpha=0.001)
 lasso.fit(X_scaled,y_log )
df['text'] = df.text.map(lambda x: ' '.join(x))
sent = np.array(df.sent)+ 1
X = df.drop(['author','sent'], axis=1)
 y = df.author
count_vect = CountVectorizer()
tfidf_transformer = TfidfTransformer()
X = count_vect.fit_transform(X.text)
 X = tfidf_transformer.fit_transform(X)
 X = X.toarray()
print(X.shape )<concatenate>
FI_lasso = pd.DataFrame({"Feature Importance":lasso.coef_}, index=data_pipe.columns )
sent = sent.reshape(( sent.shape[0],1))
X = np.hstack(( X, sent))
X = sparse.csr_matrix(X)
 print(X.shape )<compute_train_metric>
class add_feature(BaseEstimator, TransformerMixin):
def __init__(self,additional=1):
self.additional = additional
def fit(self,X,y=None):
return self
def transform(self,X):
if self.additional==1:
X["TotalHouse"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"]
X["TotalArea"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"] + X["GarageArea"]
else:
X["TotalHouse"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"]
X["TotalArea"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"] + X["GarageArea"]
X["+_TotalHouse_OverallQual"] = X["TotalHouse"] * X["OverallQual"]
X["+_GrLivArea_OverallQual"] = X["GrLivArea"] * X["OverallQual"]
X["+_oMSZoning_TotalHouse"] = X["oMSZoning"] * X["TotalHouse"]
X["+_oMSZoning_OverallQual"] = X["oMSZoning"] + X["OverallQual"]
X["+_oMSZoning_YearBuilt"] = X["oMSZoning"] + X["YearBuilt"]
X["+_oNeighborhood_TotalHouse"] = X["oNeighborhood"] * X["TotalHouse"]
X["+_oNeighborhood_OverallQual"] = X["oNeighborhood"] + X["OverallQual"]
X["+_oNeighborhood_YearBuilt"] = X["oNeighborhood"] + X["YearBuilt"]
X["+_BsmtFinSF1_OverallQual"] = X["BsmtFinSF1"] * X["OverallQual"]
X["-_oFunctional_TotalHouse"] = X["oFunctional"] * X["TotalHouse"]
X["-_oFunctional_OverallQual"] = X["oFunctional"] + X["OverallQual"]
X["-_LotArea_OverallQual"] = X["LotArea"] * X["OverallQual"]
X["-_TotalHouse_LotArea"] = X["TotalHouse"] + X["LotArea"]
X["-_oCondition1_TotalHouse"] = X["oCondition1"] * X["TotalHouse"]
X["-_oCondition1_OverallQual"] = X["oCondition1"] + X["OverallQual"]
X["Bsmt"] = X["BsmtFinSF1"] + X["BsmtFinSF2"] + X["BsmtUnfSF"]
X["Rooms"] = X["FullBath"]+X["TotRmsAbvGrd"]
X["PorchArea"] = X["OpenPorchSF"]+X["EnclosedPorch"]+X["3SsnPorch"]+X["ScreenPorch"]
X["TotalPlace"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"] + X["GarageArea"] + X["OpenPorchSF"]+X["EnclosedPorch"]+X["3SsnPorch"]+X["ScreenPorch"]
return X
clf = MultinomialNB(alpha=0.1)
 scores = cross_val_score(clf, X, df.author, cv=5)
 print('accuracy CV:',scores )<train_model>
pipe = Pipeline([ 
('labenc', labelenc()), 
('add_feature', add_feature(additional=2)) , 
('skew_dummies', skew_dummies(skew=1)) ,
] )
final = MultinomialNB(alpha=0.1)
 final.fit(X, df.author)
<feature_engineering>
full_pipe = pipe.fit_transform(full )
df_t = pd.read_csv('.. /input/test.csv')
 df_t['sent'] = df_t['text'].map(lambda x: sid.polarity_scores(x)['compound'])
 df_t = clean_ColText(df_t, 'text', stem=True)
 df_t['text'] = df_t.text.map(lambda x: ' '.join(x))
X_t = df_t.drop(['sent','id'], axis=1 ).text
X_t = count_vect.transform(X_t)
 X_t = tfidf_transformer.transform(X_t)
 X_t = X_t.toarray()
sent_t = np.array(df_t.sent)+ 1
sent_t = sent_t.reshape(( sent_t.shape[0],1))
print(X_t.shape, sent_t.shape)
 X_t = np.hstack(( X_t, sent_t))
X_t = sparse.csr_matrix(X_t)
 print(X_t.shape )<predict_on_test>
n_train=train.shape[0]
X = full_pipe[:n_train]
test_X = full_pipe[n_train:]
y= train.SalePrice
X_scaled = scaler.fit(X ).transform(X)
 y_log = np.log(train.SalePrice)
 test_X_scaled = scaler.transform(test_X )
preds = final.predict_proba(X_t)
 df_t['EAP'] = preds[:,0]
df_t['HPL'] = preds[:,1]
df_t['MWS'] = preds[:,2]
df_t.head()<save_to_csv>
pca = PCA(n_components=410 )
df_t = df_t.drop(['text', 'sent'], axis=1)
 df_t.to_csv('results.csv', index=False )<define_variables>
X_scaled=pca.fit_transform(X_scaled)
 test_X_scaled = pca.transform(test_X_scaled )
glove_file = '.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt'<load_from_csv>
def rmse_cv(model,X,y):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=5))
return rmse
df_train = pd.read_csv('.. /input/spooky-author-identification/train.csv')
 df_test = pd.read_csv('.. /input/spooky-author-identification/test.csv')
 df_sample = pd.read_csv('.. /input/spooky-author-identification/sample_submission.csv')
 df_sample.head()<string_transform>
models = [LinearRegression() ,Ridge() ,Lasso(alpha=0.01,max_iter=10000),RandomForestRegressor() ,GradientBoostingRegressor() ,SVR() ,LinearSVR() ,
ElasticNet(alpha=0.001,max_iter=10000),SGDRegressor(max_iter=1000,tol=1e-3),BayesianRidge() ,KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5),
ExtraTreesRegressor() ,XGBRegressor() ]
stopWords = set(stopwords.words('english'))<feature_engineering>
names = ["LR", "Ridge", "Lasso", "RF", "GBR", "SVR", "LinSVR", "Ela","SGD","Bay","Ker","Extra","Xgb"]
for name, model in zip(names, models):
score = rmse_cv(model, X_scaled, y_log)
 print("{}: {:.6f}, {:.4f}".format(name,score.mean() ,score.std()))
def processing(df):
df['processed'] = df['text'].apply(lambda x: re.sub(r'[^\w\s]','', x.lower()))
df['length'] = df['processed'].apply(lambda x: len(x))
df['words'] = df['processed'].apply(lambda x: len(x.split(' ')))
 df['words_not_stopword'] = df['processed'].apply(lambda x: len([t for t in x.split(' ')if t not in stopWords]))
df["num_unique_words"] = df["text"].apply(lambda x: len(set(str(x ).split())))
 df["num_chars"] = df["text"].apply(lambda x: len(str(x)))
 df["num_words_upper"] = df["text"].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
df["num_words_title"] = df["text"].apply(lambda x: len([w for w in str(x ).split() if w.istitle() ]))
df['avg_word_length'] = df['processed'].apply(lambda x: np.mean([len(t)for t in x.split(' ')if t not in stopWords])if len([len(t)for t in x.split(' ')if t not in stopWords])> 0 else 0)
 df['commas'] = df['text'].apply(lambda x: x.count(','))
return(df)
 df_train = processing(df_train)
 df_train.head()<split>
class grid() :
def __init__(self,model):
self.model = model
def grid_get(self,X,y,param_grid):
grid_search = GridSearchCV(self.model,param_grid,cv=5, scoring="neg_mean_squared_error")
 grid_search.fit(X,y)
 print(grid_search.best_params_, np.sqrt(-grid_search.best_score_))
grid_search.cv_results_['mean_test_score'] = np.sqrt(-grid_search.cv_results_['mean_test_score'])
 print(pd.DataFrame(grid_search.cv_results_)[['params','mean_test_score','std_test_score']] )
features= [c for c in df_train.columns.values if c not in ['id','text','author']]
numeric_features= [c for c in df_train.columns.values if c not in ['id','text','author','processed']]
target = 'author'
X_train, X_test, y_train, y_test = train_test_split(df_train[features], df_train[target], test_size=0.40, random_state=42)
 X_train.head()<categorify>
class AverageWeight(BaseEstimator, RegressorMixin):
def __init__(self,mod,weight):
self.mod = mod
self.weight = weight
def fit(self,X,y):
self.models_ = [clone(x)for x in self.mod]
for model in self.models_:
model.fit(X,y)
 return self
def predict(self,X):
w = list()
pred = np.array([model.predict(X)for model in self.models_])
 for data in range(pred.shape[1]):
single = [pred[model,data]*weight for model,weight in zip(range(pred.shape[0]),self.weight)]
w.append(np.sum(single))
return w
class TextSelector(BaseEstimator, TransformerMixin):

 def __init__(self, key):
self.key = key
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.key]
class NumberSelector(BaseEstimator, TransformerMixin):

 def __init__(self, key):
self.key = key
def fit(self, X, y=None):
return self
def transform(self, X):
return X[[self.key]]<categorify>
lasso = Lasso(alpha=0.0005,max_iter=10000)
 ridge = Ridge(alpha=60)
 svr = SVR(gamma= 0.0004,kernel='rbf',C=13,epsilon=0.009)
 ker = KernelRidge(alpha=0.2 ,kernel='polynomial',degree=3 , coef0=0.8)
 ela = ElasticNet(alpha=0.005,l1_ratio=0.08,max_iter=10000)
 bay = BayesianRidge()
text = Pipeline([ 
('selector', TextSelector(key='processed')) , 
('tfidf', TfidfVectorizer(stop_words='english'))
])
 text.fit_transform(X_train )<normalization>
w1 = 0.02
w2 = 0.2
w3 = 0.25
w4 = 0.3
w5 = 0.03
w6 = 0.2
length = Pipeline([ 
('selector', NumberSelector(key='length')) , 
('standard', StandardScaler())
 ])
 words = Pipeline([ 
('selector', NumberSelector(key='words')) , 
('standard', StandardScaler())
 ])
 words_not_stopword = Pipeline([ 
('selector', NumberSelector(key='words_not_stopword')) , 
('standard', StandardScaler())
 ])
 num_unique_words = Pipeline([ 
('selector', NumberSelector(key='num_unique_words')) , 
('standard', StandardScaler()),
])
 num_chars = Pipeline([ 
('selector', NumberSelector(key='num_chars')) , 
('standard', StandardScaler()),
])
 num_words_upper = Pipeline([ 
('selector', NumberSelector(key='num_words_upper')) , 
('standard', StandardScaler()),
])
 num_words_title = Pipeline([ 
('selector', NumberSelector(key='num_words_title')) , 
('standard', StandardScaler()),
])
 avg_word_length = Pipeline([ 
('selector', NumberSelector(key='avg_word_length')) , 
('standard', StandardScaler())
 ])
 commas = Pipeline([ 
('selector', NumberSelector(key='commas')) , 
('standard', StandardScaler()),
] )<import_modules>
weight_avg = AverageWeight(mod = [lasso,ridge,svr,ker,ela,bay],weight=[w1,w2,w3,w4,w5,w6] )
feats = FeatureUnion([('text', text), 
('length', length), 
('words', words), 
('num_unique_words', num_unique_words), 
('num_chars', num_chars), 
('num_words_upper', num_words_upper), 
('num_words_title', num_words_title), 
('words_not_stopword', words_not_stopword), 
('avg_word_length', avg_word_length), 
('commas', commas)])
 feature_processing = Pipeline([('feats', feats)])
 feature_processing.fit_transform(X_train )<train_on_grid>
score = rmse_cv(weight_avg,X_scaled,y_log)
 print(score.mean() )
<train_on_grid>
score = rmse_cv(weight_avg,X_scaled,y_log)
 print(score.mean() )
<string_transform>
class stacking(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self,mod,meta_model):
self.mod = mod
self.meta_model = meta_model
self.kf = KFold(n_splits=5, random_state=42, shuffle=True)
 def fit(self,X,y):
self.saved_model = [list() for i in self.mod]
oof_train = np.zeros(( X.shape[0], len(self.mod)))
 for i,model in enumerate(self.mod):
for train_index, val_index in self.kf.split(X,y):
renew_model = clone(model)
 renew_model.fit(X[train_index], y[train_index])
 self.saved_model[i].append(renew_model)
 oof_train[val_index,i] = renew_model.predict(X[val_index])
 self.meta_model.fit(oof_train,y)
 return self
def predict(self,X):
whole_test = np.column_stack([np.column_stack(model.predict(X)for model in single_model ).mean(axis=1)
 for single_model in self.saved_model])
 return self.meta_model.predict(whole_test)
 def get_oof(self,X,y,test_X):
oof = np.zeros(( X.shape[0],len(self.mod)))
 test_single = np.zeros(( test_X.shape[0],5))
test_mean = np.zeros(( test_X.shape[0],len(self.mod)))
 for i,model in enumerate(self.mod):
for j,(train_index,val_index)in enumerate(self.kf.split(X,y)) :
clone_model = clone(model)
 clone_model.fit(X[train_index],y[train_index])
 oof[val_index,i] = clone_model.predict(X[val_index])
 test_single[:,j] = clone_model.predict(test_X)
 test_mean[:,i] = test_single.mean(axis=1)
 return oof, test_mean
train_corpus = []
for i in range(len(df_train)) :
review = re.sub('[^a-zA-Z0-9]', ' ', df_train['text'][i])
 review = review.lower()
review = review.split()
review = [word for word in review if not word in set(stopwords.words('english')) ]
review = ' '.join(review)
 train_corpus.append(review)
 test_corpus = []
for i in range(len(df_test)) :
review = re.sub('[^a-zA-Z0-9]', ' ', df_test['text'][i])
 review = review.lower()
review = review.split()
review = [word for word in review if not word in set(stopwords.words('english')) ]
review = ' '.join(review)
 test_corpus.append(review)
 X_Train = np.array(train_corpus)
 X_Test = np.array(test_corpus)
 y = df_train.iloc[:, 2].values 
<string_transform>
a = Imputer().fit_transform(X_scaled)
 b = Imputer().fit_transform(y_log.values.reshape(-1,1)).ravel()
<train_model>
stack_model = stacking(mod=[lasso,ridge,svr,ker,ela,bay],meta_model=ker )
classifier = Pipeline([('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('clf', MultinomialNB()),
])
 classifier.fit(X_Train, y)
<train_on_grid>
score = rmse_cv(stack_model,a,b)
 print(score.mean() )
parameters = {'vect__ngram_range': [(1, 1),(1, 2),(1, 3)],
'tfidf__use_idf':(True, False),
'clf__alpha':(0.05, 1.0)
 }
gs_clf = GridSearchCV(classifier, parameters)
 gs_clf.fit(X_Train, y)
<find_best_params>
X_train_stack, X_test_stack = stack_model.get_oof(a,b,test_X_scaled )
gs_clf.best_params_<save_to_csv>
X_train_add = np.hstack(( a,X_train_stack))
gs_clf.refit
y_pred_proba = gs_clf.predict_proba(X_Test)
 tdf = pd.DataFrame(y_pred_proba)
 tdf.columns = ['EAP', 'HPL', 'MWS']
submission = pd.read_csv('.. /input/spooky-author-identification/sample_submission.csv')
 result = pd.concat([submission[['id']], tdf], axis=1)
 result.head()
result.to_csv('naive_bayes_old.csv', index=False)
<import_modules>
X_test_add = np.hstack(( test_X_scaled,X_test_stack))
stop_words = stopwords.words('english' )<compute_test_metric>
score = rmse_cv(stack_model,X_train_add,b)
 print(score.mean() )
def multiclass_logloss(actual, predicted, eps=1e-15):

 if len(actual.shape)== 1:
actual2 = np.zeros(( actual.shape[0], predicted.shape[1]))
for i, val in enumerate(actual):
actual2[i, val] = 1
actual = actual2
clip = np.clip(predicted, eps, 1 - eps)
 rows = actual.shape[0]
vsota = np.sum(actual * np.log(clip))
return -1.0 / rows * vsota<categorify>
stack_model.fit(a,b )
lbl_enc = preprocessing.LabelEncoder()
y = lbl_enc.fit_transform(df_train.author.values)
 y<split>
pred = np.exp(stack_model.predict(test_X_scaled))
<train_model><EOS> 
result=pd.DataFrame({'Id':test.Id, 'SalePrice':pred})
 result.to_csv("submission.csv",index=False )