Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<predict_on_test>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats
from statsmodels.graphics.gofplots import qqplot
from sklearn.preprocessing import LabelEncoder
from scipy.stats import norm, skew, boxcox_normmax
pred_proba = model.predict(X_test)[:,1]
pred = np.where(pred_proba > 0.5,1,0 )<save_to_csv>
hp_train = pd.read_csv(".. /input/train.csv")
 hp_test = pd.read_csv(".. /input/test.csv" )
submission = pd.DataFrame({'PassengerId': test['PassengerId'], 'Survived': pred})
 submission.to_csv('submission_titanic.csv', index=False )<install_modules>
train_ID = hp_train['Id']
test_ID = hp_test['Id']
hp_train.drop(['Id'], axis=1, inplace=True)
 hp_test.drop(['Id'], axis=1, inplace=True )
!pip install tweet-preprocessor<import_modules>
print(hp_train.shape)
 hp_train = hp_train[hp_train['GrLivArea'] < 4500]
hp_train = hp_train[hp_train['LotArea'] < 550000]
hp_train = hp_train[hp_train['TotalBsmtSF'] < 3000]
hp_train = hp_train[hp_train['1stFlrSF'] < 2500]
hp_train = hp_train[hp_train['BsmtFinSF1'] < 2000]
import os
from google.cloud import storage, automl_v1beta1 as automl
import numpy as np
import pandas as pd
from sklearn import feature_extraction, linear_model, model_selection, preprocessing
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
import scipy as sp
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold, KFold, GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score
from sklearn.naive_bayes import GaussianNB, MultinomialNB
import seaborn as sns
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from nltk.stem import WordNetLemmatizer
import nltk
from collections import Counter
import string
import re
from nltk.corpus import wordnet as wn
from statistics import mean
import preprocessor<import_modules>
ntrain = hp_train.shape[0]
ntest = hp_test.shape[0]
y_train = hp_train.SalePrice.values
df_all = pd.concat(( hp_train, hp_test), sort=False ).reset_index(drop=True)
 df_all.drop(['SalePrice'], axis=1, inplace=True)
 print("df_all size after concatenation of train and test data is : {}".format(df_all.shape))
from sklearn import metrics<string_transform>
missing_data = pd.DataFrame({'total_missing': df_all.isnull().sum() , 'perc_missing':(df_all.isnull().sum() /len(df_all)) *100})
 len(missing_data[missing_data.total_missing>0] )
stop = set(STOPWORDS ).union(set(['FAV' , 'RT']))
lemma = WordNetLemmatizer()
preprocessor.set_options(preprocessor.OPT.URL, preprocessor.OPT.MENTION, preprocessor.OPT.NUMBER, preprocessor.OPT.RESERVED)
 def clean(text):
text = preprocessor.clean(text)
 text = re.sub(r'[^\w\s]','',text)
 stop_free = " ".join([i for i in text.split(' ')if(i not in stop)])
 normalized = " ".join(lemma.lemmatize(word)for word in stop_free.split())
 return normalized<load_from_csv>
no_amen_cat = ['PoolQC','MiscFeature','Alley','Fence','GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType', 'FireplaceQu']
no_amen_num = ['GarageYrBlt', 'GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'MasVnrArea']
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<feature_engineering>
df_all1 = df_all.copy()
train_df.text = train_df.text.apply(clean)
 test_df.text = test_df.text.apply(clean )<define_variables>
for col in no_amen_cat:
df_all1[col] = df_all1[col].fillna('None' )
PROJECT_ID = 'automl-kaggle-263107'<define_variables>
for col in no_amen_num:
df_all1[col] = df_all1[col].fillna(0 )
BUCKET_NAME = 'automl-disaster-tweet-cleaned'
BUCKET_REGION = 'us-central1'<choose_model_class>
mode_replace = ['Electrical', 'MSZoning', 'Utilities', 'SaleType', 'KitchenQual', 'Exterior2nd', 'Exterior1st']
for col in mode_replace:
df_all1[col] = df_all1[col].fillna(df_all1[col].mode() [0] )
storage_client = storage.Client(project=PROJECT_ID)
 tables_gcs_client = automl.GcsClient(client=storage_client, bucket_name=BUCKET_NAME)
 automl_client = automl.AutoMlClient()
prediction_client = automl.PredictionServiceClient()
tables_client = automl.TablesClient(project=PROJECT_ID, region=BUCKET_REGION, client=automl_client, gcs_client=tables_gcs_client, prediction_client=prediction_client )<prepare_output>
df_all1["Functional"] = df_all1["Functional"].fillna("Typ" )
bucket = storage.Bucket(storage_client, name=BUCKET_NAME)
 if not bucket.exists() :
bucket.create(location=BUCKET_REGION )<load_pretrained>
df_all1.isna().sum() [df_all1.isna().sum() >0].sort_values(ascending=False )
def upload_blob(bucket_name, source_file_name, destination_blob_name):

 bucket = storage_client.get_bucket(bucket_name)
 blob = bucket.blob(destination_blob_name)
 blob.upload_from_filename(source_file_name)
 print('File {} uploaded to {}.'.format(
 source_file_name,
destination_blob_name))
def download_to_kaggle(bucket_name,destination_directory,file_name,prefix=None):

 os.makedirs(destination_directory, exist_ok = True)
 full_file_path = os.path.join(destination_directory, file_name)
 blobs = storage_client.list_blobs(bucket_name,prefix=prefix)
 for blob in blobs:
blob.download_to_filename(full_file_path )<save_to_csv>
df_all1["LotFrontage"] = df_all1.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
train_df[['id','text','target']].to_csv('/kaggle/working/train.csv', index=False)
 test_df[['id','text']].to_csv('/kaggle/working/test.csv', index=False )<load_pretrained>
df_all1.isna().sum() [df_all1.isna().sum() >0].sort_values(ascending=False )
upload_blob(BUCKET_NAME, '/kaggle/working/train.csv', 'train.csv')
 upload_blob(BUCKET_NAME, '/kaggle/working/test.csv', 'test.csv' )<create_dataframe>
df_all2 = df_all1.copy()
dataset_display_name = 'tweet_disaster_cleaned'
new_dataset = False
try:
dataset = tables_client.get_dataset(dataset_display_name=dataset_display_name)
 except:
new_dataset = True
dataset = tables_client.create_dataset(dataset_display_name )<train_model>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(df_all2[c].values))
df_all2[c] = lbl.transform(list(df_all2[c].values))
print('Shape : {}'.format(df_all2.shape))
if new_dataset:
gcs_input_uris = ['gs://' + BUCKET_NAME + '/train.csv']
import_data_operation = tables_client.import_data(
 dataset=dataset,
gcs_input_uris=gcs_input_uris 
)
 print('Dataset
import_data_operation.result()<define_variables>
skewness = skewness[abs(skewness)> 0.5]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.45
for feat in skewed_features:
df_all2[feat] = boxcox1p(df_all2[feat], lam )
ID_COLUMN = 'id'<choose_model_class>
df_all2 = df_all2.drop(['Street', 'Utilities', 'Condition2', 'RoofMatl', 'Heating', 'PoolQC'], axis = 1 )
TRAIN_BUDGET = 1000
model = None
model_display_name = 'tweet_disaster_model_clean'
try:
model = tables_client.get_model(model_display_name=model_display_name)
 except:
response = tables_client.create_model(
 model_display_name,
dataset=dataset,
train_budget_milli_node_hours=TRAIN_BUDGET,
exclude_column_spec_names=[TARGET_COLUMN,ID_COLUMN] 
)
 print('Create model operation: {}'.format(response.operation))
model = response.result()
print(model )<define_variables>
df_all2 = pd.get_dummies(df_all2 ).reset_index(drop=True)
 print(df_all2.shape )
gcs_input_uris = 'gs://' + BUCKET_NAME + '/test.csv'
gcs_output_uri_prefix = 'gs://' + BUCKET_NAME + '/predictions'
batch_predict_response = tables_client.batch_predict(
 model=model,
gcs_input_uris=gcs_input_uris,
gcs_output_uri_prefix=gcs_output_uri_prefix, 
)
 print('Batch prediction operation: {}'.format(batch_predict_response.operation))
batch_predict_result = batch_predict_response.result()
batch_predict_response.metadata<load_pretrained>
df_all2['TotalSF']=df_all2['TotalBsmtSF'] + df_all2['1stFlrSF'] + df_all2['2ndFlrSF']
df_all2['Total_sqr_footage'] =(df_all2['BsmtFinSF1'] + df_all2['BsmtFinSF2'] +
df_all2['1stFlrSF'] + df_all2['2ndFlrSF'])
 df_all2['Total_Bathrooms'] =(df_all2['FullBath'] +(0.5 * df_all2['HalfBath'])+
df_all2['BsmtFullBath'] +(0.5 * df_all2['BsmtHalfBath']))
df_all2['Total_porch_sf'] =(df_all2['OpenPorchSF'] + df_all2['3SsnPorch'] +
df_all2['EnclosedPorch'] + df_all2['ScreenPorch'] +
df_all2['WoodDeckSF'])
gcs_output_folder = batch_predict_response.metadata.batch_predict_details.output_info.gcs_output_directory.replace('gs://' + BUCKET_NAME + '/','')
 download_to_kaggle(BUCKET_NAME,'/kaggle/working','submissions.csv', prefix=gcs_output_folder )<load_from_csv>
df_all2['haspool'] = df_all2['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_all2['has2ndfloor'] = df_all2['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_all2['hasgarage'] = df_all2['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df_all2['hasbsmt'] = df_all2['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_all2['hasfireplace'] = df_all2['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
preds_df = pd.read_csv("/kaggle/working/submissions.csv")
 preds_df = preds_df.sort_values(by=['id'])
 preds_df['target'] =(preds_df['target_1_score'] >= 0.5 ).astype(int )<save_to_csv>
train = df_all2[:ntrain]
test = df_all2[ntrain:]
preds_df[['id','target']].to_csv("submission.csv", index=False )<feature_engineering>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, AdaBoostRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline, Pipeline
from sklearn.preprocessing import RobustScaler, StandardScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer, mean_squared_error
from sklearn.linear_model import LassoLarsCV, RidgeCV, ElasticNetCV
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import MinMaxScaler
from sklearn.svm import LinearSVR
from sklearn.pipeline import make_pipeline, make_union
from tpot.builtins import StackingEstimator
from tpot.builtins import ZeroCount
from sklearn.decomposition import PCA
from imblearn.pipeline import make_pipeline
tfidf_vectorizer = feature_extraction.text.TfidfVectorizer(ngram_range =(1,2), stop_words='english',strip_accents='unicode' )<categorify>
def score(model, train_x, train_y):
score = np.sqrt(-cross_val_score(model, train_x, train_y, cv=5, scoring="neg_mean_squared_error"))
print(score.mean() , score.std() )
train_vectors = tfidf_vectorizer.fit_transform(train_df["text"])
 test_vectors = tfidf_vectorizer.transform(test_df["text"] )<find_best_model_class>
tpot = TPOTRegressor(generations=1,verbosity=2,scoring='neg_mean_squared_error' )
feature_cols = ['keyword', 'location']
X = train_df[feature_cols]
y = train_df.target
one_hot_encoded_training_predictors = pd.get_dummies(X)
 clf = RandomForestClassifier(n_estimators = 100)
 scores = model_selection.cross_val_score(clf, one_hot_encoded_training_predictors, y, cv=5, scoring="f1")
 scores<train_model>
tpot.fit(train.values, y_train )
clf.fit(one_hot_encoded_training_predictors, y )<choose_model_class>
tpot.score(train.values, y_train )
clf = linear_model.RidgeClassifier() 
<compute_train_metric>
models_tested = pd.DataFrame(tpot.evaluated_individuals_ ).transpose()
scores = model_selection.cross_val_score(clf, train_vectors, train_df["target"], cv=10, scoring="f1")
 scores<train_model>
models_tested.sort_values(['internal_cv_score'], ascending=False ).head(10 )
clf.fit(train_vectors, train_df["target"] )<train_on_grid>
exported_pipeline1 = make_pipeline(
 StandardScaler() ,
ElasticNetCV(l1_ratio=0.7000000000000001, tol=0.001, cv=5)
)
 exported_pipeline1.fit(train.values, y_train )
parameters = {
'gamma': [0.7, 1, 'auto', 'scale']
}
clf = GridSearchCV(SVC(kernel='rbf'), parameters, cv=5, n_jobs=-1, scoring="f1" ).fit(train_vectors, train_df["target"] )<find_best_params>
score(exported_pipeline1, train.values, y_train)
 exported_pipeline_pred1 = exported_pipeline1.predict(train.values)
 score(exported_pipeline1, train.values, exported_pipeline_pred1 )
clf.best_estimator_<find_best_score>
exported_pipeline2 = make_pipeline(
 MinMaxScaler() ,
ElasticNetCV(l1_ratio=0.7000000000000001, tol=0.01, cv=5)
)
 exported_pipeline2.fit(train.values, y_train )
clf.best_score_<load_from_csv>
score(exported_pipeline2, train.values, y_train)
 exported_pipeline_pred2 = exported_pipeline2.predict(train)
 score(exported_pipeline2, train.values, exported_pipeline_pred2 )
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = clf.predict(test_vectors)
 df = pd.DataFrame({'text' : test_df['text'], 'prediction' : sample_submission["target"]} )<save_to_csv>
exported_pipeline3 = make_pipeline(
 ZeroCount() ,
PCA(iterated_power=4, svd_solver='randomized'),
RidgeCV() 
)
 exported_pipeline3.fit(train.values, y_train )
sample_submission.to_csv("submission1.csv", index=False )<import_modules>
score(exported_pipeline3, train.values, y_train)
 exported_pipeline_pred3 = exported_pipeline3.predict(train)
 score(exported_pipeline3, train.values, exported_pipeline_pred3 )
print(tf.__version__)
<split>
exported_pipeline4 = make_pipeline(
 LassoLarsCV(normalize=True, max_iter=60, cv=5)
)
 exported_pipeline4.fit(train.values, y_train )
X = train_df["text"]
y = train_df["target"]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42 )<define_variables>
score(exported_pipeline4, train.values, y_train)
 exported_pipeline_pred4 = exported_pipeline4.predict(train)
 score(exported_pipeline4, train.values, exported_pipeline_pred4 )
vocab_size = 10000
embedding_dim = 16
max_length = 30
trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"<string_transform>
exported_pipeline5 = make_pipeline(
 RidgeCV() 
)
 exported_pipeline5.fit(train.values, y_train )
tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)
 tokenizer.fit_on_texts(X_train)
 word_index = tokenizer.word_index
sequences = tokenizer.texts_to_sequences(X_train)
 padded = pad_sequences(sequences,maxlen=max_length, padding=padding_type, truncating=trunc_type)
 testing_sequences = tokenizer.texts_to_sequences(X_val)
 testing_padded = pad_sequences(testing_sequences,maxlen=max_length, padding=padding_type, truncating=trunc_type )<string_transform>
score(exported_pipeline5, train.values, y_train)
 exported_pipeline_pred5 = exported_pipeline5.predict(train)
 score(exported_pipeline5, train.values, exported_pipeline_pred5 )
reverse_word_index = dict([(value, key)for(key, value)in word_index.items() ])
 def decode_sentence(text):
return ' '.join([reverse_word_index.get(i, '?')for i in text])
 print(decode_sentence(padded[0]))<define_variables>
np.sqrt(mean_squared_error(y_train,(exported_pipeline_pred1*0.2 + exported_pipeline_pred2*0.2 +
exported_pipeline_pred3*0.2 + exported_pipeline_pred4*0.2 + exported_pipeline_pred5*0.2)) )
!wget --no-check-certificate \
https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt \
-O /tmp/glove.6B.100d.txt
embeddings_index = {};
vocab_size=len(word_index)
 embedding_dim = 100
with open('/tmp/glove.6B.100d.txt')as f:
for line in f:
values = line.split() ;
word = values[0];
coefs = np.asarray(values[1:], dtype='float32');
embeddings_index[word] = coefs;
embeddings_matrix = np.zeros(( vocab_size+1, embedding_dim)) ;
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word);
if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector;<choose_model_class>
best_value = []
min_value=1
for i in range(10000):
random = np.random.dirichlet(np.ones(5),size=1)
 best_value.append(np.sqrt(mean_squared_error(y_train,(exported_pipeline_pred1*random[0][0] + exported_pipeline_pred2*random[0][1] +
exported_pipeline_pred3*random[0][2] + exported_pipeline_pred4*random[0][3] + exported_pipeline_pred5*random[0][4]))))
if(np.min(best_value)< min_value):
min_value = np.min(best_value)
 min_array = random
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
tf.keras.layers.GlobalAveragePooling1D() ,
tf.keras.layers.Dense(24, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy',tf.keras.metrics.AUC() ])
 model.summary()<train_model>
def blend_models(sub):
return(exported_pipeline1.predict(sub)*min_array[0][0] +
exported_pipeline2.predict(sub)*min_array[0][1] +
exported_pipeline3.predict(sub)*min_array[0][2] +
exported_pipeline4.predict(sub)*min_array[0][3] +
exported_pipeline4.predict(sub)*min_array[0][4] )
num_epochs = 3
history = model.fit(padded, y_train, epochs=num_epochs, validation_data=(testing_padded, y_val))<prepare_output>
print('Predict submission')
 submission = pd.DataFrame()
submission['Id'] = test_ID
submission['SalePrice'] = np.floor(np.expm1(blend_models(test)) )
<string_transform><EOS> 
submission.to_csv('submission.csv',index=False )