Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
warnings.filterwarnings('ignore' )
%%time
test_samples = [Parallel(n_jobs=4 )(delayed(feature_gen )(test_path)for test_path in tqdm_notebook(test_paths)) ]<drop_column>
import os
import pandas as pd
import numpy as np
from datetime import datetime
import scipy
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import Pipeline
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.model_selection import GridSearchCV
from xgboost.sklearn import XGBRegressor
from sklearn.ensemble import GradientBoostingRegressor
from mlxtend.regressor import StackingCVRegressor
from sklearn import metrics
import statsmodels.api as sm
test_samples = np.array(test_samples ).reshape(-1,40)
 del(test_paths)
 gc.collect()<predict_on_test>
train_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 train_df.head()
preds = cat.predict(test_samples )<prepare_output>
test_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 test_df.head()
sub['time_to_eruption'] = preds<save_to_csv>
train_Ids = train_df.pop('Id')
 test_Ids = test_df.pop('Id')
 test_df['SalePrice'] = np.NaN
sub.to_csv('submission.csv', header=True, index=False )<import_modules>
housing_all_df.drop(columns=['Utilities'], inplace = True )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
from nltk.corpus import stopwords
import tensorflow as tf
import tensorflow_addons as tfa
from transformers import TFAutoModel, AutoTokenizer<categorify>
null_counts = housing_all_df.isnull().sum()
null_counts[null_counts.values > 0].index
tweet_tokenizer = TweetTokenizer()
def normalizeToken(token):
lowercased_token = token.lower()
if token.startswith("@"):
return "@USER"
elif lowercased_token.startswith("http")or lowercased_token.startswith("www"):
return "HTTPURL"
elif len(token)== 1:
return demojize(token)
 else:
if token == "’":
return "'"
elif token == "…":
return "..."
else:
return token
def normalizeTweet(tweet):
tokens = tweet_tokenizer.tokenize(tweet.replace("’", "'" ).replace("…", "..."))
normTweet = " ".join([normalizeToken(token)for token in tokens])
 normTweet = normTweet.replace("cannot ", "can not " ).replace("n't ", " n't " ).replace("n 't ", " n't " ).replace("ca n't", "can't" ).replace("ai n't", "ain't")
 normTweet = normTweet.replace("'m ", " 'm " ).replace("'re ", " 're " ).replace("'s ", " 's " ).replace("'ll ", " 'll " ).replace("'d ", " 'd " ).replace("'ve ", " 've ")
 normTweet = normTweet.replace(" p.m.", " p.m." ).replace(" p.m ", " p.m " ).replace(" a.m.", " a.m." ).replace(" a.m ", " a.m ")
 normTweet = re.sub(r",([0-9]{2,4}),([0-9]{2,4})", r",\1,\2", normTweet)
 normTweet = re.sub(r"([0-9]{1,3})/([0-9]{2,4})", r"\1/\2", normTweet)
 normTweet = re.sub(r"([0-9]{1,3})-([0-9]{2,4})", r"\1-\2", normTweet)
 return " ".join(normTweet.split() )<load_from_csv>
print("Null count: ", housing_all_df['MSZoning'].isnull().sum())
 print(housing_all_df['MSZoning'].value_counts())
 housing_all_df['MSZoning'].fillna(housing_all_df['MSZoning'].mode().values[0], inplace = True )
def load_train_set() :
df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")[["text", "target"]]
df["text"] = df["text"].apply(normalizeTweet)
 return df
def load_test_set() :
df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")[["id", "text"]]
df["text"] = df["text"].apply(normalizeTweet)
 return df<count_values>
print("Null count: ", housing_all_df['PoolQC'].isnull().sum())
 housing_all_df['PoolQC'].value_counts()
print(train['target'].value_counts())
 print()
print(train['target'].value_counts(normalize=True))<string_transform>
housing_all_df[(housing_all_df['PoolQC'].isnull())&(housing_all_df['PoolArea'] > 0)][['PoolArea','PoolQC','OverallQual']]
disaster_tweets = train[train['target']==1]['text']
non_disaster_tweets = train[train['target']==0]['text']
freq_dist_disaster_tweets= nltk.FreqDist([word for tweet in disaster_tweets for word in tweet.lower().split() if word not in stopwords.words("english")and len(word)> 2])
 freq_dist_non_disaster_tweets= nltk.FreqDist([word for tweet in non_disaster_tweets for word in tweet.lower().split() if word not in stopwords.words("english")and len(word)> 2])
 results = {
'word' : [],
'disaster_count' : [],
'non_disaster_count' : []
}
for word, value in freq_dist_disaster_tweets.most_common(200):
results['word'].append(word)
 results['disaster_count'].append(freq_dist_disaster_tweets[word])
 results['non_disaster_count'].append(freq_dist_non_disaster_tweets[word])
 df = pd.DataFrame(results)
 df.head(30 )<define_variables>
housing_all_df['PoolQC'].fillna(value='None', inplace=True)
 housing_all_df['PoolQC'] = housing_all_df['PoolQC'].map({'None':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
 housing_all_df[housing_all_df.index == 2420]['PoolQC'] = 4
housing_all_df[housing_all_df.index == 2503]['PoolQC'] = 6
housing_all_df[housing_all_df.index == 2599]['PoolQC'] = 3
MAX_LENGTH = 50
short_tweets = sum(np.array(tweets_length)<= MAX_LENGTH)
 long_tweets = sum(np.array(tweets_length)> MAX_LENGTH)
 print("{} reviews with LEN > {}({:.2f} % of total data)".format(
 long_tweets,
MAX_LENGTH,
100 * long_tweets / len(train)
))<categorify>
housing_all_df.drop(columns=['MiscFeature','MiscVal'], inplace=True )
def encode_tweets(tokenizer, tweets, max_len):
nb_tweets = len(tweets)
 tokens = np.ones(( nb_tweets,max_len),dtype='int32')
 masks = np.zeros(( nb_tweets,max_len),dtype='int32')
 segs = np.zeros(( nb_tweets,max_len),dtype='int32')
 for k in range(nb_tweets):
tweet = tweets[k]
enc = tokenizer.encode(tweet)
 if len(enc)<max_len-2:
tokens[k,:len(enc)+2] = [0] + enc + [2]
masks[k,:len(enc)+2] = 1
else:
tokens[k,:max_len] = [0] + enc[:max_len-2] + [2]
masks[k,:max_len] = 1
return tokens,masks,segs
def build_model(max_len):
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 bertweet = TFAutoModel.from_pretrained("vinai/bertweet-base")
 x,_ = bertweet(ids,attention_mask=att,token_type_ids=tok)
 out=tf.keras.layers.Dense(1,activation='sigmoid' )(x[:,0,:])
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=out)
 optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 return model
model = build_model(MAX_LENGTH)
 model.summary()<categorify>
print("Null count: ", housing_all_df['Alley'].isnull().sum())
 print(housing_all_df['Alley'].value_counts())
 housing_all_df['Alley'].fillna(value='None', inplace = True )
train_tokens, train_masks, train_segs = encode_tweets(tokenizer,train["text"].to_list() , MAX_LENGTH)
 train_labels = train["target"]<train_model>
print("Null count: ", housing_all_df['Fence'].isnull().sum())
 print(housing_all_df['Fence'].value_counts())
 housing_all_df['Fence'].fillna(value='None', inplace=True )
es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', patience=3, restore_best_weights=True, verbose=1)
 train_labels = train['target']
train_history = model.fit(
 [train_tokens,train_masks,train_segs], train_labels,
validation_split=0.2,
epochs=5,
batch_size=16,
verbose = 1,
callbacks = [es]
)<save_to_csv>
housing_all_df['FireplaceQu'].fillna(value='None', inplace=True )
test_tokens, test_masks, test_segs = encode_tweets(tokenizer,test["text"].to_list() , MAX_LENGTH)
 test["target"] = model.predict([test_tokens, test_masks, test_segs] ).round().astype(int)
 submission = test[["id", "target"]]
submission.to_csv("submission.csv",index=False )<import_modules>
grouped_by_neighborhood = housing_all_df.groupby('Neighborhood')
 lot_frontage_median = grouped_by_neighborhood['LotFrontage'].median()
lot_frontage_substitute = housing_all_df['Neighborhood'].map(lot_frontage_median.to_dict())
 housing_all_df['LotFrontage'] = lot_frontage_substitute.where(pd.isnull(housing_all_df['LotFrontage']), housing_all_df['LotFrontage'] )
for dirname, _, filenames in os.walk('/kaggle/working'):
for filename in filenames:
print(os.path.join(dirname, filename))
print('นำเข้าไลบรารี่ข้อมูลเรียบร้อย' )<load_from_csv>
housing_all_df['GarageYrBlt'] = housing_all_df['YearBuilt'].where(pd.isnull(housing_all_df['GarageYrBlt']), housing_all_df['GarageYrBlt'])
 housing_all_df.loc[housing_all_df['GarageType'].isnull() &(housing_all_df['GarageArea'] == 0), 'GarageType'] = 'No Garage'
housing_all_df.loc[housing_all_df['GarageFinish'].isnull() &(housing_all_df['GarageArea'] == 0), 'GarageFinish'] = 'No Garage'
housing_all_df.loc[housing_all_df['GarageQual'].isnull() &(housing_all_df['GarageArea'] == 0), 'GarageQual'] = 'None'
housing_all_df.loc[housing_all_df['GarageCond'].isnull() &(housing_all_df['GarageArea'] == 0), 'GarageCond'] = 'None'
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sample_submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 print('นำเข้าชุดข้อมูลเรียบร้อย' )<count_missing_values>
housing_all_df[(housing_all_df['GarageQual'].isnull())|(housing_all_df['GarageCond'].isnull())][['GarageCars','GarageArea',
'GarageType','GarageFinish','GarageQual','GarageCond']]
print(train.isnull().sum() )<count_missing_values>
housing_all_df.loc[housing_all_df.index==2576, 'GarageCars']= 0
housing_all_df.loc[housing_all_df.index==2576, 'GarageArea']= 0
housing_all_df.loc[housing_all_df.index==2576, 'GarageType']= 'No Garage'
housing_all_df.loc[housing_all_df.index==2576, 'GarageFinish']= 'No Garage'
housing_all_df.loc[housing_all_df.index==2576, 'GarageQual']= 'None'
housing_all_df.loc[housing_all_df.index==2576, 'GarageCond']= 'None'
print(test.isnull().sum() )<count_missing_values>
housing_all_df.loc[housing_all_df['GarageFinish'].isnull() ,'GarageFinish'] = housing_all_df['GarageFinish'].mode().values[0]
housing_all_df.loc[housing_all_df['GarageCond'].isnull() ,'GarageCond'] = housing_all_df['GarageCond'].mode().values[0]
housing_all_df.loc[housing_all_df['GarageQual'].isnull() ,'GarageQual'] = housing_all_df['GarageQual'].mode().values[0]
print(sample_submission.isnull().sum() )<load_from_url>
print("Null count: ", housing_all_df['Exterior1st'].isnull().sum())
 print(housing_all_df['Exterior1st'].value_counts())
 housing_all_df['Exterior1st'].fillna(value=housing_all_df['Exterior1st'].mode().values[0], inplace=True )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
print('ดาวน์โหลด Algorithm สำเร็จ!!!' )<import_modules>
print("Null count: ",housing_all_df['Exterior2nd'].isnull().sum())
 print(housing_all_df['Exterior2nd'].value_counts())
 housing_all_df['Exterior2nd'].fillna(housing_all_df['Exterior2nd'].mode().values[0], inplace = True )
print('นำเข้า "tokenization" เรียบร้อย' )<train_model>
print("Null count: ", housing_all_df['MasVnrType'].isnull().sum())
 print(housing_all_df['MasVnrType'].value_counts())
 housing_all_df.loc[housing_all_df['MasVnrType'].isnull() & housing_all_df['MasVnrArea'].isnull() , 'MasVnrType'] = 'None'
housing_all_df.loc[housing_all_df['MasVnrType'].isnull() & housing_all_df['MasVnrArea'], 'MasVnrType'] = housing_all_df['MasVnrType'].mode().values[0]
housing_all_df['MasVnrArea'].fillna(value=0, inplace=True )
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]

 input_sequence = ["[CLS]"] + text + ["[SEP]"]

 pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 print("เสร็จสิ้น" )<train_model>
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtFinSF1'] = 0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtFinSF2'] =0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtUnfSF'] =0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtFullBath'] =0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtHalfBath'] =0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'TotalBsmtSF'] =0
"Let's learn deep learning!"
['Let', "'", 's', 'learn', 'deep', 'learning', '!']
['[CLS]', 'Let', "'", 's', 'learn', 'deep', 'learning', '!', '[SEP]']
['[CLS]', 'Let', "'", 's', 'learn', 'deep', 'learning', '!', '[SEP]', '[PAD]','[PAD]','[PAD]','[PAD]','[PAD]']
[101, 2421, 112, 188, 3858, 1996, 3776, 106, 102, 0, 0, 0, 0, 0 ,0 ]<choose_model_class>
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtQual'].isnull()), 'BsmtQual'] = 'None'
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtCond'].isnull()), 'BsmtCond'] = 'None'
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtExposure'].isnull()), 'BsmtExposure'] = 'None'
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtFinType1'].isnull()), 'BsmtFinType1'] = 'None'
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtFinType2'].isnull()), 'BsmtFinType2'] = 'None'
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
print("เสร็จสิ้น" )<choose_model_class>
housing_all_df.loc[housing_all_df['BsmtQual'].isnull() , 'BsmtQual'] = housing_all_df['BsmtQual'].mode().values[0]
housing_all_df.loc[housing_all_df['BsmtCond'].isnull() , 'BsmtCond'] = housing_all_df['BsmtCond'].mode().values[0]
housing_all_df.loc[housing_all_df['BsmtExposure'].isnull() , 'BsmtExposure'] = housing_all_df['BsmtExposure'].mode().values[0]
housing_all_df.loc[housing_all_df['BsmtFinType2'].isnull() , 'BsmtFinType2'] = housing_all_df['BsmtFinType2'].mode().values[0]
housing_all_df.loc[housing_all_df['BsmtFullBath'].isnull() , 'BsmtFullBath'] = 0
housing_all_df.loc[housing_all_df['BsmtHalfBath'].isnull() , 'BsmtHalfBath'] = 0
%%time
print('กำลังดาวน์โหลดโมเดลอาจใช้เวลาสักครู่...')
 module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 print('ดาวน์โหลดโมเดลสำเร็จ!!!' )<data_type_conversions>
print("Null counts: " , housing_all_df['Electrical'].isnull().sum())
 print(housing_all_df['Electrical'].value_counts())
 housing_all_df['Electrical'].fillna(housing_all_df['Electrical'].mode().values[0], inplace=True )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
print("Null count: " , housing_all_df['KitchenQual'].isnull().sum())
 print(housing_all_df['KitchenQual'].value_counts())
 housing_all_df['KitchenQual'].fillna(housing_all_df['KitchenQual'].mode().values[0], inplace=True )
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<train_model>
print("Null count: ", housing_all_df['Functional'].isnull().sum())
 print(housing_all_df['Functional'].value_counts())
 housing_all_df['Functional'].fillna(housing_all_df['Functional'].mode().values[0], inplace = True )
start_time = time.time()
train_history = model.fit(train_input,
train_labels,
validation_split = 0.2,
epochs = 3,
batch_size = 16)
 print(' สำเร็จ!!! ')
 print("---ใช้เวลาทั้งหมด %s วินาที ---" %(time.time() - start_time))<predict_on_test>
print("Null count: ", housing_all_df['SaleType'].isnull().sum())
 print(housing_all_df['SaleType'].value_counts())
 housing_all_df['SaleType'].fillna(housing_all_df['SaleType'].mode().values[0], inplace=True )
test_pred = model.predict(test_input)
 print(' สำเร็จ!!! ' )<save_to_csv>
high_corr_matrix = housing_corr[abs(housing_corr.loc[:,:])>= 0.5]
high_corr_matrix.to_csv('./correlations.csv' )
sample_submission['target'] = test_pred.round().astype(int)
 sample_submission.to_csv('sample_submission.csv', index=False)
 print('สร้างไฟล์ submission.csv เรียบร้อย' )<count_missing_values>
cols_to_drop = ['GarageYrBlt','YearRemodAdd','1stFlrSF','TotalBsmtSF','TotRmsAbvGrd','GrLivArea','GarageArea']
housing_all_df.drop(columns=cols_to_drop, inplace=True )
sample_submission.isnull().sum()<import_modules>
train_df[train_df['OverallQual'] == 10][['GrLivArea', 'SalePrice']].sort_values('GrLivArea', ascending=False )
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer ,TfidfTransformer<import_modules>
housing_all_df.drop(index=[523,1298], inplace=True )
nlp = spacy.load("en_core_web_lg" )<load_from_csv>
numerical_cols = housing_all_df.select_dtypes(include='number' ).columns.tolist()
numerical_cols = [col for col in numerical_cols if col not in ['MSSubClass','OverallQual','OverallCond','PoolQC','MoSold','SalePrice','IsRemod','IsNew','YearBuilt','YrSold']]
for col in numerical_cols:
skew_val = scipy.stats.skew(housing_all_df[col])
 if abs(skew_val)> 0.8:
housing_all_df[col] = np.log(housing_all_df[col] + 1 )
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_data =pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train_data.head(5 )<define_variables>
for col in ['FireplaceQu', 'ExterQual', 'ExterCond', 'BsmtQual','BsmtCond','KitchenQual','GarageQual','GarageCond','HeatingQC']:
housing_all_df[col] = housing_all_df[col].map({'None':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
 housing_all_df['BsmtExposure'] = housing_all_df['BsmtExposure'].map({'None':0,'No':1,'Mn':2,'Av':3,'Gd':4})
 for col in ['BsmtFinType1','BsmtFinType2']:
housing_all_df[col] = housing_all_df[col].map({'None':0,'Unf':1,'LwQ':2,'Rec':3,'BLQ':4,'ALQ':5,'GLQ':6})
 housing_all_df['GarageFinish'] = housing_all_df['GarageFinish'].map({'No Garage':0,'Unf':1,'RFn':2,'Fin':3} )
train_data_shape = train_data.shape[0]<categorify>
groupd_by_neighborhood = housing_all_df[~housing_all_df['SalePrice'].isnull() ].groupby('Neighborhood')
 neighborhood_median_prices = groupd_by_neighborhood['SalePrice'].median()
neighborhood_median_prices = neighborhood_median_prices.sort_values()
neighborhood_mean_prices = groupd_by_neighborhood['SalePrice'].mean()
neighborhood_mean_prices = neighborhood_mean_prices.sort_values()
def clean_text(text):
url = re.compile(r'https?://\S+|www\.\S+')
 text = url.sub(r'', text)
 html = re.compile(r'<.*?>')
 text = html.sub(r'', text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags = re.UNICODE)
 text = emoji_pattern.sub(r'', text)
 table = str.maketrans('', '', string.punctuation)
 text = text.translate(table)
 text = text.lower()
text = re.sub(r"i'm", "i am", text)
 text = re.sub(r"you'll", "you will", text)
 text = re.sub(r"i'll", "i will", text)
 text = re.sub(r"she'll", "she will", text)
 text = re.sub(r"he'll", "he will", text)
 text = re.sub(r"he's", "he is", text)
 text = re.sub(r"she's", "she is", text)
 text = re.sub(r"that's", "that is", text)
 text = re.sub(r"what's", "what is", text)
 text = re.sub(r"where's", "where is", text)
 text = re.sub(r"there's", "there is", text)
 text = re.sub(r"here's", "here is", text)
 text = re.sub(r"who's", "who is", text)
 text = re.sub(r"how's", "how is", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"can't", "cannot", text)
 text = re.sub(r"won't", "will not", text)
 text = re.sub(r"don't", "do not", text)
 text = re.sub(r"shouldn't", "should not", text)
 tweet = re.sub(r"&gt;", ">", text)
 tweet = re.sub(r"&lt;", "<", text)
 tweet = re.sub(r"&amp;", "&", text)
 tweet = re.sub(r"w/e", "whatever", text)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", text)
 tweet = re.sub(r"recentlu", "recently", text)
 tweet = re.sub(r"Ph0tos", "Photos", text)
 tweet = re.sub(r"amirite", "am I right", text)
 tweet = re.sub(r"exp0sed", "exposed", text)
 tweet = re.sub(r"<3", "love", text)
 tweet = re.sub(r"amageddon", "armageddon", text)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", text)
 tweet = re.sub(r"WindStorm", "Wind Storm", text)
 tweet = re.sub(r"8/6/2015", "2015-08-06", text)
 tweet = re.sub(r"10:38PM", "10:38 PM", text)
 tweet = re.sub(r"10:30pm", "10:30 PM", text)
 tweet = re.sub(r"16yr", "16 year", text)
 tweet = re.sub(r"lmao", "laughing my ass off", text)
 tweet = re.sub(r"TRAUMATISED", "traumatized", text)
 text = re.sub(r"n't", " not", text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[{}]'.format(re.escape(string.punctuation)) , '', text)
 text = re.sub('\w*\d\w*', '', text)
 text = re.sub(r" ", " ", text)
 text = "".join([c if c not in string.punctuation else "" for c in text])
 text = ''.join(c for c in text if not c.isdigit())
 text = text.replace(r"÷", "")
 text = re.sub(' +', ' ', text)
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 return text
train_data['text'] = train_data['text'].apply(lambda x : clean_text(x))
test_data['text'] = test_data['text'].apply(lambda x : clean_text(x))<feature_engineering>
housing_all_df.loc[~housing_all_df['Neighborhood'].isin(['StoneBr','NridgHt','NoRidge','MeadowV','IDOTRR','BrDale']), 'Neighborhood'] = 1
housing_all_df.loc[housing_all_df['Neighborhood'].isin(['StoneBr','NridgHt','NoRidge']), 'Neighborhood'] = 2
housing_all_df.loc[housing_all_df['Neighborhood'].isin(['MeadowV','IDOTRR','BrDale']), 'Neighborhood'] = 0
def massage_text(text):
tweet = re.sub("[^a-zA-Z]", ' ', text)
 tweet = tweet.lower()
tweet = tweet.split()
lem = WordNetLemmatizer()
tweet = [lem.lemmatize(word)for word in tweet
if word not in set(stopwords.words('english')) ]
tweet = ' '.join(tweet)
 return tweet
print('--here goes nothing')
 print(text)
 print(tweet)
 train_data['text'] = train_data['text'].apply(lambda x : massage_text(x))
test_data['text'] = test_data['text'].apply(lambda x : massage_text(x))<train_on_grid>
housing_all_df['Neighborhood'] = housing_all_df['Neighborhood'].astype(int )
count_vectorizer = feature_extraction.text.CountVectorizer()
train_vectors = count_vectorizer.fit_transform(train_data["text"])
 test_vectors = count_vectorizer.transform(test_data["text"])
 vectorizer = TfidfVectorizer()
Train = vectorizer.fit_transform(train_data['text'])
 test = vectorizer.transform(test_data['text'])
 clf = linear_model.RidgeClassifier()
clf.fit(Train, train_data["target"])
 scores = model_selection.cross_val_score(clf, Train, train_data["target"], cv=3, scoring="f1")
 scores<train_model>
all_dummies = []
cat_cols = housing_all_df.select_dtypes(include='object' ).columns.tolist()
cat_cols.extend(['MSSubClass', 'MoSold','YrSold'])
 for col in cat_cols:
dummies = pd.get_dummies(housing_all_df[col], prefix=col, prefix_sep='_', drop_first=True)
 for dum_col in dummies.columns.tolist() :
if dummies[dum_col].sum() < 10:
print("Dummy: {0} has less than 10 records".format(dum_col))
dummies.drop(columns=[dum_col], inplace =True)
 all_dummies.extend(dummies)
 housing_all_df.drop(columns=col, inplace=True)
 housing_all_df = pd.concat([housing_all_df, dummies], axis=1 )
clf.fit(Train, train_data["target"] )<load_from_csv>
X = housing_train_df.drop(columns=['SalePrice', 'SalePrice_log'])
 y = housing_train_df['SalePrice_log']
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size = 0.7, test_size = 0.3, random_state = 100 )
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = clf.predict(test)
 sample_submission.head()<save_to_csv>
def root_mean_squared_error(actual, predicted):
return np.sqrt(metrics.mean_squared_error(actual,predicted))
custom_scorer = metrics.make_scorer(root_mean_squared_error, greater_is_better = False )
sample_submission.to_csv("submission.csv", index=False )<import_modules>
def grid_search_cv(params, model, X_train, y_train, X_test, y_test):
folds = 5
model_cv = GridSearchCV(estimator = model,
param_grid = params,
scoring = custom_scorer,
cv = folds,
verbose = True,
return_train_score = True,
n_jobs=-1)
 model_cv.fit(X_train, y_train)
 print("Best score: ", model_cv.best_score_)
 print("Best params: ", model_cv.best_params_)
 y_test_pred = model_cv.best_estimator_.predict(X_test)
 print("Test score: ", root_mean_squared_error(y_test, y_test_pred))
nlp = spacy.load("en_core_web_lg" )<load_from_csv>
params = {'model__alpha' : [0.0001, 0.001,0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.2, 0.5, 1, 2,
5, 10, 50]}
lasso = Pipeline([('scaler',RobustScaler()),('model',Lasso(random_state = 100, normalize=True)) ])
 grid_search_cv(params, lasso, X_train, y_train, X_test, y_test )
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 train_data<load_from_csv>
lasso_final = Pipeline([('scaler', RobustScaler()),('model', Lasso(random_state=100, alpha=0.0001, normalize=True)) ])
 lasso_final.fit(X, y)
 y_pred = lasso_final.predict(X)
 print("Lasso rmse: ", root_mean_squared_error(y, y_pred))
test_data = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 test_data<define_variables>
ridge_final = Pipeline([('scaler',RobustScaler()),('model',Ridge(random_state=100, alpha =0.04, normalize=True)) ])
 ridge_final.fit(X, y)
 y_pred = ridge_final.predict(X)
 print("Ridge rmse: ", root_mean_squared_error(y, y_pred))
train_data_shape = train_data.shape[0]<concatenate>
xgb = XGBRegressor(random_state=100, learning_rate=0.1,max_depth=5, min_child_weight=1,subsample=0.8,
colsample_by_tree=0.8)
 hyper_params = {'n_estimators':[50,100,200]}
grid_search_cv(hyper_params, xgb, X_train, y_train, X_test, y_test )
df = pd.concat([train_data, test_data])
 df.shape<categorify>
xgb = XGBRegressor(random_state=100, learning_rate=0.1,max_depth=5, min_child_weight=1,subsample=0.8,
colsample_by_tree=0.8)
 hyper_params = {'n_estimators':[250,300,350]}
grid_search_cv(hyper_params, xgb, X_train, y_train, X_test, y_test )
def clean_text(text):
url = re.compile(r'https?://\S+|www\.\S+')
 text = url.sub(r'', text)
 html = re.compile(r'<.*?>')
 text = html.sub(r'', text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags = re.UNICODE)
 text = emoji_pattern.sub(r'', text)
 table = str.maketrans('', '', string.punctuation)
 text = text.translate(table)
 return text
df['text'] = df['text'].apply(lambda x : clean_text(x))<define_variables>
xgb = XGBRegressor(random_state=100, learning_rate=0.1,n_estimators=200, subsample=0.8,
colsample_by_tree=0.8)
 hyper_params = {'max_depth':range(3,10,2), 'min_child_weight':range(1,9,2)}
grid_search_cv(hyper_params, xgb, X_train, y_train, X_test, y_test )
with nlp.disable_pipes() :
doc_vectors = np.array([nlp(text ).vector for text in df["text"]] )<import_modules>
xgb = XGBRegressor(random_state=100, learning_rate=0.1,n_estimators=200, max_depth=3, min_child_weight=5,
subsample=0.8,colsample_by_tree=0.8)
 hyper_params = {'gamma':[0,0.1,0.2,0.3,0.4,0.5]}
grid_search_cv(hyper_params, xgb, X_train, y_train, X_test, y_test )
<define_variables>
xgb = XGBRegressor(random_state=100, learning_rate=0.1, n_estimators=200, max_depth=3, min_child_weight=5,
gamma = 0)
 hyper_params = {'subsample':[0.6,0.7,0.8,0.9], 'colsample_by_tree':[0.6,0.7,0.8,0.9]}
grid_search_cv(hyper_params, xgb, X_train, y_train, X_test, y_test )
train_doc_vectors = doc_vectors[:train_data_shape]
submission_doc_vectors = doc_vectors[train_data_shape:]<import_modules>
xgb = XGBRegressor(random_state=100, learning_rate=0.1, n_estimators=200, max_depth=3, min_child_weight=5,
gamma = 0, subsample=0.8, colsample_by_tree=0.6)
 hyper_params = {'reg_alpha':[0.0001,0.001,0.01,0.1,1,10]}
grid_search_cv(hyper_params, xgb, X_train, y_train, X_test, y_test )
from sklearn.ensemble import RandomForestClassifier 
<split>
xgb = XGBRegressor(random_state=100, learning_rate=0.1, n_estimators=200, max_depth=3, min_child_weight=5,
gamma = 0, subsample=0.8, colsample_by_tree=0.6,reg_alpha=0.001)
 hyper_params = {'reg_lambda':[0.0001,0.001,0.01,0.1,1,10]}
grid_search_cv(hyper_params, xgb, X_train, y_train, X_test, y_test )
X = train_doc_vectors
y = train_data.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1, random_state = 1 )<choose_model_class>
xgb = XGBRegressor(random_state=100, learning_rate=0.1, n_estimators=200, max_depth=3, min_child_weight=5,
gamma = 0, subsample=0.8, colsample_by_tree=0.6,reg_alpha=0.001, reg_lambda=1)
 xgb.fit(X,y)
 y_pred = xgb.predict(X)
 print("Train error: ", root_mean_squared_error(y, y_pred))
model = svm.SVC(kernel='linear' )<train_model>
xgb = XGBRegressor(random_state=100, learning_rate=0.05, n_estimators=400, max_depth=3, min_child_weight=5,
gamma = 0, subsample=0.8, colsample_by_tree=0.6,reg_alpha=0.001, reg_lambda=1)
 xgb.fit(X,y)
 y_pred = xgb.predict(X)
 print("Train error: ", root_mean_squared_error(y, y_pred))
model.fit(X_train, y_train )<predict_on_test>
xgb = XGBRegressor(random_state=100, learning_rate=0.01, n_estimators=2000, max_depth=3, min_child_weight=5,
gamma = 0, subsample=0.8, colsample_by_tree=0.6,reg_alpha=0.001, reg_lambda=1)
 xgb.fit(X,y)
 y_pred = xgb.predict(X)
 print("Train error: ", root_mean_squared_error(y, y_pred))
model_final =RandomForestClassifier(n_estimators=100)
 model_final.fit(X, y)
 predictions_final = model_final.predict(submission_doc_vectors )<load_from_csv>
xgb_final = XGBRegressor(random_state=100, learning_rate=0.01, n_estimators=2000, max_depth=3, min_child_weight=5,
gamma = 0, subsample=0.8, colsample_by_tree=0.6,reg_alpha=0.001, reg_lambda=1)
 xgb_final.fit(X,y)
 y_pred = xgb_final.predict(X)
 print("XGBRegressor rmse: ", root_mean_squared_error(y, y_pred))
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission = model.predict(submission_doc_vectors)
 submission<prepare_output>
elastic_net = Pipeline([('scaler',RobustScaler()),('model',ElasticNet(random_state=100, normalize=True)) ])
 hyper_params = [{'model__alpha':[0.0001, 0.001,0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.2, 0.5, 1, 2,
5, 10, 50], 'model__l1_ratio':[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]}]
grid_search_cv(hyper_params, elastic_net, X_train, y_train, X_test, y_test )
submission = pd.DataFrame({'id': test_data.id, 'target' : predictions_final})
 submission.head()<save_to_csv>
elastic_net_final = Pipeline([('scaler',RobustScaler()), 
('model',ElasticNet(random_state=100, normalize=True, alpha=0.0001, l1_ratio=0.9)) ])
 elastic_net_final.fit(X, y)
 y_pred = elastic_net_final.predict(X)
 print("ElasticNet rmse: ", root_mean_squared_error(y, y_pred))
submission.to_csv('submission1.csv', index=False )<install_modules>
grb = GradientBoostingRegressor(random_state=100, learning_rate=0.1,subsample=0.8,min_samples_split=150,
min_samples_leaf=50, max_depth=5,max_features='sqrt')
 hyper_params = {'n_estimators':[50,150,200,250,300,350,400]}
grid_search_cv(hyper_params, grb, X_train, y_train, X_test, y_test )
!pip install transformers==2.11.0 --quiet
!pip install simpletransformers==0.41.0 --quiet
!pip install pyspellchecker --quiet<import_modules>
grb = GradientBoostingRegressor(random_state=100, learning_rate=0.1, n_estimators=350, subsample=0.8,
min_samples_leaf=50, max_features='sqrt')
 hyper_params = {'max_depth':range(3,16,2), 'min_samples_split':range(150,300,50)}
grid_search_cv(hyper_params, grb, X_train, y_train, X_test, y_test )
import random
import torch<import_modules>
grb = GradientBoostingRegressor(random_state=100, learning_rate=0.1, n_estimators=350, max_depth=5,
min_samples_split=150, subsample=0.8, max_features='sqrt')
 hyper_params = {'min_samples_leaf': range(50,101,10)}
grid_search_cv(hyper_params, grb, X_train, y_train, X_test, y_test )
from simpletransformers.classification import ClassificationModel
import pandas as pd<set_options>
grb = GradientBoostingRegressor(random_state=100, learning_rate=0.1, n_estimators=350, max_depth=5,
min_samples_split=150, min_samples_leaf=50, max_features='sqrt')
 hyper_params = {'subsample':[0.6,0.7,0.75,0.8,0.85,0.9]}
grid_search_cv(hyper_params, grb, X_train, y_train, X_test, y_test )
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )<set_options>
grb = GradientBoostingRegressor(random_state=100, learning_rate=0.1, n_estimators=350, max_depth=5,
min_samples_split=150, min_samples_leaf=50, max_features='sqrt', subsample=0.75)
 grb.fit(X, y)
 y_pred = grb.predict(X)
 print("grb rmse: ", root_mean_squared_error(y, y_pred))
warnings.simplefilter('ignore')
<set_options>
grb = GradientBoostingRegressor(random_state=100, learning_rate=0.05, n_estimators=700, max_depth=5,
min_samples_split=150, min_samples_leaf=50, max_features='sqrt', subsample=0.75)
 grb.fit(X, y)
 y_pred = grb.predict(X)
 print("grb rmse: ", root_mean_squared_error(y, y_pred))
def seed_all(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_all(79 )<load_from_csv>
grb = GradientBoostingRegressor(random_state=100, learning_rate=0.01, n_estimators=3500, max_depth=5,
min_samples_split=150, min_samples_leaf=50, max_features='sqrt', subsample=0.75)
 grb.fit(X, y)
 y_pred = grb.predict(X)
 print("grb rmse: ", root_mean_squared_error(y, y_pred))
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 print("Shape of train data : ",train.shape)
 print("Shape of test data : ",test.shape )<groupby>
grb_final = GradientBoostingRegressor(random_state=100, learning_rate=0.01, n_estimators=3500, max_depth=5,
min_samples_split=150, min_samples_leaf=50, max_features='sqrt', subsample=0.75)
 grb_final.fit(X, y)
 y_pred = grb_final.predict(X)
 print("grb rmse: ", root_mean_squared_error(y, y_pred))
<feature_engineering>
stacked = StackingCVRegressor(regressors=(lasso_final, ridge_final, elastic_net_final, grb_final),
meta_regressor=xgb_final, use_features_in_secondary=True)
 stacked.fit(X, y)
 y_pred = stacked.predict(X)
 print("StackingCVRegressor rmse: ", root_mean_squared_error(y, y_pred))
train['keyword'].fillna('', inplace=True)
 train['final_text'] = train['keyword'] + ' ' + train['text']
test['keyword'].fillna('', inplace=True)
 test['final_text'] = test['keyword'] + ' ' + test['text']<drop_column>
def get_blended_predictions(df):
return(
(0.05 * lasso_final.predict(df)) + 
(0.05 * ridge_final.predict(df)) + 
(0.05 * elastic_net_final.predict(df)) + 
(0.15 * grb_final.predict(df)) + 
(0.25 * xgb_final.predict(df)) + 
(0.45 * stacked.predict(df))
)
train=train.drop(['id'],axis=1)
 train=train.drop(['keyword'],axis=1)
 train=train.drop(['text'],axis=1)
 train=train.drop(['location'],axis=1)
 train.head()<prepare_output>
y_pred = get_blended_predictions(X)
 print("Blended model rmse: ", root_mean_squared_error(y, y_pred))
<drop_column><EOS> 
housing_test_df.pop('SalePrice')
 housing_test_pred = get_blended_predictions(housing_test_df)
 housing_test_submission = pd.DataFrame()
housing_test_submission['Id'] = test_Ids
housing_test_submission['SalePrice'] = np.exp(housing_test_pred)
 housing_test_submission.to_csv('./submission.csv', index=False )