Kaggle competition: House Prices - Advanced Regression Techniques
train_target0_words_dict = dict_formation(train[train_filter0])
 train_target1_words_dict = dict_formation(train[train_filter1])
 test_words_dict = dict_formation(test)
 train_word_dict = dict_formation(train )<feature_engineering>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id' )
def get_ngram_dataframe(n, data, label):
train_ngram = ngrams(data.text.str.cat(sep=' ' ).split() , n=n)
 train_ngram = Counter(train_ngram)
 train_ngram = dict(train_ngram)
 train_ngram = dict(sorted(train_ngram.items() , key=lambda x: x[1], reverse=True))
train_ngram_df = pd.DataFrame()
train_ngram_df[label] = train_ngram.keys()
train_ngram_df['Count'] = train_ngram.values()
return train_ngram_df<load_pretrained>
train['IsTrain'] = 1
test['IsTrain'] = 0
DataRaw = pd.concat([train, test] )
%%time
glove = '.. /input/glove6b100dtxt/glove.6B.100d.txt'
print("Extracting GloVe embedding")
 embed_glove = load_embed(glove )<feature_engineering>
print("Number of features not counting the target:" + str(len(DataRaw.columns)- 1 - 1))
def build_vocab(texts):

 sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word.lower() ] += 1
except KeyError:
vocab[word.lower() ] = 1
return vocab<define_variables>
C =(DataRaw.dtypes == 'object')
 CategoricalVariables = list(C[C].index)
 print(CategoricalVariables)
 print("")
 print("The number of categorical variables:" + str(len(CategoricalVariables)) )
def check_coverage(vocab, embeddings_index):

 known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words 
<categorify>
Integer =(DataRaw.dtypes == 'int64')
 Float =(DataRaw.dtypes == 'float64')
 NumericVariables = list(Integer[Integer].index)+ list(Float[Float].index)
 print(NumericVariables)
 print("")
 print("The number of numeric variables:" + str(len(NumericVariables)) )
vocab_train = build_vocab(train['text'])
 print("Glove : Train")
 oov_glove_train = check_coverage(vocab_train, embed_glove)
 vocab_test = build_vocab(test['text'])
 print("Glove : Test")
 oov_glove_test = check_coverage(vocab_test, embed_glove )<define_variables>
Missing_Percentage =(DataRaw.isnull().sum() ).sum() /np.product(DataRaw.shape)*100
print("The number of missing entries: " + str(round(Missing_Percentage,2)) + " %" )
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because",
"could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not",
"don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would",
"he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you",
"how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have",
"I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would",
"i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am",
"i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have",
"it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us",
"ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not",
"mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have",
"o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have",
"she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have",
"shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as",
"this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is",
"what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have",
"why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have",
"would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all",
"y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have",
"you'll": "you will", "you'll've": "you will have",
"you're": "you are", "you've": "you have" }<feature_engineering>
Numeric_NaN = DataRaw[NumericVariables].isnull().sum()
RowsCount = len(DataRaw.index)
 print("The percentage number of missing entries per variable: ", format(round(Numeric_NaN/RowsCount * 100)) )
%%time
train.text = train.text.apply(lambda x: x.lower())
 test.text = test.text.apply(lambda x: x.lower() )<string_transform>
CoefVar = pd.DataFrame(variation(CleanedNumeric),index=NumericVariables,columns=['CoefVar'] ).sort_values(by=['CoefVar'])
 CoefVar
%%time
train.text = train.text.apply(lambda x : " ".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in x.split(" ")]))
test.text = test.text.apply(lambda x : " ".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in x.split(" ")]))<categorify>
Categorical_NaN = DataRaw[CategoricalVariables].isnull().sum()
RowsCount = len(DataRaw.index)
 print("The percentage number of missing entries per variable: ", format(round(Categorical_NaN/RowsCount * 100)) )
vocab_train = build_vocab(train['text'])
 print("Glove : Train")
 oov_glove_train = check_coverage(vocab_train, embed_glove)
 vocab_test = build_vocab(test['text'])
 print("Glove : Test")
 oov_glove_test = check_coverage(vocab_test, embed_glove )<categorify>
LuxuriousCategoricalVariables = ['Alley','FireplaceQu','PoolQC','Fence','MiscFeature']
CategoricalVariables = [x for x in CategoricalVariables if x not in LuxuriousCategoricalVariables]
print(CategoricalVariables)
 print(LuxuriousCategoricalVariables )
def split_textnum(text):

 match = re.match(r"([a-z]+ )([0-9]+)", text, re.I)
 if match:
items = " ".join(list(match.groups()))
else:
match = re.match(r"([0-9]+ )([a-z]+)", text, re.I)
 if match:
items = " ".join(list(match.groups()))
else:
return text
return(items )<categorify>
CleanedCategorical= DataRaw[CategoricalVariables].fillna('Unknown' )
def clean_text(text):
text = re.sub(r"%20", " ", text)
 text = text.replace(r"@", " ")
 text = text.replace(r"
text = text.replace(r"'", " ")
 text = text.replace(r"\x89û_", " ")
 text = text.replace(r"??????", " ")
 text = text.replace(r"\x89ûò", " ")
 text = text.replace(r"16yr", "16 year")
 text = text.replace(r"re\x89û_", " ")
 text = text.replace(r"mh370", " ")
 text = text.replace(r"prebreak", "pre break")
 text = re.sub(r"\x89û", " ", text)
 text = re.sub(r"re\x89û", "re ", text)
 text = text.replace(r"nowplaying", "now playing")
 text = re.sub(r"\x89ûª", "'", text)
 text = re.sub(r"\x89û", " ", text)
 text = re.sub(r"\x89ûò", " ", text)
 text = re.sub(r"\x89Û_", "", text)
 text = re.sub(r"\x89ÛÒ", "", text)
 text = re.sub(r"\x89ÛÓ", "", text)
 text = re.sub(r"\x89ÛÏWhen", "When", text)
 text = re.sub(r"\x89ÛÏ", "", text)
 text = re.sub(r"China\x89Ûªs", "China's", text)
 text = re.sub(r"let\x89Ûªs", "let's", text)
 text = re.sub(r"\x89Û÷", "", text)
 text = re.sub(r"\x89Ûª", "", text)
 text = re.sub(r"\x89Û\x9d", "", text)
 text = re.sub(r"å_", "", text)
 text = re.sub(r"\x89Û¢", "", text)
 text = re.sub(r"\x89Û¢åÊ", "", text)
 text = re.sub(r"fromåÊwounds", "from wounds", text)
 text = re.sub(r"åÊ", "", text)
 text = re.sub(r"åÈ", "", text)
 text = re.sub(r"JapÌ_n", "Japan", text)
 text = re.sub(r"Ì©", "e", text)
 text = re.sub(r"å¨", "", text)
 text = re.sub(r"SuruÌ¤", "Suruc", text)
 text = re.sub(r"åÇ", "", text)
 text = re.sub(r"å£3million", "3 million", text)
 text = re.sub(r"åÀ", "", text)
 text = re.sub(r'http\S+', ' ', text)
 text = re.sub(r"ªs", " ", text)
 text = re.sub(r"ª", " ", text)
 text = re.sub(r"\x9d", " ", text)
 text = re.sub(r"ò", " ", text)
 text = re.sub(r"ªt", " ", text)
 text = re.sub(r"ó", " ", text)
 text = text.replace(r"11yearold", "11 year old")
 text = re.sub(r"typhoondevastated", "typhoon devastated", text)
 text = re.sub(r"bestnaijamade", "best nijamade", text)
 text = re.sub(r"gbbo", "The Great British Bake Off", text)
 text = re.sub(r"ï", "", text)
 text = re.sub(r"ïwhen", "when", text)
 text = re.sub(r"selfimage", "self image", text)
 text = re.sub(r"20150805", "2015 08 05", text)
 text = re.sub(r"20150806", "2015 08 06", text)
 text = re.sub(r"subreddits", "website for weird public sentiment", text)
 text = re.sub(r"disea", "chinese famous electronic company", text)
 text = re.sub(r"lmao", "funny", text)
 text = text.replace(r"companyse", "company")
 text = text.replace(r"worldnews", "world news")
 text = text.replace(r"animalrescue", "animal rescue")
 text = text.replace(r"freakiest", "freak")
 text = text.replace(r"irandeal", "iran deal")
 text = text.replace(r"directioners", "mentor")
 text = text.replace(r"justinbieber", "justin bieber")
 text = text.replace(r"okwx", "okay")
 text = text.replace(r"trapmusic", "trap music")
 text = text.replace(r"djicemoon", "music ice moon")
 text = text.replace(r"icemoon", "ice moon")
 text = text.replace(r"mtvhottest", "tv hottest")
 text = text.replace(r"rì©union", "reunion")
 text = text.replace(r"abcnews", "abc news")
 text = text.replace(r"tubestrike", "tube strike")
 text = text.replace(r"prophetmuhammad", "prophet muhammad muslim dharma")
 text = text.replace(r"chicagoarea", "chicago area")
 text = text.replace(r"yearold", "year old")
 text = text.replace(r"meatloving", "meat love")
 text = text.replace(r"standuser", "standard user")
 text = text.replace(r"pantherattack", "panther attack")
 text = text.replace(r"youngheroesid", "young hearos id")
 text = text.replace(r"idk", "i do not know")
 text = text.replace(r"usagov", "united state of america government")
 text = text.replace(r"injuryi", "injury")
 text = text.replace(r"summerfate", "summer fate")
 text = text.replace(r"kerricktrial", "kerrick trial")
 text = text.replace(r"viralspell", "viral spell")
 text = text.replace(r"collisionno", "collision")
 text = text.replace(r"socialnews", "social news")
 text = text.replace(r"nasahurricane", "nasa hurricane")
 text = text.replace(r"strategicpatience", "strategic patience")
 text = text.replace(r"explosionproof", "explosion proof")
 text = text.replace(r"selfies", "photo")
 text = text.replace(r"selfie", "photo")
 text = text.replace(r"worstsummerjob", "worst summer job")
 text = text.replace(r"realdonaldtrump", "real america president")
 text = text.replace(r"omfg", "oh my god")
 text = text.replace(r"japìn", "japan")
 text = text.replace(r"breakingnews", "breaking news")
 text = " ".join([split_textnum(word)for word in text.split(" ")])
 text = "".join([c if c not in string.punctuation else "" for c in text])
 text = ''.join(c for c in text if not c.isdigit())
 text = text.replace(r"÷", "")
 text = re.sub(' +', ' ', text)
 return text<string_transform>
LuxuriousCategorical = DataRaw[LuxuriousCategoricalVariables]
LuxuriousCategorical = pd.concat([LuxuriousCategorical, pd.DataFrame(DataRaw[LuxuriousCategoricalVariables].isnull().sum(axis = 1),
columns=['Luxurious_Features'])], axis=1,sort=False)
 LuxuriousCategorical['Luxurious_Features']=-LuxuriousCategorical['Luxurious_Features']+6
LuxuriousCategorical.head()
%%time
train.text = train.text.apply(lambda x : clean_text(x))
test.text = test.text.apply(lambda x : clean_text(x))
train.text = train.text.apply(lambda x : " ".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in x.split(" ")]))
test.text = test.text.apply(lambda x : " ".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in x.split(" ")]))<categorify>
CleanedCategorical = pd.merge(CleanedCategorical,
LuxuriousCategorical['Luxurious_Features'],
on='Id')
 CleanedCategorical.head()
vocab_train = build_vocab(train['text'])
 print("Glove : Train")
 oov_glove_train = check_coverage(vocab_train, embed_glove)
 vocab_test = build_vocab(test['text'])
 print("Glove : Test")
 oov_glove_test = check_coverage(vocab_test, embed_glove )<categorify>
Quality_map = {'NaN':1, 'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5}
CleanedCategorical['ExterQual'] = CleanedCategorical['ExterQual'].map(Quality_map)
 CleanedCategorical['ExterCond'] = CleanedCategorical['ExterCond'].map(Quality_map)
 CleanedCategorical['HeatingQC'] = CleanedCategorical['HeatingQC'].map(Quality_map)
 CleanedCategorical['KitchenQual'] = CleanedCategorical['KitchenQual'].map(Quality_map )
lemmatizer = WordNetLemmatizer()
train.text = train.text.apply(lambda x : "".join([lemmatizer.lemmatize(word)for word in x]))
test.text = test.text.apply(lambda x : "".join([lemmatizer.lemmatize(word)for word in x]))<categorify>
Quality2_map = {'NaN':1, 'NA':1,'Po':2,'Fa':3,'TA':4,'Gd':5,'Ex':6}
CleanedCategorical['BsmtQual'] = CleanedCategorical['BsmtQual'].map(Quality2_map)
 CleanedCategorical['BsmtCond'] = CleanedCategorical['BsmtCond'].map(Quality2_map)
 CleanedCategorical['GarageQual'] = CleanedCategorical['GarageQual'].map(Quality2_map)
 CleanedCategorical['GarageCond'] = CleanedCategorical['GarageCond'].map(Quality2_map)
 Quality3_map = {'NaN':1, 'NA':1,'No':2,'Mn':3,'Av':4,'Gd':5}
CleanedCategorical['BsmtExposure'] = CleanedCategorical['BsmtExposure'].map(Quality3_map)
 Quality4_map = {'NaN':1, 'NA':1,'Unf':2,'LwQ':3,'Rec':4,'BLQ':5,'ALQ':7,'GLQ':7}
CleanedCategorical['BsmtFinType1'] = CleanedCategorical['BsmtFinType1'].map(Quality4_map)
 CleanedCategorical['BsmtFinType2'] = CleanedCategorical['BsmtFinType2'].map(Quality4_map)
 Quality5_map = {'NaN':1, 'Sal':1,'Sev':2,'Maj2':3,'Maj1':3,'Mod':4,'Min1':5,'Min2':5,'Typ':6}
CleanedCategorical['Functional'] = CleanedCategorical['Functional'].map(Quality5_map)
 Quality6_map = {'NaN':1, 'NA':1,'Unf':2,'RFn':3,'Fin':4}
CleanedCategorical['GarageFinish'] = CleanedCategorical['GarageFinish'].map(Quality6_map)
 OrdinalVariables = ['ExterQual','ExterCond','HeatingQC','KitchenQual','BsmtQual','BsmtCond',
'GarageQual','GarageCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
'Functional','GarageFinish']
CleanedOrdinal = CleanedCategorical[OrdinalVariables]
CleanedOrdinal = pd.merge(CleanedOrdinal,
LuxuriousCategorical['Luxurious_Features'],
on='Id')
 OrdinalVariables = ['ExterQual','ExterCond','HeatingQC','KitchenQual','BsmtQual','BsmtCond',
'GarageQual','GarageCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
'Functional','GarageFinish','Luxurious_Features']
CleanedOrdinal= CleanedOrdinal[OrdinalVariables].fillna(1)
 CleanedOrdinal.head()
vocab_train = build_vocab(train['text'])
 print("Glove : Train")
 oov_glove_train = check_coverage(vocab_train, embed_glove)
 vocab_test = build_vocab(test['text'])
 print("Glove : Test")
 oov_glove_test = check_coverage(vocab_test, embed_glove )<set_options>
print(CleanedOrdinal['BsmtQual'].loc[[18]] )
del oov_glove_test
del embed_glove
gc.collect()<prepare_x_and_y>
NominalVariables = [x for x in CategoricalVariables if x not in OrdinalVariables]
AllLevelsPerVar = CleanedCategorical[NominalVariables].nunique()
AllLevels = CleanedCategorical[NominalVariables].nunique().sum()
print(AllLevelsPerVar)
 print("Number of all levels coming from nominal variables: " + str(AllLevels))
class ClassificationReport(Callback):
def __init__(self, train_data=() , validation_data=()):
super(Callback, self ).__init__()
self.X_train, self.Y_train = train_data
self.X_val, self.Y_val = validation_data
self.train_precision_score = []
self.train_recall_score = []
self.train_f1_score = []
self.val_precision_score = []
self.val_recall_score = []
self.val_f1_score = []
def on_epoch_end(self, epoch, logs={}):
train_prediction = np.round(self.model.predict(self.X_train, verbose=0))
train_precision = precision_score(self.Y_train, train_prediction, average='macro')
 train_recall = recall_score(self.Y_train, train_prediction, average='macro')
 train_f1 = f1_score(self.Y_train, train_prediction, average='macro')
 self.train_precision_score.append(train_precision)
 self.train_recall_score.append(train_recall)
 self.train_f1_score.append(train_f1)
 val_prediction = np.round(self.model.predict(self.X_val, verbose=0))
val_precision = precision_score(self.Y_val, val_prediction, average='macro')
 val_recall = recall_score(self.Y_val, val_prediction, average='macro')
 val_f1 = f1_score(self.Y_val, val_prediction, average='macro')
 self.val_precision_score.append(val_precision)
 self.val_recall_score.append(val_recall)
 self.val_f1_score.append(val_f1)
 print('
Epoch - {} - Training Precision - {:.6} - Training Recall - {:.6} - Training F1-Score - {:.6}'.format(
 epoch+1, train_precision, train_recall, train_f1))
print('
Epoch - {} - Validation Precision - {:.6} - Validation Recall - {:.6} - Validation F1-Score - {:.6}'.format(
 epoch+1, val_precision, val_recall, val_f1)) 
<install_modules>
CleanedCategoricalDummy = pd.get_dummies(CleanedCategorical[NominalVariables], columns=NominalVariables)
 CleanedCategoricalDummy.head()
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
bert_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1', trainable=True )<load_pretrained>
CleanedTotal = pd.merge(CleanedNumeric,
CleanedOrdinal,
on='Id')
 CleanedTotal = pd.merge(CleanedTotal,
CleanedCategoricalDummy,
on='Id')
 CleanedTotal.head()
class BertTraining:
def __init__(self, bert_layer, fold_k=2, dropout=0.2, max_seq_len=160, lr=0.0001, epochs=15, batch_size=32):
self.fold_k = fold_k
self.bert_layer = bert_layer
self.max_seq_len = max_seq_len
self.lr = lr
self.dropout = dropout
self.epochs = epochs
self.batch_size = batch_size
self.models = []
self.scores = {}
vocab_file = self.bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = self.bert_layer.resolved_object.do_lower_case.numpy()
self.tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 def bert_encode(self, texts):
all_token_ids = []
all_masks = []
all_segments = []
for text in texts:
text = self.tokenizer.tokenize(text)
 text = text[: self.max_seq_len-2]
input_seqence = ['[CLS]'] + text + ['[SEP]']
padding_length = self.max_seq_len - len(input_seqence)
 tokens = self.tokenizer.convert_tokens_to_ids(input_seqence)
 tokens = tokens + [0]*padding_length
pad_masks = [1]*len(input_seqence)+ [0]*padding_length
segment_ids = [0]*self.max_seq_len
all_token_ids.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_token_ids), np.array(all_masks), np.array(all_segments)
 def bert_model(self):
input_token_id = Input(shape=(self.max_seq_len,), dtype=tf.int32, name='input_token_id')
 input_mask = Input(shape=(self.max_seq_len,), dtype=tf.int32, name='input_mask')
 input_segment = Input(shape=(self.max_seq_len,), dtype=tf.int32, name='input_segment')
 _, sequence_output = self.bert_layer([input_token_id, input_mask, input_segment])
 clf_output = sequence_output[:, 0, :]
if self.dropout == 0:
output = Dense(1, activation='sigmoid' )(clf_output)
 else:
dropout = Dropout(self.dropout )(clf_output)
 output = Dense(1, activation='sigmoid' )(dropout)
 model = Model(inputs=[input_token_id, input_mask, input_segment], outputs=output)
 optimizer = SGD(learning_rate=self.lr, momentum=0.8)
 model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
 return model
def train_model(self, X):
skf = StratifiedKFold(n_splits=self.fold_k, random_state=SEED, shuffle=True)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X.text, X.text)) :
print("
Fold {}
".format(fold))
X_Train_Encoded = self.bert_encode(X.loc[train_idx, 'text'].str.lower())
 Y_Train = X.loc[train_idx, 'target']
X_Val_Encoded = self.bert_encode(X.loc[val_idx, 'text'].str.lower())
 Y_Val = X.loc[val_idx, 'target']
metrics = ClassificationReport(train_data=(X_Train_Encoded, Y_Train), validation_data=(X_Val_Encoded, Y_Val))
model = self.bert_model()
model.fit(X_Train_Encoded, Y_Train, epochs=self.epochs, batch_size=self.batch_size, callbacks=[metrics],
validation_data=(X_Val_Encoded, Y_Val), verbose=1)
 self.models.append(model)
 self.scores[fold] = {
'train' : {
'precision' : metrics.train_precision_score,
'recall' : metrics.train_recall_score,
'f1_score': metrics.train_f1_score
},
'validation' : {
'precision' : metrics.val_precision_score,
'recall' : metrics.val_recall_score,
'f1_score': metrics.val_f1_score
}
}
def plot_learning_curve(self):
fig, axes = plt.subplots(nrows=self.fold_k, ncols=2, figsize=(20, self.fold_k * 6), dpi=100)
 for i in range(self.fold_k):
sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[i].history.history['val_accuracy'], ax=axes[i][0], label='val_accuracy')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['precision'], ax=axes[i][0], label='val_precision')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['recall'], ax=axes[i][0], label='val_recall')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['f1_score'], ax=axes[i][0], label='val_f1')
 axes[i][0].legend()
axes[i][0].set_title('Fold {} Validation Classification Report'.format(i), fontsize=14)
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['loss'], ax=axes[i][1], label='train_loss')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['val_loss'], ax=axes[i][1], label='val_loss')
 axes[i][1].legend()
axes[i][1].set_title('Fold {} Train / Validation Loss'.format(i), fontsize=14)
 for j in range(2):
axes[i][j].set_xlabel('Epoch', size=12)
 axes[i][j].tick_params(axis='x', labelsize=12)
 axes[i][j].tick_params(axis='y', labelsize=12)
 plt.show()
def predict(self, X_test):
X_test_encode = self.bert_encode(X_test.text.str.lower())
 Y_pred = np.zeros(( X_test_encode[0].shape[0], 1))
for model in self.models:
Y_pred += model.predict(X_test_encode)/len(self.models)
 return Y_pred<train_model>
Target = ['IsTrain','SalePrice']
AllVariables = list(CleanedTotal.columns)
 NumericVariablesNoTarget = [x for x in NumericVariables if x not in Target]
AllVariablesNoTarget = [x for x in AllVariables if x not in Target]
ScaledCleanedTotal = CleanedTotal
ScaledCleanedTotal[NumericVariablesNoTarget] = minmax_scaling(CleanedTotal, columns=NumericVariablesNoTarget)
 ScaledCleanedTotal.head()
SEED = 42
clf = BertTraining(bert_layer, fold_k=3, dropout=0.5, max_seq_len=140, lr=0.0001, epochs=20, batch_size=64)
 clf.train_model(train )<predict_on_test>
DataTrain=CleanedTotal[ScaledCleanedTotal.IsTrain==1]
DataTest=CleanedTotal[ScaledCleanedTotal.IsTrain==0]
prediction = clf.predict(test)
 prediction<prepare_output>
selector_F = SelectKBest(f_classif, k=VarNo)
 Selected_F = selector_F.fit_transform(DataTrain[AllVariablesNoTarget], DataTrain['SalePrice'])
 SelectedOrdered_F = pd.DataFrame(selector_F.inverse_transform(Selected_F), index=DataTrain.index, columns=AllVariablesNoTarget)
 SelectedOrdered_F.head()
prediction = np.where(prediction < 0.5, 0, 1)
 prediction<prepare_output>
L1_par = 0.22
LogisReg = LogisticRegression(C=L1_par, penalty="l1", solver='liblinear', random_state=RandState ).fit(DataTrain[AllVariablesNoTarget], DataTrain['SalePrice'])
 LASSO = SelectFromModel(LogisReg, prefit=True)
 LASSO_transform = LASSO.transform(DataTrain[AllVariablesNoTarget])
 SelectedOrdered_LASSO = pd.DataFrame(LASSO.inverse_transform(LASSO_transform), index=DataTrain[AllVariablesNoTarget].index,columns=DataTrain[AllVariablesNoTarget].columns)
 SelectedVariables_LASSO = list(SelectedOrdered_LASSO.columns[SelectedOrdered_LASSO.var() > 0])
 DataTrain[SelectedVariables_LASSO].head()
result = pd.DataFrame()
result["id"] = test['id']
result["target"] = np.squeeze(prediction)
 result.head()<count_values>
SelectedVariables = pd.DataFrame(SelectedVariables_F,columns=['F variables'] ).sort_values(by=['F variables'])
 SelectedVariables['LASSO variables'] = SelectedVariables_LASSO
SelectedVariables
result.target.value_counts()<save_to_csv>
Target= DataTrain['SalePrice']
DataTrainFinal = DataTrain.drop(['SalePrice','IsTrain'],axis=1)
 DataTestFinal = DataTest.drop(['SalePrice','IsTrain'],axis=1)
 x_train,x_test,y_train,y_test = train_test_split(DataTrainFinal,Target,test_size=0.2,random_state=0)
 print("Train set contains: " + str(x_train.shape[1])+ " variables in " + str(x_train.shape[0])+ " rows.")
 print("Test set contains: " + str(x_test.shape[1])+ " variables in " + str(x_test.shape[0])+ " rows." )
result.to_csv('submission.csv', index=False )<import_modules>
Predictions = pd.DataFrame(y_test,columns=['SalePrice'])
 Predictions['ModelAverage'] = ModelAverage
ScoreAverage = math.sqrt(metrics.mean_squared_error(y_test, Predictions['ModelAverage']))
print('Average: RMSE = ' + str(ScoreAverage))
Predictions.head()
import numpy as np
import pandas as pd
import os
import seaborn as sns
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
import string, re
from bs4 import BeautifulSoup
from wordcloud import WordCloud
from keras.preprocessing import text, sequence
from nltk.tokenize.toktok import ToktokTokenizer
from sklearn.model_selection import train_test_split
import keras
from keras.models import Sequential
from keras.layers import Dense,Embedding,LSTM,Dropout, Bidirectional
from keras.callbacks import ReduceLROnPlateau
import tensorflow as tf<define_variables>
NormalReg = TweedieRegressor(power=0, alpha=0, link='identity')
 PoissonReg = TweedieRegressor(power=1, alpha=0, link='log')
 GammaReg = TweedieRegressor(power=2, alpha=0, link='log')
 NormalReg.fit(x_train[SelectedVariables_LASSO],y_train)
 PoissonReg.fit(x_train[SelectedVariables_LASSO],y_train)
 GammaReg.fit(x_train[SelectedVariables_LASSO],y_train)
 PredictNormalReg = NormalReg.predict(x_test[SelectedVariables_LASSO])
 PredictPoissonReg = PoissonReg.predict(x_test[SelectedVariables_LASSO])
 PredictGammaReg = GammaReg.predict(x_test[SelectedVariables_LASSO] )
np.random.seed(1)
 tf.random.set_seed(1 )<load_from_csv>
print('Normal Dist: RMSE = ' + str(math.sqrt(metrics.mean_squared_error(y_test, PredictNormalReg))))
print('Poisson Dist: RMSE = ' + str(math.sqrt(metrics.mean_squared_error(y_test, PredictPoissonReg))))
print('Gamma Dist: RMSE = ' + str(math.sqrt(metrics.mean_squared_error(y_test, PredictGammaReg))))
print('Poisson wins')
 ScoreGLM = math.sqrt(metrics.mean_squared_error(y_test, PredictPoissonReg))
Predictions['GLM Poisson'] = PredictGammaReg
Predictions.head()
train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv', index_col='id')
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv', index_col='id' )<string_transform>
RandomForest = RandomForestRegressor(random_state=RandState)
 RandomForest.fit(x_train, y_train)
 PredictRandomForest = RandomForest.predict(x_test)
 ScoreRandomForest = math.sqrt(metrics.mean_squared_error(y_test, PredictRandomForest))
print('Random Forest: RMSE = ' + str(ScoreRandomForest))
stop = set(stopwords.words('english'))
punctuation = list(string.punctuation)
 stop.update(punctuation )<string_transform>
Predictions['Random Forest'] = PredictRandomForest
Predictions.head()
def strip_html(text):
soup = BeautifulSoup(text, "html.parser")
 return soup.get_text()
def remove_between_square_brackets(text):
return re.sub('\[[^]]*\]', '', text)
 def remove_url(text):
return re.sub(r'http\S+', '', text)
 def add_space(text):
return re.sub('%20', ' ', text)
 def remove_stopwords(text):
final_text = []
for i in text.split() :
if i.strip().lower() not in stop:
final_text.append(i.strip())
 return " ".join(final_text)
 def denoise_text(text):
text = strip_html(text)
 text = remove_between_square_brackets(text)
 text = add_space(text)
 text = remove_url(text)
 text = remove_stopwords(text)
 return text 
<feature_engineering>
XBoost_1 =XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0,
importance_type='gain', learning_rate=0.008, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=4000, objective='reg:linear',
reg_alpha=0.5, reg_lambda=0.5, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 XBoost_1.fit(x_train, y_train)
 PredictXBoost = XBoost_1.predict(x_test)
 print('Extreme boosting for first try: RMSE = ' + str(math.sqrt(metrics.mean_squared_error(y_test, PredictXBoost))))
def preprocess_df(df):
df = df.fillna("")
 df['text'] = df['location'] + " " + df['keyword'] + " " + df['text']
del df['keyword']
del df['location']
df['text'] = df['text'].apply(denoise_text)
 return df<drop_column>
XBoost_final =XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0,
importance_type='gain', learning_rate=0.0081, max_delta_step=0,
max_depth=4, min_child_weight=1.8, n_estimators=4200, objective='reg:linear',
reg_alpha=0.6, reg_lambda=0.51, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 XBoost_final.fit(x_train, y_train)
 PredictXBoost_final = XBoost_final.predict(x_test)
 ScoreXBoost = math.sqrt(metrics.mean_squared_error(y_test, PredictXBoost_final))
print('Final extreme boosting: RMSE = ' + str(ScoreXBoost))
train_df = preprocess_df(train_df)
 test_df = preprocess_df(test_df )<split>
Predictions['Extreme boosting'] = PredictXBoost_final
Predictions.head()
X_train, X_dev, y_train, y_dev = train_test_split(train_df.text.values, train_df.target.values )<define_variables>
FinalRMSE = pd.DataFrame([[ScoreAverage],[ScoreGLM],[ScoreRandomForest],[ScoreXBoost]],columns=["RMSE"],index=['Expected value','GLM Poisson','Random Forest','Extreme boosting'])
 FinalRMSE
max_features = 10000
max_len = 300<train_model>
XBoost_final.fit(DataTrainFinal, Target)
 FinalPrediction = XBoost_final.predict(DataTestFinal )
<string_transform><EOS> 
Submission = pd.DataFrame({'Id': test.index, 'SalePrice': FinalPrediction})
 Submission.to_csv('Submission.csv', index=False)
 Submission