Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<string_transform>
from sklearn.ensemble import\
GradientBoostingRegressor, RandomForestRegressor, ExtraTreesRegressor
callbacks, layers, optimizers, regularizers, Sequential
sns.set(style="white")
 warnings.filterwarnings("ignore")
 %matplotlib inline
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return round(float(len(c)) /(len(a)+ len(b)- len(c)) , 4 )<find_best_params>
def reload(df="train.csv", dropped_columns=['Id'], msg=True):
data_path = ".. /input/house-prices-advanced-regression-techniques/"
if df == "all_data":
train = pd.read_csv(data_path + "train.csv")
 test = pd.read_csv(data_path + "test.csv")
 data = train.drop(columns="SalePrice" ).append(test)
 else:
data = pd.read_csv(data_path + df)
 if msg:
print(df + " loaded successfully!")
 return data.reset_index(drop=True ).drop(columns=dropped_columns)
 train = reload()
test = reload("test.csv")
 all_data = reload("all_data" )
results_jaccard = []
for index, row in train.iterrows() :
sentence1 = row.keyword
sentence2 = row.text
jaccard_score = jaccard(sentence1, sentence2)
 results_jaccard.append([sentence1, sentence2, jaccard_score] )<create_dataframe>
train = reload()
class PreProcessor(BaseEstimator, TransformerMixin):
def __init__(self, old_new, numeric_to_object):
self.numeric_to_object = numeric_to_object
self.old_new = old_new
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X = X.copy()
sold_time = X[["MoSold", "YrSold"]].astype(str ).agg("/".join, axis=1)
 sold_time = pd.to_datetime(sold_time)
 for old in self.old_new:
X[self.old_new[old]] =(sold_time - pd.to_datetime(X[old], format="%Y")) \
/ pd.Timedelta(days=365.25)
 X[self.numeric_to_object] = X[self.numeric_to_object].astype(str)
 X.drop(columns=list(self.old_new.keys()), inplace=True)
 return X
class RelationImputer(BaseEstimator, TransformerMixin):
def __init__(self, related, threshold=None, strategy="most_frequent", fill_value=None
, missing_num=np.nan, missing_obj=np.nan
, final_num=0, final_obj="None"):
self.related = related if isinstance(related[0], list)else [related]
self.threshold = threshold
if strategy == 'constant':
self.fill_value = fill_value
elif strategy not in ['mean', 'median', 'most_frequent']:
raise Exception("Wrong strategy type! --{}".format(strategy))
self.strategy = strategy
self.missings = [missing_num, missing_obj, final_num, final_obj]
self.missing_num, self.missing_obj, self.final_num, self.final_obj =\
self.missings
def fit(self, X, y=None):
if self.strategy == 'constant':
self.statistics_ = pd.Series(data=self.fill_value, index=X.columns)
 else:
self.statistics_ = X[X.select_dtypes(object ).columns].mode().T[0]
num_cols = X.select_dtypes("number" ).columns
if self.strategy == "most_frequent":
self.statistics_ = self.statistics_.append(X[num_cols].mode().T[0])
 elif self.strategy in ["mean", "median"]:
strat = eval("X[num_cols].{}() ".format(self.strategy))
self.statistics_ = self.statistics_.append(strat)
 return self
def _indexes(self, X, cols):

 if not self.threshold:
l = len(cols)
 elif(self.threshold > 0)&(self.threshold < 1):
l = round(len(cols)*self.threshold)
 else:
l = self.null_threshold
missing = X[cols].isin(self.missings ).sum(axis=1)
 index =(missing > 0)&(missing < l)
 return index
def transform(self, X, y=None):
X = X.copy()
obj_cols = list(X.select_dtypes(object ).columns)
 num_cols = list(X.select_dtypes("number" ).columns)
 obj_len = len(obj_cols)
 num_len = len(num_cols)
 to_replace = pd.Series(
 data=[self.missing_obj]*obj_len + [self.missing_num]*num_len
, index=obj_cols + num_cols)
 final_value = pd.Series(
 data=[self.final_obj]*obj_len + [self.final_num]*num_len
, index=obj_cols + num_cols)
 for cols in self.related:
index = self._indexes(X, cols)
 X.loc[index, cols] = X.loc[index, cols].replace(
 to_replace=to_replace, value=self.statistics_[cols])
 X[cols] = X[cols].replace(to_replace=to_replace, value=final_value)
 flat_related = [c for sublist in self.related for c in sublist]
remaining = [c for c in X.columns if c not in flat_related]
X[remaining] = X[remaining].replace(to_replace=to_replace
, value=self.statistics_[remaining])
 return X
RELATED = [["Alley"]
, ["Condition1", "Condition2"]
, ["Exterior2nd"]
, ["BsmtQual", "BsmtCond", "BsmtExposure", "BsmtFinType1"
, "BsmtFinSF1", "BsmtUnfSF", "TotalBsmtSF"
, "BsmtFullBath", "BsmtHalfBath"]
, ["BsmtFinType2", "BsmtFinSF2"]
, ["Heating", "HeatingQC"]
, ["KitchenQual", "KitchenAbvGr"]
, ["FireplaceQu", "Fireplaces"]
, ["GarageType", "YrSinceGarageBlt", "GarageFinish"
, "GarageCars", "GarageArea", "GarageQual", "GarageCond"]
, ["MasVnrType", "MasVnrArea"]
, ["PoolArea", "PoolQC"]
, ["Fence"]
, ["MiscFeature", "MiscVal"]
]
nonrelated = [
"MSSubClass", "MSZoning", "LotArea", "Street", "Alley"
, "LotShape", "LandContour", "Utilities", "LotConfig", "LandSlope"
, "Neighborhood", "Condition1", "Condition2", "BldgType", "HouseStyle"
, "OverallQual", "OverallCond", "RoofStyle", "RoofMatl", "Exterior1st"
, "Exterior2nd", "ExterQual", "ExterCond", "Foundation", "CentralAir"
, "Electrical", "1stFlrSF", "2ndFlrSF", "LowQualFinSF", "GrLivArea"
, "BedroomAbvGr", "TotRmsAbvGrd", "Functional", "PavedDrive"
, "WoodDeckSF", "Fence", "MoSold", "YrSold", "SaleType", "LotFrontage"
, "SaleCondition", "Age", "YrSinceRemod", "YrSinceGarageBlt"
]
flat_related = [c for cols in RELATED for c in cols]
old_new = {"YearBuilt":"Age", "YearRemodAdd":"YrSinceRemod"
, "GarageYrBlt": "YrSinceGarageBlt"}
num_to_obj = ["MSSubClass", "MoSold", "YrSold"]
categorical_columns = list(all_data.select_dtypes(object ).columns)+ num_to_obj
cat_to_transform = [c for c in categorical_columns if c in flat_related]
PreI = PreProcessor(old_new, num_to_obj)
 RI = RelationImputer(related = RELATED, final_obj="None")
 train = PreI.fit_transform(train)
 train = RI.fit_transform(train)
 numerical_columns = [col for col in all_data.columns if col not in categorical_columns]
jaccard_score = pd.DataFrame(results_jaccard, columns=['keyword', 'text', 'jaccard_score'] )<string_transform>
train.BsmtHalfBath.value_counts()
stopwords_en = stopwords.words('english' )<categorify>
train["BsmtFinSF1_UnfSF"] = train.BsmtFinSF1 + train.BsmtUnfSF
train["BsmtFinSF"] = train.BsmtFinSF1 + train.BsmtFinSF2
train["BsmtBathPerSF"] =(train.BsmtFullBath + train.BsmtHalfBath/2)/ train.TotalBsmtSF
train["BsmtBathPerFinSF"] =(train.BsmtFullBath + train.BsmtHalfBath/2)\
/(train.TotalBsmtSF - train.BsmtUnfSF )
def unicode_to_ascii(s):
return ''.join(c for c in unicodedata.normalize('NFD', s)if unicodedata.category(c)!= 'Mn' )<string_transform>
Score_map = {"Ex":5, "Gd":4, "TA":3, "Fa":2, "None":1, "Po":0}
BsmtFin_map = {"GLQ":5, "ALQ":4, "BLQ":3, "Rec":2, "LwQ":1, "Unf":0, "None":0}
BsmtEx_map = {"Gd":3, "Av":2, "Mn":1, "No":0, "None":0}
train["BsmtFinType1"] = train.BsmtFinType1.replace(BsmtFin_map)
 train["BsmtFinType2"] = train.BsmtFinType2.replace(BsmtFin_map)
 train["BsmtFinTypeAvg"] =(train["BsmtFinType1"] + train["BsmtFinType2"])/ 2
train["BsmtFinTypeProd"] = train["BsmtFinType1"] * train["BsmtFinType2"]
twitter_p.set_options(twitter_p.OPT.URL)
 def preprocess_sentence(w):
w = twitter_p.clean(w)
 w = unicode_to_ascii(w.lower().strip())
 w = re.sub(r"([@
w = re.sub(r'[" "]+', " ", w)
 w = re.sub(r"[^a-zA-Z@
w = ' '.join([word for word in w.split(' ')if word not in stopwords_en])
 w = w.rstrip().strip()
return w<feature_engineering>
test_features = [["BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF"
,"BsmtFinType1", "BsmtFinType2", "BsmtUnfExp"]]
train['text'] = train['text'].apply(func=preprocess_sentence)
 train['keyword'] = train['keyword'].apply(func=preprocess_sentence)
 print(train.head(10))<feature_engineering>
test_features.append(
 ["AllBath", "BsmtHalfBath", "BsmtFullBath", "HalfBath", "FullBath"]
)
test['text'] = test['text'].apply(func=preprocess_sentence)
 test['keyword'] = test['keyword'].apply(func=preprocess_sentence)
 print(test.head(10))<feature_engineering>
test_features.append(["GarageCars", "GarageArea"] )
text_list = np.stack([*train['text'], *train['keyword'], *test['text'], *test['keyword']])
 tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')
 tokenizer.fit_on_texts(text_list)
 print(len(tokenizer.word_index))<load_pretrained>
train.KitchenAbvGr.value_counts()
glove = np.load('/kaggle/input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', allow_pickle=True)
<define_variables>
test_features.append(["KitchenAbvGr"] )
input_vocab_size = len(tokenizer.word_index)+ 3
d_model = 300<define_search_model>
train.PoolArea.value_counts()
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english" )<feature_engineering>
test_features.append(["PoolArea", "PoolQC"] )
words = glove.keys()
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts )<categorify>
test_features.append(["MiscFeature", "MiscVal"] )
def create_embedding_matrix(vectors, to_word_it, inp_vocab_size, d_m, lemma_dict):
no_in_vocab = []
matrix = np.random.uniform(low=-1, high=1, size=(inp_vocab_size, d_m))
unknown_vector = np.zeros(( d_m,), dtype=np.float32)- 1
for key, index in to_word_it:
word = key
try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = key.lower()
try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = key.upper()
try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = key.capitalize()
try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = ps.stem(key)
 try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = lc.stem(key)
 try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = sb.stem(key)
 try:
matrix[index] = vectors[word]
continue
except KeyError:
''
try:
word = lemma_dict[key]
matrix[index] = vectors[word]
continue
except KeyError:
''
if len(key)> 1:
word = correction(key)
 try:
matrix[index] = vectors[word]
continue
except KeyError:
''
try:
matrix[index] = vectors[word]
except KeyError:
matrix[index] = unknown_vector
no_in_vocab.append(word)
 del vectors
print("no_in_vocab size:", len(no_in_vocab))
print("embedding_matrix shape:", matrix.shape)
 return matrix, no_in_vocab<feature_engineering>
print(train.Utilities.value_counts() , end="
")
 print(train.Electrical.value_counts() )
print("Spacy NLP...")
 text_list = pd.concat([train['text'], test['text']])
 print(len(tokenizer.word_index))
nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()<drop_column>
class FeatureEngineer(BaseEstimator, TransformerMixin):

 Score_map = {"Ex":5, "Gd":4, "TA":3, "Fa":2, "None":1, "Po":0}
BsmtFin_map = {"GLQ":5, "ALQ":4, "BLQ":3, "Rec":2, "LwQ":1, "Unf":0, "None":0}
BsmtEx_map = {"Gd":3, "Av":2, "Mn":1, "No":0, "None":0}
GargFin_map = {"Fin": 3, "RFn": 2, "Unf": 1, "None": 0}
GargType_map = {"BuiltIn": "Attchd", "Basment": "Attchd"
, "CarPort": "Detchd",}
StreetAlley_map = {"Pave":2, "Grvl":1, "None":0}
CentralAir_map = {"Y":1, "N":0}
LotShape_map = {"Reg":0, "IR1":1, "IR2":2, "IR3":3}
LandSlope_map = {"Gtl":0, "Mod":1, "Sev":2}
Electrical_map = {"SBrkr":3, "FuseA":2, "FuseF":1, "FuseP":0}
ElectricMix_map = {"Mix":1}
Functional_map = {"Typ":5, "Min1":4, "Min2":4, "Mod":3
, "Maj1":2, "Maj2":2, "Sev":1, "Sal":0}
PavedDrive_map = {"Y":2, "P":1, "N":0}
Fence_map = {"GdPrv":4, "MnPrv":3, "GdWo":2, "MnWw":1, "None":0}
CondStr_map = {"Feedr":1, "Artery":2}
CondNSRR_map = {"RRNn":1, "RRAn":2}
CondEWRR_map = {"RRNe":1, "RRAe":2}
CondOff_map = {"PosN":1, "PosA":2}
to_drop_ = ["MSSubClass", "GarageCond", "1stFlrSF", "2ndFlrSF"]
test_cols_ = [["BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF"
,"BsmtFinType1", "BsmtFinType2", "BsmtUnfExp"]
, ["AllBath", "BsmtFullBath", "BsmtHalfBath"
, "FullBath", "HalfBath"]
, ["GarageCars", "GarageArea"]
, ['KitchenAbvGr']
, ["PoolArea", "PoolQC"]
, ["MiscFeature", "MiscVal"]
, ["Utilities", "Electrical", "OverallCond", "LowQualFinSF"]
, ["Age", "YrSinceRemod"]
]
def __init__(self, execute=True
, cols_to_keep_or_drop=None
, keep_or_drop=None):

 self.execute = execute
self.cols_to_keep_or_drop = cols_to_keep_or_drop
self.keep_or_drop=keep_or_drop
def map_val(self, x, mp):
return mp[x] if x in mp else 0
def map_s(self, s, mp):
return s.apply(lambda x: self.map_val(x, mp))
def _feature_engineer(self, X):
X = X.copy()
for c in ["BsmtQual", "BsmtCond", "GarageQual"
, "HeatingQC", "KitchenQual", "FireplaceQu"
, "PoolQC", "ExterQual", "ExterCond"]:
X[c] = self.map_s(X[c], self.Score_map)
 for c, mpping in(
 ["BsmtFinType1", self.BsmtFin_map]
, ["BsmtFinType2", self.BsmtFin_map]
, ["BsmtExposure", self.BsmtEx_map]
, ["GarageFinish", self.GargFin_map]
, ["Alley", self.StreetAlley_map]
, ["Street", self.StreetAlley_map]
, ["CentralAir", self.CentralAir_map]
, ["LotShape", self.LotShape_map]
, ["LandSlope", self.LandSlope_map]
, ["Electrical", self.Electrical_map]
, ["Functional", self.Functional_map]
, ["PavedDrive", self.PavedDrive_map]
, ["Fence", self.Fence_map] 
):
if c == "Electrical":
X[c + "Mix"] = self.map_s(X[c], self.ElectricMix_map)
 X[c] = self.map_s(X[c], mpping)
 X["GarageType"] = X.GarageType.replace(self.GargType_map)
 for c, m in [
["Str", self.CondStr_map]
, ["NSRR", self.CondNSRR_map]
, ["EWRR", self.CondEWRR_map]
, ["Offsite", self.CondOff_map]
]:
cond1 = self.map_s(X.Condition1, m)
 cond2 = self.map_s(X.Condition2, m)
 X[c] = pd.concat([cond1, cond2], axis=1 ).max(axis=1)
 X.drop(columns = ["Condition1", "Condition2"], inplace=True)
 X["BsmtHfBthDummy"] = np.where(X.BsmtHalfBath == 0, 0, 1)
 X["BsmtFinType_SF1"] = X.BsmtFinType1 * X.BsmtFinSF1
X["BsmtFinType_SF2"] = X.BsmtFinType2 * X.BsmtFinSF2
X["BsmtUnfExp"] = X.BsmtUnfSF * X.BsmtExposure
X["AllBath"] = X.BsmtFullBath + X.FullBath +\ 
(X.BsmtHalfBath + X.HalfBath)/2
X["KitchenScore"] = X.KitchenQual * X.KitchenAbvGr
X.drop(columns = self.to_drop_, inplace=True)
 return X
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
if self.execute:
X = self._feature_engineer(X)
 if self.cols_to_keep_or_drop:
if self.keep_or_drop == "keep":
X = X[self.cols_to_keep_or_drop]
elif self.keep_or_drop == "drop":
X = X.drop(columns=self.cols_to_keep_or_drop, inplace=True)
 return X
class CustomizedOHE(BaseEstimator, TransformerMixin):

 def __init__(self, columns="categorical"
, OHE=OneHotEncoder(handle_unknown="ignore")
 , keep_header=True):
self.columns = columns
self.OHE = OHE
self.keep_header = keep_header
def fit(self, X, y=None):
self.header_ = list(X.columns)
 if self.columns == "categorical":
self.transformed_cols_ = X.select_dtypes(object ).columns
else:
self.transformed_cols_ = self.columns
self.not_transformed_cols_ = [
c for c in self.header_ if c not in self.transformed_cols_
]
if self.keep_header:
self.CT = ColumnTransformer(
 [("OHE", self.OHE, self.transformed_cols_)]
, remainder="passthrough"
, sparse_threshold=0 
)
 self.CT.fit(X)
 if len(self.transformed_cols_)> 0:
self.transformed_cols_new_ = \
self.CT.transformers_[0][1].get_feature_names()
else:
self.transformed_cols_new_ = []
self.header_new_ = np.concatenate(
(self.transformed_cols_new_,
self.not_transformed_cols_)
)
 else:
self.CT = ColumnTransformer(
 [("OHE", self.OHE, self.transformed_cols_)]
, remainder="passthrough" 
)
 self.CT.fit(X)
 return self
def transform(self, X, y=None):
transformed = self.CT.transform(X)
 self.a_ = transformed
if self.keep_header:
return pd.DataFrame(data=transformed, columns=self.header_new_)
 else:
return transformed
class CustomizedScaler(BaseEstimator, TransformerMixin):

 def __init__(self, scaler=StandardScaler() , keep_header=True):
self.scaler,self.keep_header = scaler, keep_header
def fit(self, X, y=None):
self.header = list(X.columns)
 self.scaler.fit(X, y)
 return self
def transform(self, X, y=None):
transformed = self.scaler.transform(X)
 if self.keep_header:
return pd.DataFrame(data=transformed, columns=self.header)
 else:
return transformed
embedding_matrix, no_in_vocab = create_embedding_matrix(glove, tokenizer.word_index.items() , input_vocab_size, d_model, lemma_dict)
 del glove<string_transform>
ADD_NN = False
ALPHAS = [0.1, 0.3, 1, 3, 10]
RS = 713
KFOLDS = 3
test_features = [['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF'
,'BsmtFinType1', 'BsmtFinType2', 'BsmtUnfExp']
, ['AllBath', 'BsmtHalfBath', 'BsmtFullBath'
, 'HalfBath', 'FullBath']
, ['GarageCars', 'GarageArea']
, ['KitchenAbvGr']
, ['PoolArea', 'PoolQC']
, ['MiscFeature', 'MiscVal']
, ['Utilities', 'Electrical', 'OverallCond', 'LowQualFinSF']
, ['Age', 'YrSinceRemod']]
def get_pipe(additional_transform=[]):
transform = [("PreI", PreProcessor(old_new, num_to_obj))
,("RI", RelationImputer(related = RELATED)) ]
transform += additional_transform
return Pipeline(transform)
 def get_GBR() :
return GradientBoostingRegressor(n_estimators=500
, max_depth=2
, max_features=0.8
, random_state=RS)
 def get_ETR() :
return ExtraTreesRegressor(n_estimators=700
, max_depth=3
, max_features=0.8
, random_state=RS)
 def get_RFR() :
return RandomForestRegressor(n_estimators=700
, max_depth=3
, max_features=0.8
, max_samples=0.8
, random_state=RS)
 def get_XGBR() :
return XGBRegressor(n_estimators=500
, learning_rate=0.1
, max_depth=3
, colsample_bytree=0.8
, n_jobs=-1, random_state=RS)
 def get_Ridge() :
return RidgeCV(alphas=ALPHAS, normalize=False, store_cv_values=True)
 def get_Enet() :
return ElasticNetCV(l1_ratio=0.5
, alphas=ALPHAS
, normalize=False
, max_iter=1000
, random_state=RS 
)
 def get_NN(hidden_layers=1, compiled=True):
NN = Sequential()
regu = regularizers.l1_l2(l1=0.01, l2=0.01)
 for _ in range(hidden_layers):
NN.add(layers.Dense(100
, activation="relu"
, kernel_regularizer=regu
, bias_regularizer=regu))
NN.add(layers.Dense(1))
if compiled:
optimizer = optimizers.SGD(0.001)
 NN.compile(optimizer=optimizer, loss="mse", metrics=["MSE"])
 return NN
def get_SVM() :
return LinearSVR(random_state=RS, max_iter=1000)
 def get_LGBR() :
return LGBMRegressor(max_depth=3, n_estimators=500
, random_state=RS, n_jobs=-1)
 estimators = [get_GBR, get_ETR, get_RFR, get_XGBR, get_Ridge
, get_Enet, get_SVM, get_LGBR]
if ADD_NN:
estimators.add(get_NN)
 early_stop = callbacks.EarlyStopping(monitor='val_loss', patience=5)
 NN_kwargs = dict(epochs=100
, batch_size=10
, validation_split=0.2
, verbose=0
, callbacks=[early_stop])
 def RMSE(y_true, y_pred):
return math.sqrt(mean_squared_error(y_true, y_pred))
def reload_xy(msg=False):
df = reload(msg=msg)
 X = df[[c for c in df.columns if c != "SalePrice"]].copy()
y = df["SalePrice"].copy()
return X, np.array(np.log1p(y))
X, y_trans = reload_xy()
def new_output(bl):
cols = ["Est", bl, "Score_mean", "Score_delt(%)"
, "Score_std", "Time_run", "Time_delt(%)"]
return pd.DataFrame(columns=cols)
 def fit_timer(get_est, X, y, params={}):
e = get_est()
t = dt.now()
e.fit(X, y, **params)
 return timer(t)
 def scoring(get_est, additional_transform
, X=X, y=y_trans, scorer=RMSE, splitter=KFold, cv=KFOLDS):
scores = []
splitter = splitter(cv, shuffle=True, random_state=713)
 kw = {}
if get_est.__name__ == 'get_NN':
kw = NN_kwargs
for tr, te in splitter.split(X):
X_tr, X_te = X.iloc[tr, :], X.iloc[te, :]
y_tr, y_te = y[tr], y[te]
p = get_pipe(additional_transform)
 e = get_est()
X_tr = p.fit_transform(X_tr)
 X_te = p.transform(X_te)
 e.fit(X_tr, y_tr, **kw)
 y_pre = e.predict(X_te)
 scores.append(RMSE(y_te, y_pre))
pipe = get_pipe(additional_transform)
 X_trans = pipe.fit_transform(X)
 t = fit_timer(get_est, X_trans, y_trans, params=kw)
 return scores, t
def new_row(df, tactics, i, get_est, bl, scores, time):
scr_m, scr_std = np.mean(scores), np.std(scores)
 est = get_est()
if(i % len(tactics)) == 0:
scr_d, time_d = np.nan, np.nan
else:
last_scr, last_t = df.loc[i-1, ["Score_mean", "Time_run"]]
scr_d =(scr_m - last_scr)/(last_scr)* 100
time_d =(t - last_t)/(last_t)* 100
output.loc[i] = [
type(est ).__name__, bl, scr_m, scr_d, scr_std, time, time_d
]
def timer(t):
t = dt.now() - t
return t.seconds + t.microseconds/1e6
def for_loop(bl, estimators=estimators):
l = list(product(estimators, bl))
return tqdm(l, position=0 )
def inter_section(texts, keywords, niv):
niv = set(niv)
 text_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')
 text_tokenizer.fit_on_texts(np.stack([*texts, *keywords]))
vocab = set(text_tokenizer.word_index.keys())
 text_in_niv = vocab.intersection(niv)
 print("vocab:", len(vocab), len(text_in_niv), len(text_in_niv)/len(vocab))<concatenate>
add_trans = [("FE", FeatureEngineer())
 ,("CusOHE", CustomizedOHE())
 ,("Stdize", CustomizedScaler())]
pipe = get_pipe(add_trans)
 X_trans = pipe.fit_transform(X, y_trans)
 def col_name(col, ohe_cols=pipe.steps[3][1].transformed_cols_):
mat = re.match(r"(^x\d{1,2})_ (.+)", col)
 if mat:
return ohe_cols[int(mat.group(1)[1:])]
else:
return col
get_ests = [get_GBR, get_ETR, get_RFR, get_XGBR]
feat_imp = pd.DataFrame(
 columns = ["OHE_feat", "Features"]
+ [type(e() ).__name__ for e in get_ests]
+ ["Average"] 
)
 feat_imp.OHE_feat = list(X_trans.columns)
 feat_imp.Features = feat_imp.OHE_feat.apply(lambda x: col_name(x))
for get_est in get_ests:
est = get_est()
est.fit(X_trans, y_trans)
 feat_imp[type(est ).__name__[:-9]] = est.feature_importances_
feat_imp = feat_imp.groupby(["Features"] ).sum().reset_index()
feat_imp["Average"] = feat_imp.iloc[:,[1,2,3,4]].mean(axis=1)
 feat_imp = feat_imp.sort_values(by="Average", ascending=False)
 feat_imp = feat_imp.reset_index(drop=True)
 n = len(feat_imp)
 all_columns = list(feat_imp.Features)
 weak_cols = list(feat_imp[feat_imp.Average <(1/n)].Features)
 insig_cols = list(feat_imp[ 
(feat_imp.iloc[:,1:5] >(1/n)).sum(axis=1)== 0
].Features)
 zero_cols = list(feat_imp[feat_imp.Average == 0].Features)
 print(f"Features with AVERAGE importance score <(1/n_features):\
{len(weak_cols)}/{n}")
 print(*weak_cols[:5], sep=", ", end=", etc.
 ")
 print(f"Features with ALL importance scores <(1/len(features)) :\
{len(insig_cols)}/{n}")
 print(*insig_cols[:5], sep=", ", end=", etc.
 ")
 print(f"Features with 0 importance scores: {len(zero_cols)}/{n}")
 print(*zero_cols, sep=", ")
 feat_imp.to_csv("Column_importances.csv", index=False)
 a = feat_imp.iloc[:25,:]
ax = a.plot(y=[1,2,3,4], kind="line", figsize=(13,7), rot=90)
 a.plot(x="Features", y="Average"
, kind="bar", ax=ax, color="lightsteelblue"
, title="Top 25 Most Important Features" )
inter_section(test['text'], test['keyword'], no_in_vocab )<string_transform>
print(f"Average importance: {round(1/n,3)}")
 for c in test_features:
print("
" + "="*20)
 print(feat_imp[feat_imp.Features.isin(c)].round(3))
def incorrect_count(train_texts, test_texts, vocab):
vocab = set(vocab)
 wrong_words = []
for text in train_texts:
intersection = set(text.split() ).intersection(vocab)
 if len(intersection)>0:
wrong_words.extend(intersection)
 train_ww_count = np.asarray(Counter(wrong_words ).most_common())
 train_ww_count = np.concatenate([train_ww_count, np.asarray(['train']*len(train_ww_count)) [:, np.newaxis]], axis=-1)
 wrong_words = []
for text in test_texts:
intersection = set(text.split() ).intersection(vocab)
 if len(intersection)>0:
wrong_words.extend(intersection)
 test_ww_count = np.asarray(Counter(wrong_words ).most_common())
 test_ww_count = np.concatenate([test_ww_count, np.asarray(['test']*len(test_ww_count)) [:, np.newaxis]], axis=-1)
 ww_count = np.concatenate([train_ww_count, test_ww_count], axis=0)
 ww_count = pd.DataFrame(ww_count, columns=['wrong_text', 'count','set'])
 ww_count['count'] = ww_count['count'].astype('int')
 ww_count = ww_count.sort_values(by='count', ascending=False)
 print('head:
', ww_count.head())
 print('
count<2:', len(ww_count[ww_count['count']<2])/len(ww_count))
plt.figure(figsize=(6,36))
sns.barplot(x='count', y='wrong_text', hue='set', orient='h', data=ww_count.head(100))
plt.show()<compute_test_metric>
class ColumnDropper(BaseEstimator, TransformerMixin):
col_imp = pd.read_csv("/kaggle/working/Column_importances.csv")
 all_cols = list(col_imp.Features)
 n = len(all_cols)
 weak_cols = list(col_imp[col_imp.Average <(1/n)].Features)
 insig_cols = list(col_imp[ 
(col_imp.iloc[:,1:5] >(1/n)).sum(axis=1)== 0
].Features)
 insig2_cols = list(col_imp[ 
(col_imp.iloc[:,1:5] >(1/n/10)).sum(axis=1)== 0
].Features)
 zero_cols = list(col_imp[col_imp.Average == 0].Features)
 def __init__(self, strategy=None, others=None):

 self.strategy = strategy
self.others = others
if isinstance(strategy,(int,np.integer)) :
self.columns = self.all_cols[strategy:]
elif strategy == "weak":
self.columns = self.weak_cols
elif strategy == "insignificant":
self.columns = self.insig_cols
elif strategy == "insignificant2":
self.columns = self.insig2_cols
elif strategy == "zero":
self.columns = self.zero_cols
elif strategy is None:
self.columns = []
else:
raise Exception(f"Wrong 'strategy' input: "{strategy}"")
 if others:
self.columns += others
self.columns = list(set(self.columns))
def fit(self, X, y=None):
if X.shape[1] != self.n:
msg = f"Wrong number of features: expecting {self.n}, got {X.shape[1]}"
raise Exception(msg)
 missing_cols = [c for c in X.columns if c not in self.all_cols]
if len(missing_cols)> 0:
msg = "Columns not existing: " + ", ".join(missing_cols)
 raise Exception(msg)
 else:
return self
def transform(self, X, y=None):
if len(self.columns)> 0:
X = X.drop(columns=self.columns)
 return X
incorrect_count(train['text'], test['text'], no_in_vocab )<define_variables>
cols_to_drop = ["BsmtUnfExp", "BsmtUnfSF", "BsmtFinType2"
, "BsmtFinSF2", "BsmtHalfBath", "BsmtFullBath"
, "HalfBath", "KitchenAbvGr", "PoolQC"
, "MiscFeature", "MiscVal", "Electrical"
, "LowQualFinSF", "Utilities"]
output = new_output("Drop?")
 i = 0
tactics = [None, cols_to_drop]
for get_est, drop in for_loop(tactics):
add_trans = [("FE", FeatureEngineer())
 ,("Drop", ColumnDropper(others=drop))
,("CusOHE", CustomizedOHE())
 ,("Stdize", CustomizedScaler(keep_header=False)) ]
scores, t = scoring(get_est, add_trans)
 d = True if drop else False
new_row(output, tactics, i, get_est, d, scores, t)
 i += 1
output.round(3 )
num_texts =len_sentence(train, 'text' )<define_variables>
scrs = {}
time = {}
tactics = np.arange(84, 0, -1)
 for get_est, tac in for_loop(tactics):
dropper = ColumnDropper(strategy=tac, others=cols_to_drop)
 add_trans = [("FE", FeatureEngineer())
 ,("Drop", dropper)
 ,("CusOHE", CustomizedOHE())
 ,("Stdize", CustomizedScaler(keep_header=False)) ]
scores, t = scoring(get_est, add_trans)
 est_name = type(get_est() ).__name__
if est_name in scrs:
scrs[est_name].append(np.mean(scores))
time[est_name].append(t)
 else:
scrs[est_name] = [np.mean(scores)]
time[est_name] = [t]
num_keyword = len_sentence(train, 'keyword' )<categorify>
DROP = 42
train = reload(msg=False)
 X_train = train[[c for c in train.columns if c != "SalePrice"]].copy()
y_train = train["SalePrice"].copy()
X_test = reload("test.csv", msg=False)
 add_trans = [("FE", FeatureEngineer())
 ,("Drop", ColumnDropper(strategy=DROP, others=cols_to_drop))
,("CusOHE", CustomizedOHE())]
pipe = get_pipe(add_trans)
 X_train = pipe.fit_transform(X_train)
 X_test = pipe.transform(X_test)
 dropped_cols = pipe.steps[3][1].columns
print(f"Columns dropped: {len(dropped_cols)}/84")
 print("Examples: " + ", ".join(dropped_cols[:6])+ ", etc.")
 submission = reload("sample_submission.csv", dropped_columns=[], msg=False)
 est = get_XGBR()
est.fit(X_train, np.log1p(y_train))
submission["SalePrice"] = np.expm1(est.predict(X_test))
<categorify><EOS> 
X_train["SalePrice"] = train["SalePrice"]
submission.to_csv("Submission.csv", index=False)
 X_train.to_csv("train_preprocessed.csv", index=False)
 X_test.to_csv("test_preprocessed.csv", index=False )