Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
warnings.filterwarnings('ignore')
 %matplotlib inline
pd.set_option('display.max_columns', 300 )
wc_params = dict(
 max_font_size=42,
background_color=None,
mode='RGBA',
max_words=200,
width=300,
height=300,
collocations=False,
relative_scaling=0.6, 
)
 wordcloud_disaster = WordCloud(**wc_params ).generate(
 ' '.join(
 df.loc[
df['keyword_normalized'].replace({'nan': np.nan} ).isin(
 keywords_grouped[keywords_grouped['mean_target'] >= 0.5].index.tolist() 
),
'keyword_normalized'
].values.tolist() 
)
)
 wordcloud_neutral = WordCloud(**wc_params ).generate(
 ' '.join(
 df.loc[
df['keyword_normalized'].replace({'nan': np.nan} ).isin(
 keywords_grouped[keywords_grouped['mean_target'] <= 0.33].index.tolist() 
),
'keyword_normalized'
].values.tolist() 
)
)
 plt.figure(figsize=(7, 7))
plt.title('`Disastrous` keywords')
 plt.imshow(wordcloud_disaster, interpolation="bilinear")
 plt.axis("off")
 plt.show()
plt.figure(figsize=(7, 7))
plt.title('`Neutral` keywords')
 plt.imshow(wordcloud_neutral, interpolation="bilinear")
 plt.axis("off")
 plt.show()<string_transform>
df_train = pd.read_csv('.. /input/train.csv', index_col="Id")
 df_test = pd.read_csv('.. /input/test.csv', index_col="Id")
 y_train = df_train.SalePrice
df_train = df_train.drop(columns='SalePrice')
 print(len(df_train._get_numeric_data().columns))
print(len(df_train.select_dtypes(include='object' ).columns))
tparser = ttp.Parser()
def extract_tweet_entities(tweet_text):

 return tparser.parse(tweet_text)
 tweet_entities = {
tweet_id: extract_tweet_entities(text)
 for(tweet_id, text)in df.reset_index() [['id', 'text']].values.tolist()
}<sort_values>
categorical_features = df_train.select_dtypes(include='object' ).columns
df_train[categorical_features] = df_train[categorical_features].fillna('None')
 df_test[categorical_features] = df_test[categorical_features].fillna('None' )
hashtags_stats[ 
(hashtags_stats.tweet_cnt > 6)
 &(hashtags_stats.mean_target > 0.75)
 ].sort_values(by=['mean_target', 'tweet_cnt'], ascending=[False, False] )<sort_values>
df_train["LotFrontage"] = df_train.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
df_test["LotFrontage"] = df_test.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
numerical_features = df_train.select_dtypes(include='number' ).columns
my_imputer = Imputer()
df_train[numerical_features] = my_imputer.fit_transform(df_train[numerical_features])
 df_test[numerical_features] = my_imputer.transform(df_test[numerical_features])
hashtags_stats[ 
(hashtags_stats.tweet_cnt > 6)
 &(hashtags_stats.mean_target <= 0.25)
 ].sort_values(by=['mean_target', 'tweet_cnt'], ascending=[True, False] )<feature_engineering>
idx_split = df_train.shape[0]
df_train = df_train.append(df_test )
G_ht = nx.Graph()
G_ht.add_nodes_from(hashtags_stats.index.tolist())
 d = defaultdict(int)
 for taglist in df.hashtags:
for c in combinations(taglist, 2):
d[c] += 1
G_ht.add_edge(( *c))
for n1, n2, attrs in G_ht.edges(data=True):
attrs['weight'] = np.log1p(d[(n1, n2)])
 print(
 f'Init stats:
Nodes: {G_ht.number_of_nodes() }
Edges: {G_ht.number_of_edges() }')
 weak_edges = []
for n1, n2, attrs in G_ht.edges(data=True):
if attrs['weight'] <= np.log1p(1):
weak_edges.append(( n1, n2))
G_ht.remove_edges_from(weak_edges)
 print(
 f'After weak edge removal:
Nodes: {G_ht.number_of_nodes() }
Edges: {G_ht.number_of_edges() }')
 isolates = list(nx.isolates(G_ht))
G_ht.remove_nodes_from(isolates)
 print(
 f'After isolates removal:
Nodes: {G_ht.number_of_nodes() }
Edges: {G_ht.number_of_edges() }')
 node_degrees = pd.DataFrame(
 [x for x in G_ht.degree() ],
columns=['node', 'degree']
).sort_values(by='degree', ascending=False ).reset_index()
G_ht.remove_node(node_degrees.iloc[0]['node'])
 print(
 f'After highest node degree removal:
Nodes: {G_ht.number_of_nodes() }
Edges: {G_ht.number_of_edges() }')
 cc_ht = sorted(list(nx.connected_components(G_ht)) ,
key=lambda x: len(x), reverse=True)
 min_size = 5
cc_ht = [c for c in cc_ht if len(c)> min_size]
print(f'Top connected Components: {len(cc_ht)}' )<feature_engineering>
NumStr = ["MSSubClass","BsmtFullBath","BsmtHalfBath","HalfBath","BedroomAbvGr","KitchenAbvGr","MoSold","YrSold","YearBuilt","YearRemodAdd","LowQualFinSF","GarageYrBlt"]
for col in NumStr:
df_train[col]=df_train[col].astype(str)
 non_ordinal_features = ['LandSlope','Utilities','MSZoning', 'Street','Alley','LotShape','LandContour','LotConfig','Neighborhood','Condition1','Condition2','BldgType','HouseStyle', 'RoofStyle','RoofMatl','Exterior1st', 'Exterior2nd','MasVnrType','Foundation', 'BsmtExposure','BsmtFinType1','BsmtFinType2', 'Heating', 'CentralAir', 'Electrical','Functional', 'GarageType','GarageFinish','PavedDrive', 'Fence','MiscFeature','SaleType','SaleCondition', "MSSubClass","BsmtFullBath","BsmtHalfBath","HalfBath","BedroomAbvGr","KitchenAbvGr","MoSold","YrSold","YearBuilt","YearRemodAdd","LowQualFinSF","GarageYrBlt"]
ordinal_features = df_train.select_dtypes(include='object' ).columns.drop(non_ordinal_features )
df['hashtag_cluster'] = [[] for i in range(len(df)) ]
for(ri, r)in tqdm(list(df.iterrows()), total=len(df)) :
for(i, c)in enumerate(cc_ht):
if set(r['hashtags'] ).intersection(c):
df.loc[ri, 'hashtag_cluster'].append(i)
 if not df.loc[ri, 'hashtag_cluster']:
df.loc[ri, 'hashtag_cluster'].append(-1)
 t = df.explode('hashtag_cluster')['hashtag_cluster']\ 
.reset_index().rename(columns={'index': 'id'})
 df_ht_clusters = pd.crosstab(t.id, t.hashtag_cluster)
 df_ht_clusters.columns = [f'ht_cluster__{c}' for c in df_ht_clusters.columns]
print(df.shape)
 df = df.drop(df.filter(regex='^ht_cluster' ).columns, axis=1)
 df = pd.concat([df, df_ht_clusters], axis=1)
 print(df.shape)
 df_ht_clusters.head()<load_from_csv>
dummies = pd.get_dummies(df_train.loc[:,non_ordinal_features], drop_first=True)
 df_train = pd.concat([df_train,dummies], axis=1)
 df_train = df_train.drop(non_ordinal_features,axis=1 )
geodata = pd.read_csv('.. /input/disaster-tweets-geodata/geodata.csv' ).set_index('id')
 geodata.head()<merge>
def cat_to_num(x):
if x=='Ex':
return 5
if x=='Gd':
return 4
if x=='TA':
return 3
if x=='Fa':
return 2
if x=='Po':
return 1
if x=='None':
return 0
df_train.loc[:,ordinal_features] = df_train.loc[:,ordinal_features].applymap(cat_to_num )
print(df.shape)
 df = df.merge(geodata, left_index=True, right_index=True, how='left')
 df = df[~df.index.duplicated(keep='first')]
print(df.shape)
 df[['location', 'target'] + geodata.columns.tolist() ].sample(5, random_state=911 )<choose_model_class>
X = df_train.loc[:idx_split, :]
X_test = df_train.loc[idx_split+1:, :]
EMOJI_REGEX = re.compile(
 "["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+",
flags=re.UNICODE 
)
 NON_WORD_PATTERN = r"[^A-Za-z0-9\.'!\?,\$\s]"
PUNCT_DEDUPLICATION_REGEX = re.compile(r'[\?\.\!]+(?=[\?\.\!])', re.I )<feature_engineering>
X = X.assign(SalePrice=y_train )
df['text_processed'] = df.text.str.replace(EMOJI_REGEX, ' EMOJI ')\ 
.str.replace(PUNCT_DEDUPLICATION_REGEX, '')\ 
.str.replace(URL_REGEX, ' URL ' ).str.replace(NON_WORD_PATTERN, '' ).str.replace('\s+', ' ')
 df['text_processed'].sample(10, random_state=911 ).values.tolist()<compute_train_metric>
X = X.drop(columns='SalePrice' )
def compute_doc2vec(words, w2v, weight_dict, emb_size):

 doc_vector = np.zeros(emb_size, dtype=np.float32)
 weights = 0
for w in words:
try:
doc_vector += w2v.wv[w]
weights += weight_dict.get(w, 0)
 except KeyError:
pass
doc_vector /= max(0.01, weights)
 return doc_vector / np.linalg.norm(doc_vector, ord=2)
 text_field = 'text_processed'
prepared_texts = df[text_field].str.lower().str.replace(
 '', ''
).str.extractall(
 '(?P<words>[A-Za-z]+)' ).groupby(level=0)['words'].apply(list ).reindex(df.index ).fillna('EMPTY')
 w2v_size = 16
w2v = Word2Vec(
 sentences=prepared_texts,
size=w2v_size,
window=32,
min_count=7,
iter=5,
negative=5,
sg=1,
batch_words=2**11, 
)
 word_dict = np.log1p(prepared_texts.explode().value_counts() ).to_dict()
w2v_df = pd.DataFrame(
 data=np.concatenate(
 prepared_texts.apply(
 lambda x: compute_doc2vec(
 x, w2v, weight_dict=word_dict, emb_size=w2v_size)
 ).values
).reshape(( len(df), w2v_size)) ,
columns=[f'w2v__{i}' for i in range(w2v_size)] 
)
 df = df.drop(df.filter(regex='^w2v__' ).columns, axis=1)
 print(df.shape)
 df = pd.concat([df, w2v_df], axis=1)
 print(df.shape )<categorify>
X["TotalHouse"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"]
X["TotalHouse_OverallQual"] = X["TotalHouse"] * X["OverallQual"]
X["GrLivArea_OverallQual"] = X["GrLivArea"] * X["OverallQual"]
X_test["TotalHouse"] = X_test["TotalBsmtSF"] + X_test["1stFlrSF"] + X_test["2ndFlrSF"]
X_test["TotalHouse_OverallQual"] = X_test["TotalHouse"] * X_test["OverallQual"]
X_test["GrLivArea_OverallQual"] = X_test["GrLivArea"] * X_test["OverallQual"]
hashtags_sentences = df.explode('hashtags')['hashtags'].groupby(level=[0] ).apply(
 lambda x: ' '.join([re.sub(r'[^\x00-\x7f]', r'', s)for s in list(x.astype(str)) ])
)
 hashtags_sentences.sample(10, random_state=911 )<categorify>
print(X[(X['GrLivArea']>4000)&(y_train<300000)].index)
 X = X.drop(X[(X['GrLivArea']>4000)&(y_train<300000)].index)
 y_train = y_train.drop([524, 1299] )
tfidf = TfidfVectorizer(
 max_df=1.0,
min_df=5,
dtype=np.float32,
max_features=200, 
)
 tfidf.fit(hashtags_sentences)
 tag_df = pd.DataFrame(
 data=tfidf.transform(hashtags_sentences ).todense() ,
columns=[f'tag__{k}' for k, v in sorted(tfidf.vocabulary_.items() , key=lambda item: item[1])] 
)
 print(tag_df.shape)
 tag_df.head()
df = df.drop(df.filter(regex='^tag__' ).columns, axis=1)
 print(df.shape)
 df = pd.concat([df, tag_df], axis=1)
 print(df.shape )<categorify>
numeric_feats = X.dtypes[X.dtypes != "object"].index
numeric_feats = numeric_feats.drop(dummies)
 numeric_feats = numeric_feats.drop(ordinal_features)
 skewed_feats = X[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness = skewness[abs(skewness)> 0.75]
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
X[feat] = boxcox1p(X[feat], lam)
 X_test[feat] = boxcox1p(X_test[feat], lam )
embedder = hub.load("https://tfhub.dev/google/universal-sentence-encoder-large/5")
 df_univ_embeddings = pd.DataFrame(
 data=embedder(df[text_field].values ).numpy() ,
columns=[f'tf_emb__{i}' for i in range(512)] 
)
 df_univ_embeddings.head()<data_type_conversions>
y_train = np.log(y_train )
df['text_len'] = df.text.apply(len ).astype(np.int16)
 df['text_word_cnt'] = df.text.str.split('\s+' ).map(len ).astype(np.int16)
 df['has_exclamations'] = df.text.str.contains(r'[?!]' ).astype(np.int8)
 df['has_uppercased'] = df.text.str.contains(r'[A-Z]{2,}' ).astype(np.int8)
 nlp_features = [
'has_exclamations',
'has_uppercased',
'text_len',
'text_word_cnt',
]
df[nlp_features + ['target']].corr()<data_type_conversions>
multicol_features = ['GarageCars', 'PoolQC', 'FireplaceQu', 'GarageCond']
X = X.drop(multicol_features, axis = 1)
 X_test = X_test.drop(multicol_features, axis = 1 )
features_num =(
 [
'hashtag_cnt',
'url_cnt',
'user_cnt',
'lat',
'lon'
]
+ tag_df.columns.tolist()
+ w2v_df.columns.tolist()
+ df_ht_clusters.columns.tolist()
+ df_univ_embeddings.columns.tolist() 
)
 features_cat = [
'country',
'city',
'keyword_normalized',
]
cat_cols = []
for col in features_cat:
df[f'{col}_code'] = df[col].astype('category' ).cat.codes
cat_cols.append(f'{col}_code')
 features = features_num + cat_cols
print(df[features].dtypes)
 df[features].head()<init_hyperparams>
scaler = MinMaxScaler()
scaler.fit(X)
 X[X.columns] = scaler.transform(X[X.columns])
 X_test[X_test.columns] = scaler.transform(X_test[X_test.columns] )
def f1_score_lgb(preds, dtrain):
labels = dtrain.get_label()
f_score = f1_score(
 np.round(preds),
labels,
average='macro' 
)
 return 'f1_score', f_score, True
skf = StratifiedKFold(n_splits=5, random_state=911, shuffle=True)
 lgb_params = {
'num_leaves': 63,
'learning_rate': 0.01,
'max_depth': -1,
'subsample': 0.8,
'colsample_bytree': 0.25,
'cat_l2': 10.,
'cat_smooth': 10.,
'min_data_per_group': 20,
'min_data_in_leaf': 20,
'reg_lambda': 0.5,
'boost_from_average': True,
'max_cat_threshold': 64,
'max_bin': 255,
'min_data_in_bin': 5,
}
sample_weight =(
 1 / df.loc[
df.is_train,
'target'
].map(df.loc[df.is_train, 'target'].value_counts(normalize=True)) 
)
 cv_res = lgb.cv(
 params=lgb_params,
train_set=lgb.Dataset(
 data=df.loc[df.is_train, features],
label=df.loc[df.is_train, 'target'],
categorical_feature=cat_cols,
weight=sample_weight 
),
folds=skf,
metrics=['binary_logloss'],
feval=f1_score_lgb,
verbose_eval=50,
early_stopping_rounds=200,
num_boost_round=1000,
)<prepare_x_and_y>
rgr_ls = Lasso(fit_intercept = True)
 param_grid = {
"alpha": [0.0002, 0.0003, 0.0005, 0.0008, 0.001]
}
searchCV = GridSearchCV(rgr_ls, cv=5, param_grid=param_grid, scoring='neg_mean_squared_error')
 searchCV.fit(X, y_train)
 np.sqrt(searchCV.best_score_*-1 )
<save_to_csv><EOS> 
y_test_pred = searchCV.predict(X_test)
 y_test_pred = np.exp(y_test_pred)
 my_submission = pd.DataFrame({'Id': X_test.index.astype(int), 'SalePrice': y_test_pred})
 my_submission.to_csv('submission.csv', index=False )