Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<merge>
sns.set(font_scale=1.4)
 warnings.filterwarnings("ignore")
mkt_price = full_data.groupby(['building_id', 'display_address', 'bedrooms', 'bathrooms'] ).price.mean().reset_index()
mkt_price = pd.merge(full_data[['building_id', 'display_address', 'bedrooms', 'bathrooms']],
mkt_price, how='left', on=['building_id', 'display_address', 'bedrooms', 'bathrooms'] ).price
full_data['mkt_price'] = mkt_price.values
full_data['diff_to_mkt_price'] = full_data['price'] - full_data['mkt_price']
full_data['ratio_to_mkt_price'] = full_data['price'] / full_data['mkt_price']
price_vars = ['diff_to_mkt_price', 'ratio_to_mkt_price']<feature_engineering>
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )
full_data['photos_str'] = full_data['photos'].astype(str)
 full_data['listing_uid'] = full_data[['manager_id', 'building_id','photos_str']].apply(lambda x: hashlib.md5(( x[0] + x[1] + x[2] ).encode() ).hexdigest() , axis=1)
 full_data['posted_times'] = full_data.groupby('listing_uid' ).created_datetime.rank(method='first', na_option='top',pct=True)
 full_data['num_of_html_tag']=full_data.description.apply(lambda x:x.count('<'))
full_data['num_of_
full_data['num_of_!']=full_data.description.apply(lambda x:x.count('!'))
full_data['num_of_$']=full_data.description.apply(lambda x:x.count('$'))
full_data['num_of_*']=full_data.description.apply(lambda x:x.count('*'))
full_data['num_of_>']=full_data.description.apply(lambda x:x.count('>'))
full_data['num_of_puncs']=full_data['num_of_
full_data['puncs_ratio'] = full_data['num_of_puncs']/full_data['len_of_desc']
full_data['upper_char_ratio'] = full_data['description'].apply(lambda x: 0 if sum([s.isalpha() for s in x])==0 else sum([s.isalpha() &s.isupper() for s in x])/ sum([s.isalpha() for s in x]))
full_data['disp_is_street'] =(full_data['display_address'] == full_data['street_address'])*1
full_data['disp_st_addr_word_ratio'] = full_data.apply(lambda x:len(x['display_address'].split(' ')) /len(x['street_address'].split(' ')) , axis=1)
 listing_quality_vars = ['disp_is_street', 'num_of_html_tag','num_of_
'posted_times', 'disp_st_addr_word_ratio','upper_char_ratio']<prepare_x_and_y>
def add_pca(train_df, test_df, cols, n_comp=20, fit_test = True, prefix='pca_', fit_test_first=False):
pca = PCA(n_components=n_comp, random_state=42)
 pca_titles = [prefix+'_pca_'+str(x)for x in range(n_comp)]
temp_train = train_df.copy()
temp_test = test_df.copy()
for c in cols:
fv = temp_train[c].mean()
temp_train[c] = temp_train[c].fillna(value=fv)
 fv = temp_test[c].mean()
temp_test[c] = temp_test[c].fillna(value=fv)
 for p in pca_titles:
train_df[p] = 0.0
test_df[p] = 0.0
if fit_test==True:
pca_data = pd.concat([temp_train[cols], temp_test[cols]], axis=0)
 if fit_test_first==True:
pca.fit(pca_data[len(train_df):])
 pca_data = pca.transform(pca_data)
 else:
pca_data = pca.fit_transform(pca_data)
 train_df.loc[:, pca_titles] = pca_data[0:len(train_df)]
test_df.loc[:, pca_titles] = pca_data[len(train_df):]
return train_df, test_df, pca_titles
def split_distributions(train_df, test_df, cols, target, n_comp=20, prefix='gm', add_labels=False):
gm = GaussianMixture(n_components=n_comp,covariance_type='full', tol=0.001, reg_covar=1e-06, max_iter=100, n_init=1,
init_params='kmeans', weights_init=None, means_init=None, precisions_init=None,
random_state=42, warm_start=False, verbose=0, verbose_interval=10)
 temp_train = train_df.copy()
temp_test = test_df.copy()
gm_data = pd.concat([temp_train[[cols]], temp_test[[cols]]], axis=0 ).reset_index(drop=True)
 gm.fit(gm_data)
 gm_data['labels'] = gm.predict(gm_data)
 if add_labels:
gm_titles = [prefix+str(x)for x in range(n_comp)]
train_df[gm_titles]=0
test_df[gm_titles]=0
dummies = pd.get_dummies(gm_data['labels'], prefix=prefix)
 train_df.loc[:, gm_titles] = dummies[0:len(train_df)].values
test_df.loc[:, gm_titles] = dummies[len(train_df):].values
else:
gm_titles = []
train_df[prefix+'_label'] = gm.labels_[0:len(train_df)]
test_df[prefix+'_label'] = gm.labels_[len(train_df):]
means = train_df.groupby([prefix+'_label'])[target].mean()
train_df['menc_'+prefix+'_label'] = train_df[prefix+'_label'].map(means)
 test_df['menc_'+prefix+'_label'] = test_df[prefix+'_label'].map(means)
 gm_titles+=[prefix+'_label', 'menc_'+prefix+'_label']
return train_df, test_df, gm_titles
def assign_fold(df, label_column,fold_column, NFOLDS=5):
skf = StratifiedKFold(n_splits=NFOLDS)
 df[fold_column]=0
f=0
for trn_idx, val_idx in skf.split(df, df[label_column]):
df.loc[val_idx, 'fold']=f
f+=1
df[fold_column].value_counts()
return df
def st_scale(train_df, test_df, cols):
StSc = StandardScaler()
train_df[cols] = StSc.fit_transform(train_df[cols])
 test_df[cols] = StSc.transform(test_df[cols])
 return train_df, test_df
def seed_everything(seed=1234):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed )
full_num_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars +geo_num_vars + count_vars \
+ listing_vars + listing_quality_vars
full_cat_vars = LE_vars + mean_coded_vars + CB_vars
full_vars = full_num_vars + full_cat_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,"testing data size: ", test_x.shape )<groupby>
PATH = '/kaggle/input/tabular-playground-series-jan-2021/'
train = pd.read_csv(PATH+'train.csv')
 test = pd.read_csv(PATH+'test.csv')
 submission = pd.read_csv(PATH+'sample_submission.csv')
 FT_COLS = [x for x in train.columns if 'cont' in x]
TARGET='target'
print(train.shape)
 train.head(10 )
def p25(x):
return np.percentile(x, 25)
 def p50(x):
return np.percentile(x, 50)
 def p75(x):
return np.percentile(x, 75)
 def nunique(x):
return np.size(np.unique(x))
def max_min(x):
return np.max(x)-np.min(x)
 def p75_p25(x):
return np.percentile(x, 75)-np.percentile(x, 25)
 def get_group_stats(df, stat_funcs, target_column, group_column, ranking=False, ranking_pct=True):
aggr = df.groupby(group_column)[target_column].agg([v for v in stat_funcs.values() ] ).reset_index()
aggr.columns = [group_column] + [ target_column + '_' + k + '_by_' + group_column for k in stat_funcs.keys() ]
aggr = df[[group_column]].merge(aggr, how='left', on=group_column)
 if ranking:
aggr[target_column + '_rank_by_' + group_column] = df.groupby(group_column)[target_column].rank(method='dense',
na_option='top',
pct=ranking_pct)
 return aggr.drop(group_column, axis=1 )<create_dataframe>
original_feature_dict = {'cont1': 5,
'cont2': 10,
'cont3': 8,
'cont4': 8,
'cont5': 5,
'cont6': 6,
'cont7': 4,
'cont8': 3,
'cont9': 7,
'cont10':3,
'cont11': 4,
'cont12': 2,
'cont13': 3,
'cont14': 5,}
pca_dict = {
'extra_pca_0': 4,
'extra_pca_1': 1,
'extra_pca_2': 1,
'extra_pca_3': 2,
'extra_pca_4': 1,
'extra_pca_5': 1 }
MULTIPLIER = 2.0
original_feature_dict = {a:int(b*MULTIPLIER)for(a,b)in original_feature_dict.items() }
ft_dict = {**original_feature_dict, **pca_dict}
ft_dict
stat_funcs = {
'mean': np.mean,
'min': np.min,
'max': np.max,
'std': np.std,
'p25': p25,
'p50': p50,
'p75': p75,
'skew': skew,
'kurtosis': kurtosis,
'max_min': max_min,
'p75_p25': p75_p25
}
mgr_aggr = pd.DataFrame()
for num_var in num_vars + additional_num_vars + listing_quality_vars:
mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, stat_funcs,
target_column=num_var, group_column='manager_id', ranking=False)
 ],
axis=1 
)
 mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'max_min': max_min, 'p75_p25': p75_p25},
target_column='created_epoch', group_column='manager_id', ranking=False)
 ],
axis=1 
)
 mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'nunique': nunique},
target_column='created_dayofyear', group_column='manager_id', ranking=False)
 ],
axis=1 
)
 mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'nunique': nunique},
target_column='building_id', group_column='manager_id', ranking=False)
 ],
axis=1 
)
 for aggr_col in ['geo_area_50', 'geo_area_100', 'geo_area_200']:
mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'nunique': nunique},
target_column=aggr_col, group_column='manager_id', ranking=False)
 ],
axis=1 
)
 for aggr_col in ['diff_to_mkt_price', 'ratio_to_mkt_price']:
mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'mean': np.mean},
target_column=aggr_col, group_column='manager_id', ranking=False)
 ],
axis=1
)<concatenate>
mixture_title_cols=[]
for f in FT_COLS+pca_titles2:
train, test, titles = split_distributions(train, test,f,TARGET, n_comp=ft_dict[f], prefix=f+'_dim', add_labels=True)
 mixture_title_cols+=titles
mixture_value_cols = []
for count,f in enumerate(FT_COLS+pca_titles2):
for d in range(ft_dict[f]):
t_median = train[f][train[f+'_dim'+str(d)]==1].median()
t_std = train[f][train[f+'_dim'+str(d)]==1].std()
train[f+'_dim'+str(d)+'_dist'] = np.where(train[f+'_dim'+str(d)]==1, 
(train[f] - t_median)/t_std, np.nan)
 test[f+'_dim'+str(d)+'_dist'] = np.where(test[f+'_dim'+str(d)]==1, 
(test[f] - t_median)/t_std, np.nan)
 mixture_value_cols+=[f+'_dim'+str(d)+'_dist']
full_num_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars + count_vars \
+ listing_vars + listing_quality_vars
full_cat_vars = LE_vars + mean_coded_vars + CB_vars
full_vars = full_num_vars + full_cat_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse,
mgr_aggr] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse,
mgr_aggr] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,
"testing data size: ", test_x.shape )<create_dataframe>
print('Total Original Features', len(FT_COLS))
print('Total Submixtures Labels', len(mixture_title_cols))
print('Total Submixtures Values', len(mixture_value_cols))
print('Total Feature Columns', len(FT_COLS)+len(mixture_title_cols)+len(mixture_value_cols))
stat_funcs = {
'mean': np.mean,
'min': np.min,
'max': np.max,
'std': np.std,
'p25': p25,
'p50': p50,
'p75': p75,
'skew': skew,
'kurtosis': kurtosis,
'max_min': max_min,
'p75_p25': p75_p25
}
building_aggr = pd.DataFrame()
building_aggr = pd.concat([building_aggr,
get_group_stats(full_data, stat_funcs,
target_column='price', group_column='building_id', ranking=False)
 ],
axis=1 
)
 building_aggr = pd.concat([building_aggr,
get_group_stats(full_data, {'nunique': nunique},
target_column='manager_id', group_column='building_id', ranking=False)
 ],
axis=1
)<train_model>
NAN_VALUE = 0.0
for d in mixture_value_cols:
train[d] = train[d].fillna(value=NAN_VALUE)
 test[d] = test[d].fillna(value=NAN_VALUE )
park_listings = full_data[full_data[['description', 'features']].apply(lambda x: 'park' in x[0] or 'park' in x[1], axis=1)][['latitude', 'longitude']]
park_n_clusters = 25
kms = KMeans(n_clusters=park_n_clusters)
 kms.fit(park_listings)
 park_dist_data = pd.DataFrame(kms.transform(full_data[['latitude', 'longitude']]),
columns = ['dist_to_park_' + str(i)for i in range(park_n_clusters)]
)<create_dataframe>
SCALE = True
if SCALE:
train, test = st_scale(train, test, FT_COLS)
 for f in FT_COLS:
train[f] = np.clip(train[f], -2, 2)
 test[f] = np.clip(test[f], -2, 2)
 SCALE_DISTS = True
if SCALE_DISTS:
for d in mixture_value_cols:
TEMP_MAX = max(np.abs(train[d] ).max() ,
np.abs(test[d] ).max())
 train[d] = train[d] / TEMP_MAX
test[d] = test[d] / TEMP_MAX
subway_listings = full_data[full_data[['description', 'features']].apply(lambda x: 'subway' in x[0] or 'subway' in x[1], axis=1)][['latitude', 'longitude']]
subway_n_clusters = 400
kms = KMeans(n_clusters=subway_n_clusters)
 kms.fit(subway_listings)
 subway_dist_data = pd.DataFrame(kms.transform(full_data[['latitude', 'longitude']]),
columns = ['dist_to_subway_' + str(i)for i in range(subway_n_clusters)]
)<feature_engineering>
train['outlier_filter'] = np.where(train[TARGET]<4, True, False)
 print('
nlp = spacy.load("en_core_web_sm")
 def seq_to_token(seq, nlp=nlp):
doc = nlp(str(seq ).lower())
 tokens = [token.text for token in doc if not(token.is_space | token.is_stop|token.like_num)]
return tokens
def tokens_to_vec(tokens, model, vec_size=10):
if len(tokens)==0:
return np.zeors(vec_size)
 else:
return np.array([emb_model[token] for token in tokens] ).mean(axis=1 )<train_model>
print('CV Benchmark - Random Guess of Median')
 cv_bm = np.sqrt(mse(train[TARGET], np.full(( train[TARGET].shape), train[TARGET].median())))
 cv_bm
building_model = FastText(size=10, window=3, min_count=1, workers=16)
 building_model.build_vocab(sentences=building_by_mgr)
 building_model.train(sentences=building_by_mgr.values, total_examples=len(building_by_mgr.values), epochs=5)
 building_model['8a8b08e08888819a3e745005a8cd0408']<categorify>
def run_training(model, train_df, test_df,sample_submission, fold_col,
orig_features, mixture_val_cols,mixture_label_cols, target_col,
benchmark,
outlier_col=None, nn=False, epochs=10,batch_size=32, verbose=False,
dense=70, dout=0.15, dense_reg = 0.000001,act='elu'):
FOLD_VALUES = sorted([x for x in train_df[fold_col].unique() ])
 oof = np.zeros(( len(train_df),))
test_predictions = np.zeros(( len(sample_submission),))
fig,axes=plt.subplots(figsize=(15,6))
axes.set_ylim(0.69,0.75)
 RANDOM_SEEDS = [0,42,100,1000]
for rs in RANDOM_SEEDS:
seed_everything(seed=rs)
 for fold in FOLD_VALUES:
print(' --- ')
 print('running random seed', rs, 'fold', fold)
 if outlier_col:
trn_idx =(train_df[fold_col]!=fold)&(~train_df[outlier_col])
 else:
trn_idx = train_df[fold_col]!=fold
val_idx = train_df[fold_col]==fold
X_train_orig = train_df.loc[trn_idx,orig_features].values
X_train = train_df.loc[trn_idx,mixture_val_cols].values
X_train_mask = train_df.loc[trn_idx,mixture_label_cols].values
y_train = train_df.loc[trn_idx, target_col].values
X_val_orig = train_df.loc[val_idx,orig_features].values
X_val = train_df.loc[val_idx,mixture_val_cols].values
X_val_mask = train_df.loc[val_idx,mixture_label_cols].values
y_val = train_df.loc[val_idx, target_col].values
model = keras_model(orig_features, mixture_val_cols,mixture_label_cols,dout=dout,
dense=dense,act=act, dense_reg = dense_reg, descend_fraction = 0.9)
 K.clear_session()
back_count = 5
epoch_list=list(range(epochs-back_count, epochs))
class CustomModelCheckpoint(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
if epoch in epoch_list:
self.model.save_weights('model_fold_'+str(fold)+'_epoch_'+str(epoch)+'.h5', overwrite=True)
 else:
pass
cbk = CustomModelCheckpoint()
history = model.fit([X_train_orig, X_train,X_train_mask ], y_train, epochs=epochs, batch_size=batch_size,shuffle=True,
validation_data=([X_val_orig, X_val,X_val_mask], y_val), verbose=verbose,
callbacks=[cbk])
 print('Fold Last Epoch Train Error', history.history['root_mean_squared_error'][-1])
 print('Fold Last Epoch Valid Error', history.history['val_root_mean_squared_error'][-1])
 model.save_weights('model_final_'+str(fold)+'.h5')
 sns.lineplot(x=range(epochs), y=history.history['loss'], color='Blue')
 sns.lineplot(x=range(epochs), y=history.history['val_loss'], color='Red')
 val_preds = np.zeros(( len(X_val),))
for e in epoch_list:
model = keras_model(orig_features, mixture_val_cols,mixture_label_cols,dout=dout,
dense=dense,act=act, dense_reg = dense_reg, descend_fraction = 0.9)
 model.load_weights('model_fold_'+str(fold)+'_epoch_'+str(e)+'.h5')
 val_preds += model.predict([X_val_orig, X_val,X_val_mask] ).mean(axis=1)
 test_predictions += model.predict([test_df[orig_features].values,test_df[mixture_val_cols].values,
test_df[mixture_label_cols].values] ).mean(axis=1)
 val_preds = val_preds / len(epoch_list)
 oof[val_idx]+=val_preds
val_error = np.sqrt(mse(y_val, val_preds))
print('Fold multi epoch weight prediction error', val_error)
 oof = oof / len(RANDOM_SEEDS)
 total_val_error = np.sqrt(mse(train_df[target_col], oof))
print('final OOF MSE', total_val_error)
 fold_errors = []
for fold in train_df[fold_col].unique() :
val_idx = train_df[fold_col]==fold
fold_errors+=[np.sqrt(mse(train_df.loc[val_idx, target_col].values, oof[val_idx])) ]
test_predictions = test_predictions/(NFOLDS * len(RANDOM_SEEDS)* len(epoch_list))
return oof, test_predictions, fold_errors
building_emb = full_data['building_id'].apply(lambda x:building_model[x] ).values
building_emb = np.array([e.reshape(1,-1)for e in building_emb] ).reshape(-1,10)
 building_emb<train_on_grid>
def keras_model(ft_orig, mixture_values, mixture_labels, n_layer=3,bnorm=True,dout=0.2,
dense=20,act='elu', dense_reg = 0.000001, descend_fraction = 0.9):
input1 = L.Input(shape=(len(ft_orig)) , name='input_orig')
 input1_do = L.Dropout(0.1 )(input1)
 XA = L.Dense(dense, activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg), name='dense_orig' )(input1_do)
 if bnorm:
XA = L.BatchNormalization(name='bn_1' )(XA)
 XA1 = L.Dropout(dout, name='do_orig' )(XA)
 input2 = L.Input(shape=(len(mixture_values)) , name='input_new')
 input3 = L.Input(shape=(len(mixture_labels)) , name='input_new2')
 all_input_combo = tf.keras.layers.Concatenate(axis=1, name='concat_i2' )([input2, input3])
 input2_do = L.Dropout(0.2 )(all_input_combo)
 XB = L.Dense(dense, activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg), name='dense_new' )(input2_do)
 if bnorm:
XB = L.BatchNormalization(name='bn_2' )(XB)
 XB1 = L.Dropout(dout, name='do_new' )(XB)
 all_input_combo = tf.keras.layers.Concatenate(axis=1, name='concat_new' )([XA1, XB1])
 X = L.Dense(int(dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg))(all_input_combo)
 if bnorm:
X = L.BatchNormalization(name='bn_3' )(X)
 X2 = L.Dropout(dout, name='do_3' )(X)
 X2 = tf.keras.layers.concatenate([all_input_combo, X2], axis=1)
 X = L.Dense(int(( descend_fraction**2)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg))(X2)
 if bnorm:
X = L.BatchNormalization()(X)
 X3 = L.Dropout(dout )(X)
 X3 = tf.keras.layers.concatenate([X2, X3], axis=1)
 X = L.Dense(int(( descend_fraction**3)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg *2))(X3)
 if bnorm:
X = L.BatchNormalization()(X)
 X4 = L.Dropout(dout )(X)
 X4 = tf.keras.layers.concatenate([X3, X4],axis=1)
 X = L.Dense(int(( descend_fraction**4)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg*4))(X4)
 if bnorm:
X = L.BatchNormalization()(X)
 X5 = L.Dropout(dout )(X)
 X5 = tf.keras.layers.concatenate([X4, X5], axis=1)
 X = L.Dense(5, activation=act, activity_regularizer=tf.keras.regularizers.L2(0.0001))(X5)
 output1 = L.Dense(1, activation='linear' )(X)
 model = tf.keras.Model(inputs=[input1, input2, input3], outputs=output1)
 model.compile(loss=root_mean_squared_error,
optimizer=tfa.optimizers.AdamW(learning_rate=0.000809028893821181, weight_decay=9.83479875802558E-06),
metrics=tf.keras.metrics.RootMeanSquaredError())
 return model
K.clear_session()
model = keras_model(FT_COLS, mixture_title_cols, mixture_value_cols,
dense=70,
dout=0.15,
dense_reg = 0.000001,
act='elu',
descend_fraction = 0.9)
 model.summary()
manager_by_building = full_data.groupby('building_id')['manager_id'].apply(list)
 manager_model = FastText(size=10, window=3, min_count=1, workers=16)
 manager_model.build_vocab(sentences=manager_by_building)
 manager_model.train(sentences=manager_by_building.values,
total_examples=len(manager_by_building.values), epochs=5)
 manager_emb = full_data['manager_id'].apply(lambda x:manager_model[x] ).values
manager_emb = np.array([e.reshape(1,-1)for e in manager_emb] ).reshape(-1,10)
 manager_emb<train_on_grid>
oof, test_predictions, fold_errors = run_training(model, train, test,submission, 'fold', FT_COLS,
mixture_value_cols, mixture_title_cols, TARGET, cv_bm,
outlier_col='outlier_filter',
epochs=30,
batch_size=256,
verbose=False,
dense=70,
dout=0.15,
dense_reg = 0.000001,
act='elu', )
manager_by_building = full_data.groupby('building_id')['manager_id'].apply(list)
 manager_model = FastText(size=10, window=3, min_count=1, workers=16)
 manager_model.build_vocab(sentences=manager_by_building)
 manager_model.train(sentences=manager_by_building.values,
total_examples=len(manager_by_building.values), epochs=5)
 manager_emb = full_data['manager_id'].apply(lambda x:manager_model[x] ).values
manager_emb = np.array([e.reshape(1,-1)for e in manager_emb] ).reshape(-1,10)
 manager_emb<load_pretrained>
print('Save Out of Fold Predictions')
 oof = pd.DataFrame(columns=['oof_prediction'], index=train['id'], data=oof + TARGET_MEAN)
 oof.to_csv('oof_predictions.csv', index=True)
 oof.head(10 )
desc_vec = np.vstack(( pickle.load(open('/kaggle/input/text-featurizer/train_desc_vec_lg.pkl', 'rb')) ,
pickle.load(open('/kaggle/input/text-featurizer/test_desc_vec_lg.pkl', 'rb')))) 
<load_from_csv>
print('fold errors', fold_errors)
 print('fold error std', np.array(fold_errors ).std() )
<concatenate><EOS> 
submission['target'] = test_predictions + TARGET_MEAN
submission.to_csv('submission.csv', index=False)
 submission.head(5 )