Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<merge>
start_time = datetime.now()
%matplotlib inline
train_plot = train.merge(building, on='building_id', how='left')
 site = 0
meter_type = 1
primary_use = 'Education'
r = int(np.ceil(len(train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)]['building_id'].value_counts(dropna=False ).index.to_list())/2))
fig, axes = plt.subplots(r,2,figsize=(14, 36), dpi=100)
 for i, building_id in enumerate(train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)]['building_id'].value_counts(dropna=False ).index.to_list()):
train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)& 
(train_plot['building_id'] == building_id)][['timestamp', 'meter_reading']].set_index('timestamp' ).resample('H' ).mean() ['meter_reading'].plot(ax=axes[i%r][i//r],
alpha=0.8, label='By hour', color='tab:blue' ).set_ylabel('Mean meter reading', fontsize=13);
train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)& 
(train_plot['building_id'] == building_id)][['timestamp', 'meter_reading']].set_index('timestamp' ).resample('D' ).mean() ['meter_reading'].plot(ax=axes[i%r][i//r],
alpha=1, label='By day', color='tab:orange' ).set_xlabel('');
axes[i%r][i//r].legend() ;
axes[i%r][i//r].set_title('building_id: ' + str(building_id), fontsize=13);
plt.subplots_adjust(hspace=0.45)
 del train_plot,fig,axes,r
gc.collect() ;<data_type_conversions>
train = pd.read_table(".. /input/train.tsv")
 test = pd.read_table(".. /input/test.tsv")
 print(test.head() )
def compress_dataframe(df):

 result = df.copy()
for col in result.columns:
col_data = result[col]
dn = col_data.dtype.name
if dn == "object":
result[col] = pd.to_numeric(col_data.astype("category" ).cat.codes, downcast="integer")
 elif dn == "bool":
result[col] = col_data.astype("int8")
 elif dn.startswith("int")or(col_data.round() == col_data ).all() :
result[col] = pd.to_numeric(col_data, downcast="integer")
 else:
result[col] = pd.to_numeric(col_data, downcast='float')
 return result<data_type_conversions>
def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )
def set_time(df):
df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
return df
site_GMT_offsets = [-5, 0, -7, -5, -8, 0, -5, -5, -5, -6, -7, -5, 0, -6, -5, -5]
def weather_set_time(df,time_zone):
df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
 site_dfs = []
for site_id in df.site_id.unique() :
site_df = df[df.site_id == site_id].set_index("timestamp" ).reindex(time_zone)
 site_df.site_id = site_id
for col in [c for c in site_df.columns if c != "site_id"]:
site_df[f"had_{col}"] = ~site_df[col].isna()
site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
 site_df[col] = site_df[col].fillna(df[col].median())
 site_dfs.append(site_df)
 df = pd.concat(site_dfs ).reset_index()
for col in df.columns:
if df[col].isna().any() : df[f"had_{col}"] = ~df[col].isna()
return df
def _add_time_features(X):
return X.assign(tm_day_of_week=(( X.timestamp // 24)% 7), tm_hour_of_day=(X.timestamp % 24))
building = compress_dataframe(building.fillna(-1)).set_index("building_id")
 train = compress_dataframe(set_time(train))
test = compress_dataframe(set_time(test)).set_index("row_id")
 weather_train = compress_dataframe(weather_set_time(weather_train,range(8784)) ).set_index(["site_id", "timestamp"])
 weather_test = compress_dataframe(weather_set_time(weather_test,range(8784,26304)) ).set_index(["site_id", "timestamp"] )<merge>
le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train.category_name = le.transform(train.category_name)
 test.category_name = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train.brand_name = le.transform(train.brand_name)
 test.brand_name = le.transform(test.brand_name)
 print(train.head() )
def combined_data(df,weather):
df = compress_dataframe(df.join(building, on="building_id" ).join(weather,
on=["site_id", "timestamp"] ).fillna(-1))
return df.drop(columns=["meter_reading"]),df.meter_reading<define_variables>
raw_text = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text )
def make_is_bad_zero(Xy_subset, min_interval=48, summer_start=3000, summer_end=7500):
meter = Xy_subset.meter_id.iloc[0]
is_zero = Xy_subset.meter_reading == 0
if meter == 0:
return is_zero
transitions =(is_zero != is_zero.shift(1))
all_sequence_ids = transitions.cumsum()
ids = all_sequence_ids[is_zero].rename("ids")
 if meter in [2, 3]:
keep = set(ids[(Xy_subset.timestamp < summer_start)| 
(Xy_subset.timestamp > summer_end)].unique())
 is_bad = ids.isin(keep)&(ids.map(ids.value_counts())>= min_interval)
 elif meter == 1:
time_ids = ids.to_frame().join(Xy_subset.timestamp ).set_index("timestamp" ).ids
is_bad = ids.map(ids.value_counts())>= min_interval
jan_id = time_ids.get(0, False)
 dec_id = time_ids.get(8283, False)
 if(jan_id and dec_id and jan_id == time_ids.get(500, False)and
dec_id == time_ids.get(8783, False)) :
is_bad = is_bad &(~(ids.isin(set([jan_id, dec_id]))))
else:
raise Exception(f"Unexpected meter type: {meter}")
 result = is_zero.copy()
result.update(is_bad)
 return result
def find_bad_zeros(X, y):

 Xy = X.assign(meter_reading=y, meter_id=X.meter)
 is_bad_zero = Xy.groupby(["building_id", "meter"] ).apply(make_is_bad_zero)
 return is_bad_zero[is_bad_zero].index.droplevel([0, 1])
 def find_bad_sitezero(X):

 return X[(X.timestamp < 3378)&(X.site_id == 0)&(X.meter == 0)].index
def find_bad_building1099(X, y):

 return X[(X.building_id == 1099)&(X.meter == 2)&(y > 3e4)].index
def find_bad_rows(X, y):
return find_bad_zeros(X, y ).union(find_bad_sitezero(X)).union(find_bad_building1099(X, y))<save_to_csv>
train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 print(train.head() )
X, y = combined_data(train,weather_train)
 bad_rows = find_bad_rows(X, y)
 pd.Series(bad_rows.sort_values() ).to_csv("rows_to_drop.csv", header=False, index=False)
 X = X.drop(index=bad_rows)
 y = y.reindex_like(X)
 X = _add_time_features(X)
 X = compress_dataframe(X)
 X = X.drop(columns="timestamp")
 del bad_rows,train,weather_train
gc.collect() ;<prepare_output>
max_name_seq_set = 10
max_seq_item_description_set = 75
max_text = np.max([np.max(train.seq_name.max()),
np.max(test.seq_name.max()),
np.max(train.seq_item_description.max()),
np.max(test.seq_item_description.max())])+2
max_category = np.max([train.category_name.max() , test.category_name.max() ])+1
max_brand = np.max([train.brand_name.max() , test.brand_name.max() ])+1
max_cond_id = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1
y = np.log1p(y )<init_hyperparams>
max_text = np.max([np.max(train.seq_name.max()),
np.max(test.seq_name.max()),
np.max(train.seq_item_description.max()),
np.max(test.seq_item_description.max())])+ 2
max_category = np.max([train.category_name.max() , test.category_name.max() ])+ 1
max_brand = np.max([train.brand_name.max() , test.brand_name.max() ])+ 1
max_cond_id = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+ 1
params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'eval_metric': 'rmse',
'num_leaves': 40,
'subsample':0.8,
'learning_rate': 0.03,
'verbose': 1,
'reg_lambda':3
}
num_trees = 1000
categorical_features=['building_id', 'site_id', 'primary_use', 'had_air_temperature', 'had_cloud_coverage',
'had_dew_temperature', 'had_precip_depth_1_hr','had_sea_level_pressure', 'had_wind_direction',
'had_wind_speed', 'tm_day_of_week', 'tm_hour_of_day']<split>
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.90)
 print(dtrain.shape)
 print(dvalid.shape )
n_splits = 3
for val in X['meter'].unique() :
X1 = X[X['meter'] == val].drop(columns=['meter'])
 kf = StratifiedKFold(n_splits=n_splits,random_state=42)
 t = 0
for train_index, test_index in kf.split(X1, X1['tm_hour_of_day']):
train_features = X1.iloc[train_index]
train_target = y[X1.iloc[train_index].index]
test_features = X1.iloc[test_index]
test_target = y[X1.iloc[test_index].index]
d_train = lgb.Dataset(train_features, train_target, categorical_feature=categorical_features)
 d_eval = lgb.Dataset(test_features,test_target, categorical_feature=categorical_features)
 print("Building model meter :",val,'fold:',t)
 md=lgb.LGBMRegressor(boosting_type='gbdt', objective = 'regression',eval_metric='rmse',num_leaves=40,
n_estimators=num_trees, subsample=0.8, learning_rate=0.03,verbose=1,reg_lambda=3)
 md.fit(train_features,train_target,eval_set=(test_features, test_target),
early_stopping_rounds=200,verbose=20,categorical_feature=categorical_features,)
 locals() ['lgb_val{}_fold{}'.format(val,t)]=pickle.dumps(md)
 t += 1
del X1
del d_train, d_eval, train_features, test_features, md
gc.collect() ;<merge>
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=max_name_seq_set),
'item_desc': pad_sequences(dataset.seq_item_description, maxlen=max_seq_item_description),
'brand_name': np.array(dataset.brand_name),
'category_name': np.array(dataset.category_name),
'item_condition': np.array(dataset.item_condition_id),
'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test)
 print(X_train )
X = compress_dataframe(test.join(building, on="building_id" ).join(weather_test, on=["site_id", "timestamp"] ).fillna(-1))
X = compress_dataframe(_add_time_features(X))
X = X.drop(columns="timestamp")
 del test, weather_test
gc.collect() ;<prepare_output>
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.2
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(max_text, 50 )(name)
 emb_item_desc = Embedding(max_text, 50 )(item_desc)
 emb_brand_name = Embedding(max_brand, 10 )(brand_name)
 emb_category_name = Embedding(max_category, 10 )(category_name)
 emb_item_condition = Embedding(max_cond_id, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name),
Flatten()(emb_category_name),
Flatten()(emb_item_condition),
rnn_layer1,
rnn_layer2,
num_vars
])
 main_l = Dropout(dr_r )(Dense(64 )(main_l))
main_l = Activation('relu' )(main_l)
 main_l = Dropout(dr_r )(Dense(32 )(main_l))
main_l = Activation('relu' )(main_l)
 output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name, category_name, item_condition, num_vars], output)
 optimizers = Adam(lr=0.001, decay=0.0)
 model.compile(loss="mse", optimizer=optimizers, metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary()
result = np.zeros(len(X))
for val in X['meter'].unique() :
ix = np.nonzero(( X['meter'] == val ).to_numpy())
 for i in tqdm(range(n_splits)) :
model =pickle.loads(locals() ['lgb_val{}_fold{}'.format(val, i)])
 result[ix] += model.predict(X.iloc[ix].drop(columns=['meter']), num_iteration=model.best_iteration_)/n_splits
del model
gc.collect() ;
predictions = pd.DataFrame({
"row_id": X.index,
"meter_reading": np.clip(np.expm1(result), 0, None)
 })
 predictions.to_csv("submission.csv", index=False, float_format="%.4f" )<set_options>
model = get_model()
model.fit(X_train, dtrain.target, epochs=1, batch_size=1500,
validation_data=(X_valid, dvalid.target), verbose=1 )
py.init_notebook_mode(connected=True)
<set_options>
preds = model.predict(X_test, batch_size=1500)
 preds = target_scaler.inverse_transform(preds)
 preds = np.exp(preds)-1
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<prepare_x_and_y>
submission = test[["test_id"]]
submission["price"] = preds
submission.to_csv("output_submission.csv", index=False )
<drop_column><EOS> 
end_time = datetime.now()
execution_time = start_time - end_time
print(execution_time )