Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<feature_engineering>
pd.set_option('display.max_columns',500)
 pd.set_option('display.max_rows',100 )
def extract_date_columns(*dfs):
for df in dfs:
df['pickup_datetime_month'] = df.pickup_datetime.dt.month
df['pickup_datetime_hour'] = df.pickup_datetime.dt.hour
df['pickup_datetime_minute'] = df.pickup_datetime.dt.minute
df['pickup_datetime_dow'] = df.pickup_datetime.dt.dayofweek<drop_column>
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 item_cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' )
extract_date_columns(train_set, test_set )<compute_test_metric>
sales = sales[sales['item_price'] < 250000]
sales = sales[sales['item_cnt_day'] < 1000]
def _haversine_distance(lat_a, long_a, lat_b, long_b):

 del_lat_rad = lat_b - lat_a
del_long_rad = long_b - long_a
lat_a, long_a, lat_b, long_b, del_lat_rad, del_long_rad = \
map(np.radians, [lat_a, long_a, lat_b, long_b, del_lat_rad, del_long_rad])
 def _a(del_lat, del_long, lat_a, lat_b):
return(
(np.sin(del_lat / 2)** 2)+ 
(np.cos(lat_a)* np.cos(lat_b)*
np.sin(del_long / 2)** 2)
)
 def _c(a):
return 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a)) ;
R = 6371e3
a = _a(del_lat_rad, del_long_rad, lat_a, lat_b)
 c = _c(a)
 return R * c
def add_manhattan_haversine_distance(*dfs):

 for df in dfs:
inbounds = df.loc[(df['pickup_longitude'] <= -73.94)&
df['dropoff_longitude'] <= -73.94]
inbounds_idx = inbounds.index
oob = df.loc[~df.index.isin(inbounds_idx), :]
oob_idx = oob.index
ib_lat_a = inbounds['pickup_latitude']
ib_lat_b = inbounds['dropoff_latitude']
ib_long_a = inbounds['pickup_longitude']
ib_long_b = inbounds['dropoff_longitude']
oob_lat_a = oob['pickup_latitude']
oob_lat_b = oob['dropoff_latitude']
oob_long_a = oob['pickup_longitude']
oob_long_b = oob['dropoff_longitude']
df['city_distance'] = pd.Series(np.zeros(shape=(len(df.index))), index=df.index)
 df.loc[inbounds_idx, 'city_distance'] = _haversine_distance(ib_lat_a, ib_long_a,
ib_lat_a, ib_long_b)+ \
_haversine_distance(ib_lat_a, ib_long_a,
ib_lat_b, ib_long_a)
 df.loc[oob_idx, 'city_distance'] = _haversine_distance(oob_lat_a, oob_long_a,
oob_lat_b, oob_long_b)
<compute_test_metric>
sales.loc[sales.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
sales.loc[sales.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
sales.loc[sales.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
sales.loc[sales.shop_id == 39, 'shop_id'] = 40
test.loc[test.shop_id == 39, 'shop_id'] = 40
add_manhattan_haversine_distance(train_set, test_set )<drop_column>
sales['date'] = pd.to_datetime(sales['date'], format='%d.%m.%Y')
 sales['month'] = sales['date'].dt.month
sales['year'] = sales['date'].dt.year
sales = sales.drop('date', axis = 1 )
train_set.drop('pickup_datetime', axis=1, inplace=True)
 test_set.drop('pickup_datetime', axis=1, inplace=True)
 add_checkpoint('pre_modeling' )<prepare_x_and_y>
to_append = test[['shop_id', 'item_id']].copy()
to_append['date_block_num'] = sales['date_block_num'].max() + 1
to_append['year'] = 2015
to_append['month'] = 11
to_append['item_cnt_day'] = 0
to_append['item_price'] = 0
sales = pd.concat([sales, to_append], ignore_index=True, sort=False )
X = train_set.drop('trip_duration', axis=1)
 y = np.log1p(train_set['trip_duration'] )<split>
holiday_dict = {
1: 7,
2: 4,
3: 3,
4: 9,
5: 4,
6: 4,
7: 3,
8: 9,
9: 5,
10: 9,
11: 6,
12: 5,
}
sales['holidays_in_month'] = sales['month'].map(holiday_dict )
X_train, X_validate, y_train, y_validate = train_test_split(X, y )<train_model>
period = sales[['date_block_num','year','month','holidays_in_month']].drop_duplicates().reset_index(drop=True)
 period['days'] = period.apply(lambda r: monthrange(r.year, r.month)[1], axis=1 )
xgb_train = xgb.DMatrix(X_train, label=y_train)
 xgb_validate = xgb.DMatrix(X_validate, label=y_validate)
 evallist = [(xgb_train, 'training-set'),(xgb_validate, 'validation-set')]
xgb_params = {
'n_estimators': 250,
'eta': 0.2,
'subsample': 0.6,
'max_depth': 10,
'colsample_bytree': 0.75,
'min_child_weight': 25,
'objective': 'reg:linear',
'booster': 'gbtree',
'nthread': -1,
'silent': 1,
'eval_metric': 'rmse'
}<train_model>
shops['shop_city'] = cities
shops.loc[shops['shop_city'] == '!Якутск', 'shop_city' ] = 'Якутск'
ls = LabelEncoder()
shops['shop_city'] = ls.fit_transform(shops['shop_city'])
 shops['shop_type'] = shops['shop_name'].apply(lambda x: 'МТРЦ' if 'МТРЦ' in x
else 'ТРЦ' if 'ТРЦ' in x
else 'ТРК' if 'ТРК' in x
else 'ТЦ' if 'ТЦ' in x
else 'ТК' if 'ТК' in x
else 'NO_DATA')
 ls1 = LabelEncoder()
shops['shop_type'] = ls1.fit_transform(shops['shop_type'] )
if DO_XGBOOST:
xgb_regressor = xgb.train(xgb_params, xgb_train, 100, evallist,
early_stopping_rounds=100, maximize=False,
verbose_eval=10 )<choose_model_class>
ls = LabelEncoder()
item_cats['main_cat'] = ls.fit_transform(main_cat)
 sub_cat = item_cats['item_category_name'].str.split('-' ).map(lambda row : row[1].strip()
if len(row)> 1
else row[0].strip())
 item_cats['sub_cat'] = ls.fit_transform(sub_cat )
cat_features = [0, 6, 12, 13]
train_pool = Pool(X_train, label=y_train, cat_features=cat_features)
 validation_pool = Pool(X_validate, label=y_validate, cat_features=cat_features)
 cat_regressor = CatBoostRegressor(
 loss_function='RMSE',
eval_metric='RMSE',
iterations=250,
learning_rate=0.2,
depth=10
)<train_model>
items['name_1'], items['name_2'] = items['item_name'].str.split('[', 1 ).str
items['name_1'], items['name_3'] = items['item_name'].str.split('(', 1 ).str
items['name_2'] = items['name_2'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items['name_3'] = items['name_3'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items.drop('name_1', axis = 1 , inplace=True)
 items = items.fillna('0')
 ls = LabelEncoder()
items['name_2'] = ls.fit_transform(items['name_2'])
 items['name_3'] = ls.fit_transform(items['name_3'] )
if DO_CATBOOST:
cat_regressor.fit(train_pool, eval_set=validation_pool )<save_to_csv>
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
items['item_name'] = items['item_name'].apply(lambda x: name_correction(x))
items.head()
print('Unique item names after correction:', len(items['item_name'].unique()))
def create_xgb_submission() :
preds = xgb_regressor.predict(xgb.DMatrix(test_set))
preds_df = pd.DataFrame(np.expm1(preds), index=test_set.index)
 preds_df.columns = ['trip_duration']
preds_df.to_csv('submission.csv')
 display(pd.read_csv('submission.csv' ).head())
 def create_catboost_submission() :
pool = Pool(test_set, cat_features=cat_features)
 preds = cat_regressor.predict(pool)
 preds_df = pd.DataFrame(np.expm1(preds), index=test_set.index)
 preds_df.columns = ['trip_duration']
preds_df.to_csv('submission.csv')
 display(pd.read_csv('submission.csv' ).head())
 if DO_XGBOOST:
create_xgb_submission()
if DO_CATBOOST:
create_catboost_submission()<set_options>
items = items.merge(item_cats.drop('item_category_name', axis=1),
on = ['item_category_id'], how = 'left')
 items = items[['item_name','item_id','item_category_id','main_cat','sub_cat','name_2','name_3']]
%matplotlib inline 
<load_from_csv>
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype in ["float64"]]
int_cols = [c for c in df if df[c].dtype in ["int64"]]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int32)
 return df
def position_colum(df,column_name):

 assert column_name in df, "Column {} not in the dataset".format(column_name)
 colums = df.columns
for i in range(len(colums)) :
if(column_name == colums[i]):
position = i
return(position)
 def mean_encoding_reg(df, col_to_enc,target_col, nb_split):

 col = [col_to_enc,target_col]
df_small = df[col]
target_mean_fold_enc = []
kf = KFold(n_splits=nb_split, shuffle=False)
 for ind_tr, ind_val in kf.split(df_small):
mat_tr, mat_val = df_small.iloc[ind_tr], df_small.iloc[ind_val]
target_tr_mean_estimate = mat_tr.groupby(col_to_enc)[target_col].mean()
target_val_mean = df_small.loc[ind_val,col_to_enc].map(target_tr_mean_estimate)
 for mean in target_val_mean :
target_mean_fold_enc.append(mean)
 pos_categorical_feat = position_colum(df,col_to_enc)
 new_df = df.copy()
new_df.insert(pos_categorical_feat,col_to_enc +'_mean_enc',pd.Series(target_mean_fold_enc ).fillna(0))
return(( downcast_dtypes(new_df)) )
INPUT_PATH = Path('.. ','input')
 TRAIN_PATH = Path(INPUT_PATH, 'train.csv')
 TEST_PATH = Path(INPUT_PATH, 'test.csv')
 train = pd.read_csv(TRAIN_PATH, index_col='id', parse_dates=['pickup_datetime', 'dropoff_datetime'])
 train.head()<load_from_csv>
all_data = all_data.merge(period, on = ['date_block_num'], how = 'left')
 all_data = all_data.merge(shops.drop('shop_name', axis=1), on = ['shop_id'], how = 'left')
 all_data = all_data.merge(items.drop('item_name', axis=1), on = ['item_id'], how ='left' )
test = pd.read_csv(TEST_PATH, index_col='id', parse_dates=['pickup_datetime'])
 test.head()<feature_engineering>
all_data = downcast_dtypes(all_data)
 all_data.to_pickle('all_data.pkl')
 del grid, gb , cur_items
gc.collect() ;
def extract_date_info(df, cols):
for col in cols:
df[col + '_month'] = df[col].dt.month
df[col + '_week'] = df[col].dt.week
df[col + '_dow'] = df[col].dt.dayofweek
df[col + '_hour'] = df[col].dt.hour
df[col + '_date'] = df[col].dt.date
return df<drop_column>
all_data = agg_and_lag(all_data,col_groupby=['date_block_num'],
target_col='item_price_month',
new_col_name='date_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['item_id','date_block_num'],
target_col='item_price_month',
new_col_name='date_item_price_mean',
agg_function='mean',shift_range=[1,2])
 all_data = agg_and_lag(all_data,col_groupby=['shop_id','date_block_num'],
target_col='item_price_month',
new_col_name='date_shop_price_mean',
agg_function='mean',shift_range=[1,2])
 all_data = agg_and_lag(all_data,col_groupby=['sub_cat','shop_id','date_block_num'],
target_col='item_price_month',
new_col_name='date_shop_subcat_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['item_category_id','date_block_num'],
target_col='item_price_month',
new_col_name='date_itemcat_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['main_cat','date_block_num'],
target_col='item_price_month',
new_col_name='date_maincat_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['shop_type','date_block_num'],
target_col='item_price_month',
new_col_name='date_shop_type_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['item_id','shop_city','date_block_num'],
target_col='item_price_month',
new_col_name='date_item_shopcity_price_mean',
agg_function='mean',shift_range=[1] )
train = extract_date_info(train, ['pickup_datetime', 'dropoff_datetime'])
 test = extract_date_info(test, ['pickup_datetime'])
 train.head()<compute_test_metric>
all_data = agg_and_lag(all_data,col_groupby=['item_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_month_mean',
agg_function='mean', shift_range=[1,2])
 all_data = agg_and_lag(all_data,col_groupby=['shop_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_shop_month_mean',
agg_function='mean', shift_range=[1,2])
 all_data = agg_and_lag(all_data,col_groupby=['item_category_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_itemcat_month_mean',
agg_function='mean', shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['shop_id','item_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_shop_month_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['sub_cat','shop_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_subcat_month_mean',
agg_function='mean', shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['main_cat','shop_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_maincat_month_mean',
agg_function='mean', shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['item_id','shop_city','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_shopcity_month_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['date_block_num'],
target_col='item_cnt_month',
new_col_name='date_month_mean',
agg_function='mean', shift_range=[1] )
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))<concatenate>
index_cols = ['shop_id', 'item_id', 'date_block_num']
all_data = create_lags(all_data, index_cols=index_cols,
column_to_lag='item_cnt_month', shift_range=[1,2,3,12])
 all_data = create_lags(all_data, index_cols=index_cols,
column_to_lag='item_price_month', shift_range=[1,2] )
train['distance'] = train.apply(lambda row: haversine(row['pickup_latitude'], row['pickup_longitude'], row['dropoff_latitude'], row['dropoff_longitude']), axis=1 )<feature_engineering>
del sales, item_cats, items
<drop_column>
all_data = all_data[all_data['date_block_num'] >= 3]
gc.collect() ;
X = train.drop(columns=['pickup_datetime', 'dropoff_datetime', 'trip_duration',
'pickup_datetime_date', 'dropoff_datetime_month',
'dropoff_datetime_week','dropoff_datetime_dow',
'dropoff_datetime_hour', 'dropoff_datetime_date'] )<categorify>
all_data.to_pickle('all_data_lags.pkl' )
le = LabelEncoder()
le.fit(X['store_and_fwd_flag'])
 X['store_and_fwd_flag'] = le.transform(X['store_and_fwd_flag'])
 y = np.log1p(train['trip_duration'] )<categorify>
X_test = test.drop(columns=['pickup_datetime', 'pickup_datetime_date'])
 X_test['store_and_fwd_flag'] = le.transform(X_test['store_and_fwd_flag'])
 X_test['distance'] = test.apply(lambda row: haversine(row['pickup_latitude'], row['pickup_longitude'], row['dropoff_latitude'], row['dropoff_longitude']), axis=1)
 X_test.info()<compute_test_metric>
date = all_data['date_block_num']
target = all_data['item_cnt_month']
rf = RandomForestRegressor(n_estimators=100, n_jobs=-1)
<predict_on_test>
all_data = all_data.drop('item_cnt_month', axis = 1)
 all_data = all_data.drop('item_price_month', axis = 1 )
rf.fit(X, y)
 log_pred = rf.predict(X_test)
 pred = np.exp(log_pred)- np.ones(len(log_pred))<load_from_csv>
def correlatedDropper(df_data, thresh = 0.95):

 df = df_data.copy()
corr_matrix = df.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
cols_to_drop = [column for column in upper.columns if any(upper[column] > thresh)]
df = df.drop(cols_to_drop, axis=1)
 return df, cols_to_drop
SUBMIT_PATH = Path(INPUT_PATH, 'sample_submission.csv')
 submit = pd.read_csv(SUBMIT_PATH)
 arr_id = submit['id']
submission = pd.DataFrame({'id': arr_id, 'trip_duration': pred})
 submission.head()<save_to_csv>
test, col_drop = correlatedDropper(all_data, 0.95 )
submission.to_csv("submission.csv", index=False )<save_to_csv>
last_block_num = 33
X_test = all_data[date == 34]
X_valid = all_data[date == last_block_num]
X_train = all_data[date < last_block_num]
y_train = target.loc[date < last_block_num].values
y_valid = target.loc[date == last_block_num].values
categorical_feat_ind = np.where(X_train.dtypes != np.float32)[0]
print(categorical_feat_ind)
submission.to_csv("submission.csv", index=False )<define_variables>
del all_data, target, date
gc.collect()
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000<load_pretrained>
def catboost_rmse(X_train,y_train,
X_valid, y_valid,
loss_function='RMSE',
iterations=400,
random_seed=0,
learning_rate=0.15,
depth=5,
l2_leaf_reg=1,
one_hot_max_size=200,
min_data_in_leaf = 1,
bagging_temperature = 1,
border_count = 32,
max_ctr_complexity = 2 
):

 catmodel = CatBoostRegressor(
 loss_function= loss_function,
iterations=iterations,
random_seed=random_seed,
learning_rate=learning_rate,
depth=depth,
l2_leaf_reg=l2_leaf_reg,
one_hot_max_size=one_hot_max_size,
min_data_in_leaf = min_data_in_leaf,
bagging_temperature= bagging_temperature,
border_count=border_count,
max_ctr_complexity=max_ctr_complexity,
thread_count=-1,
od_type='Iter',
od_wait= 20,
task_type='GPU',
devices='0',
verbose = 100 
)
 catmodel.fit(X_train,
y_train,
eval_set=(X_valid,y_valid),
plot=False 
)
 predict = catmodel.predict(X_valid ).clip(0,20)
 best_rmse_train = catmodel.best_score_['learn']['RMSE']
best_rmse_valid = catmodel.best_score_['validation']['RMSE']
print('train_rmse : {:.4f}, test_rmse : {:.4f}'.format(best_rmse_train, best_rmse_valid))
r2 = r2_score(y_valid,predict)
 print('the R2 score is {:.4f}'.format(r2))
del predict
return - best_rmse_valid
if True:
importlib.reload(bert_utils )<init_hyperparams>
params_probe = {'bagging_temperature': 0.28300538794137076,
'border_count': 247.51179202909668,
'depth': 8.478936311173683,
'l2_leaf_reg': 7.73213368476749,
'learning_rate': 0.10290800942292017,
'max_ctr_complexity': 2.9452851946784433,
'min_data_in_leaf': 2.383458113416781,
'one_hot_max_size': 209.4502766887087}
config = {'attention_probs_dropout_prob':0.1,
'hidden_act':'gelu',
'hidden_dropout_prob':0.1,
'hidden_size':1024,
'initializer_range':0.02,
'intermediate_size':4096,
'max_position_embeddings':512,
'num_attention_heads':16,
'num_hidden_layers':24,
'type_vocab_size':2,
'vocab_size':30522}<choose_model_class>
print(Colours.green())
 optimizer_cat = optimize_catboost(X_train=X_train, y_train=y_train,
X_valid=X_valid, y_valid=y_valid,
param_probe=params_probe )
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )<categorify>
params_catboost = {'bagging_temperature': 0.28300538794137076,
'border_count': 247,
'depth': 8,
'l2_leaf_reg': 7.73213368476749,
'learning_rate': 0.10290800942292017,
'max_ctr_complexity': 2,
'min_data_in_leaf': 2.383458113416781,
'one_hot_max_size': 209,
'random_seed' : 0,
'thread_count' : -1,
'task_type' : 'GPU',
'devices' : '0',
'od_type' : 'Iter',
'od_wait' : 20,
'verbose' : 50
}
model= mk_model(config )<categorify>
catmodel = CatBoostRegressor(**params_catboost)
 catmodel.fit(X_train,y_train,
eval_set=(X_valid,y_valid),
plot=False 
)
 predict = catmodel.predict(X_valid ).clip(0,20)
 r2 = r2_score(y_valid,predict)
 print('r2_score est {}'.format(r2))
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=0.02,
n_best_size=5,
max_answer_length=30,
warmup_proportion=0.1,
learning_rate=1e-5,
num_train_epochs=3.0,
train_batch_size=32,
num_train_steps=100000,
num_warmup_steps=10000,
max_eval_steps=100,
use_tpu=False,
eval_batch_size=8,
max_predictions_per_seq=20 )<define_variables>
feature_useless_pred = feature_pred[feature_pred['Score'] < 0.01]
feature_useless_pred.Feature
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
print('number of test examples: %d, written to file: %d' %(n_examples,eval_writer.num_features))<categorify>
X_train_pred = X_train.drop(feature_useless_pred.Feature[:-1], axis = 1)
 X_valid_pred = X_valid.drop(feature_useless_pred.Feature[:-1], axis = 1)
 X_test_pred = X_test.drop(feature_useless_pred.Feature[:-1], axis = 1)
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int64)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
<categorify>
feature_useless_loss = feature_loss[feature_loss['Score'] < 0.0]
feature_useless_loss.Feature
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=32,drop_remainder=False )<predict_on_test>
catmodel_pred = CatBoostRegressor(**params_catboost)
 catmodel_pred.fit(X_train_pred,y_train,
eval_set=(X_valid_pred,y_valid),
plot=False 
)
 predict = catmodel_pred.predict(X_valid_pred ).clip(0,20)
 r2 = r2_score(y_valid,predict)
 print('r2_score est {}'.format(r2))
result=model.predict_generator(ds,verbose=1 )<load_pretrained>
print(Colours.green())
 optimizer_cat_loss = optimize_catboost(X_train=X_train_pred, y_train=y_train,
X_valid=X_valid_pred, y_valid=y_valid,
param_probe=params_probe )
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )<create_dataframe>
del X_train, X_valid
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx", "score"] )<import_modules>
X_train_pred.to_pickle('X_train_pred.pkl')
 X_valid_pred.to_pickle('X_valid_pred.pkl' )
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None<prepare_output>
def find_top_seeds(X_train,y_train,X_valid,y_valid,params, range_min = 0, range_max = 10):
metric_dict = pd.DataFrame(columns=['state','rmse'])
 for i in range(range_min,range_max):
print('--- random state : {} ---
'.format(i))
params['random_seed'] = i
catmodel = CatBoostRegressor(**params)
 catmodel.fit(X_train,y_train,
eval_set=(X_valid,y_valid),
plot=False 
)
 best_rmse_valid = catmodel.best_score_['validation']['RMSE']
metric_dict.loc[i-range_min] = [int(i)] + [best_rmse_valid]
metric_dict['state'] = metric_dict['state'].astype(np.int32)
 return metric_dict.sort_values(by='rmse', ascending = 'False' )
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}<find_best_params>
metric = find_top_seeds(X_train_pred,y_train,X_valid_pred, y_valid,
params=params_catboost, range_min=0, range_max=10 )
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]<count_duplicates>
predictions_test = []
for i in [4,8,6]:
print('random state {}'.format(i))
params_catboost['random_seed'] = i
catmodel_bagging_pred = CatBoostRegressor(**params_catboost)
 catmodel_bagging_pred.fit(X_train_pred,y_train,
eval_set=(X_valid_pred,y_valid),
plot=False 
)
 predict = catmodel_bagging_pred.predict(X_valid_pred ).clip(0,20)
 r2 = r2_score(y_valid,predict)
 print('r2_score est {}'.format(r2))
predictions_test.append(catmodel_bagging_pred.predict(X_test_pred ).clip(0,20))
def remove_duplicates(span):
start_end = []
for s in span:
cont = 0
if not start_end:
start_end.append(Span(s[0], s[1], s[2]))
cont += 1
else:
for i in range(len(start_end)) :
if start_end[i][0] == s[0] and start_end[i][1] == s[1]:
cont += 1
if cont == 0:
start_end.append(Span(s[0], s[1], s[2]))
return start_end<define_variables>
predictions = np.mean(np.array(predictions_test), axis = 0 )
def get_short_long_span(predictions, example):
sorted_predictions = sorted(predictions, reverse=True)
 short_span = []
long_span = []
for prediction in sorted_predictions:
score, _, summary, start_span, end_span = prediction
if score > 0:
short_span.append(Span(int(start_span), int(end_span), float(score)))
 short_span = remove_duplicates(short_span)
 for s in range(len(short_span)) :
for c in example.candidates:
start = short_span[s].start_token_idx
end = short_span[s].end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span.append(Span(int(c["start_token"]), int(c["end_token"]), float(short_span[s].score)))
 break
long_span = remove_duplicates(long_span)
 if not long_span:
long_span = [Span(-1, -1, -10000.0)]
if not short_span:
short_span = [Span(-1, -1, -10000.0)]
return short_span, long_span<find_best_params>
dt_test_opt = pd.DataFrame()
dt_test_opt['item_cnt_month'] = predictions
dt_test_opt.index.name='ID'
dt_test_opt.to_csv('submission_featurev11_bagging_[4_6_8].csv' )