Kaggle competition: Walmart Recruiting - Store Sales Forecasting
<SOS> metric: WMAE Kaggle data source: walmart-recruiting-store-sales-forecasting<count_duplicates>
warnings.simplefilter(action='ignore', category=FutureWarning)
 pd.set_option('display.max_columns', 200)
 pd.set_option('display.max_rows', 20)
 pd.set_option('display.min_rows', 20)
 sns.set_style(style='whitegrid' )
ans_user_and_category=df_train[df_train[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape<string_transform>
def WMAE(dataset, real, predicted):

 weights = dataset.IsHoliday.apply(lambda x: 5 if x else 1)
 return np.round(np.sum(weights*abs(real-predicted)) /(np.sum(weights)) , 2 )
def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same 
<feature_engineering>
def create_groupped_analysis(col, df):

 unique_values = df[[col, 'Weekly_Sales']].groupby(by=col ).mean().size
print(f'The number of unique values on the category '{col}' is {unique_values}
')
 print('-' * 40)
 print(f'The top 5 '{col}' with higher average 'Weekly_Sales':')
 print('-' * 40)
 return(
 df[[col, 'Weekly_Sales']] 
.groupby(by=col)
.mean() 
.sort_values('Weekly_Sales', ascending=False)
.head()
)
def add_external_features(df):
df['question_body'] = df['question_body'].progress_apply(lambda x: str(x))
df['question_body_num_words'] = df['question_body'].str.count('\S+')
 df['answer'] = df['answer'].progress_apply(lambda x: str(x))
df['answer_num_words'] = df['answer'].str.count('\S+')
 df['question_vs_answer_length'] = df['question_body_num_words']/df['answer_num_words']
df['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in tqdm(df[['answer_user_name', 'category']].values):
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
df['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
return df, np.array(handmade_features)
<create_dataframe>
def model_factory(model_type, **kwargs):

 if model_type in ['RandomForestRegressor', 'ExtraTreesRegressor'] :
n_estimators = kwargs.get('n_estimators')
 verbose = kwargs.get('verbose')
 n_jobs = kwargs.get('n_jobs')
 if n_estimators is None:
n_estimators = 50
verbose = 0 if verbose is None else verbose
if model_type == 'LinearRegression':
return LinearRegression()
elif model_type == 'KNN':
return KNeighborsRegressor()
elif model_type == 'Ridge':
return RidgeCV()
elif model_type == 'Lasso':
return LassoCV()
elif model_type == 'XGBRegressor':
return XGBRegressor()
elif model_type == 'RandomForestRegressor':
return RandomForestRegressor(
 n_estimators = n_estimators,
verbose = verbose,
n_jobs = n_jobs 
)
 elif model_type == 'ExtraTreesRegressor':
return ExtraTreesRegressor(
 n_estimators = n_estimators,
verbose = verbose,
n_jobs = n_jobs 
)
 else:
raise ValueError('Model not defined on factory.' )
df_train, handmade_features = add_external_features(df_train)
 df_test, handmade_features_test = add_external_features(df_test)
 df_train = pd.concat([df_train,pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1)
 df_test = pd.concat([df_test,pd.DataFrame(handmade_features_test, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1 )<normalization>
base_path = '/kaggle/input/walmart-recruiting-store-sales-forecasting/'
input_paths = glob.glob(f'{base_path}*.csv*')
 input_dfs = {
re.search(r'forecasting\/ (.*)\.csv', path, re.IGNORECASE)[1] : pd.read_csv(path)
 for path in input_paths
}
input_dfs.keys()
num_words_scaler = MinMaxScaler()
df_train[['question_body_num_words', 'answer_num_words']] = num_words_scaler.fit_transform(df_train[['question_body_num_words', 'answer_num_words']].values)
 df_test[['question_body_num_words', 'answer_num_words']] = num_words_scaler.transform(df_test[['question_body_num_words', 'answer_num_words']].values )<categorify>
df_train = input_dfs['features'].merge(input_dfs['stores'], how='left', on='Store')
 df_train = input_dfs['train'].merge(df_train, how='left', on=['Store', 'Date', 'IsHoliday'])
 df_train.head()
df=pd.concat([df,pd.get_dummies(df['host'], drop_first=False, prefix='host')],axis=1)
 df=pd.concat([df,pd.get_dummies(df['category'], drop_first=False, prefix='cat')],axis=1 )<define_variables>
df_train.isnull().mean() * 100
len(['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')] )<merge>
df_train.dropna(axis='columns', inplace=True )
df_train=pd.merge(df_train, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id')
 df_test = pd.merge(df_test, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id' )<categorify>
df_train.drop(['Temperature', 'Fuel_Price', 'Unemployment', 'CPI'], axis=1, inplace=True )
<categorify>
def create_features(df):

 def create_holiday_type_column(df, dates, holiday_type, name):
df.loc[
df['Date'].isin(dates),
'HolidayType'
] = holiday_type
df['HolidayType'] = -1
holiday_list = [ 
(['2010-02-12', '2011-02-11', '2012-02-10', '2013-02-08'], 'Super_Bowl'), 
(['2010-09-10','2011-09-09', '2012-09-07', '2013-09-06'], 'Labor_Day'), 
(['2010-11-26', '2011-11-25', '2012-11-23', '2013-11-29'], 'Thanksgiving'), 
(['2010-12-31', '2011-12-30', '2012-12-28', '2013-12-27'], 'Christmas')
 ]
for index in range(0, len(holiday_list)) :
holiday = holiday_list[index]
create_holiday_type_column(df, holiday[0], index, holiday[1])
 df.Date = pd.to_datetime(df.Date)
 df['Week'] = df.Date.dt.week
df['Year'] = df.Date.dt.year
df['Month'] = df.Date.dt.month
for x in df:
if df[x].dtypes == "int64":
df[x] = df[x].astype(float )
<drop_column>
create_features(df_train )
df_train.drop(['host', 'category'], inplace=True, axis=1)
 df_test.drop(['host', 'category'], inplace=True, axis=1 )<string_transform>
X = df_train.drop(['Weekly_Sales', 'Date'], axis=1)
 y = df_train['Weekly_Sales']
def _get_masks(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+[0]*(max_seq_length-len(tokens))
def _get_segments(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments=[]
first_sep=True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == '[SEP]':
current_segment_id=1
return segments+[0]*(max_seq_length-len(tokens))
def _trim_input_questions(title, question, max_sequence_length, t_max_len=30, q_max_len=479):
t = tokenizer2.tokenize(title)
 q = tokenizer2.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len+q_len+3)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if q_max_len > q_len:
q_new_len = q_len
else:
q_new_len = q_max_len
if t_new_len+q_new_len+3 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+q_new_len+3)))
 t = t[:t_new_len]
q = q[:q_new_len]
return t, q
def _get_ids(tokens, tokenizer, max_seq_length):
token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0]*(max_seq_length - len(token_ids))
return input_ids
def _convert_to_bert_inputs_questions(title, question,tokenizer, max_seq_length):
stoken = ['[CLS]']+title+['[SEP]']+question+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_questions(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
t, q = col['treated_question_title'], col['treated_question_body']
t,q = _trim_input_questions(t,q, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_questions(t,q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def _trim_input_answers(answer, max_sequence_length, a_max_len=510):
a = tokenizer2.tokenize(answer)
 a_len = len(a)
 if(a_len+2)> max_sequence_length:
if a_max_len > a_len:
a_new_len = a_len
else:
a_new_len = a_max_len
if a_new_len+2 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(a_new_len+2)))
 a = a[:a_new_len]
return a
def _convert_to_bert_inputs_answers(answer, tokenizer, max_seq_length):
stoken = ['[CLS]']+answer+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
a = col['treated_answer']
a = _trim_input_answers(a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns])
 def compute_spearmanr(trues, preds):
rhos=[]
for t, p in zip(trues.T, preds.T):
rhos.append(
 spearmanr(t,p ).correlation 
)
 return np.nanmean(rhos)
 test_predictions=[]
class CustomCallback(keras.callbacks.Callback):
def __init__(self, valid_data, test_data, test_predictions=test_predictions, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.test_predictions = test_predictions
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions=[]
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(self.valid_outputs, np.average(self.valid_predictions, axis=0))
print(f"
validation rho: {round(rho_val,4)}")
 self.test_predictions.append(self.model.predict(self.test_inputs, batch_size=self.batch_size))<load_pretrained>
categorical_features = X.columns[X.dtypes == 'O']
numeric_features = X.columns[X.dtypes == 'float64']
categories = [X[column].unique() for column in X[categorical_features]]
for cat in categories:
cat[cat == None] = 'missing'
transform_categorical_features = make_pipeline(
 SimpleImputer(missing_values=None,
strategy='constant',
fill_value='missing'),
OrdinalEncoder(categories=categories)
)
 transform_numeric_features = make_pipeline(
 SimpleImputer(strategy='constant', fill_value=-9999),
StandardScaler() 
)
 preprocessing_step = make_column_transformer(
(transform_categorical_features, categorical_features), 
(transform_numeric_features, numeric_features),
remainder='passthrough'
)
bert_config=BertConfig(unk_token="[QBODY]", pad_token="[ANS]" ).from_pretrained('.. /input/bert-tensorflow/bert-base-uncased-config.json',output_hidden_states=True)
 def bertModel() :
input_ids_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_q')
 input_mask_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_q')
 input_segments_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_q')
 input_new_features = keras.layers.Input(( 9,), dtype=tf.float32, name='input_new_features')
 input_ids_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_a')
 input_mask_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_a')
 input_segments_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_a')
 bert_model_q = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 bert_model_a = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 _, _, hidden_states_q = bert_model_q([input_ids_q,input_mask_q, input_segments_a])
 _, _, hidden_states_a = bert_model_a([input_ids_a,input_mask_a, input_segments_a])
 h12_q = tf.reshape(hidden_states_q[-1][:,0],(-1,1,768))
h11_q = tf.reshape(hidden_states_q[-2][:,0],(-1,1,768))
h10_q = tf.reshape(hidden_states_q[-3][:,0],(-1,1,768))
h09_q = tf.reshape(hidden_states_q[-4][:,0],(-1,1,768))
concat_hidden_q = keras.layers.Concatenate(axis=2 )([h12_q, h11_q, h10_q, h09_q])
 h12_a = tf.reshape(hidden_states_a[-1][:,0],(-1,1,768))
h11_a = tf.reshape(hidden_states_a[-2][:,0],(-1,1,768))
h10_a = tf.reshape(hidden_states_a[-3][:,0],(-1,1,768))
h09_a = tf.reshape(hidden_states_a[-4][:,0],(-1,1,768))
concat_hidden_a = keras.layers.Concatenate(axis=2 )([h12_a, h11_a, h10_a, h09_a])
 x = keras.layers.GlobalAveragePooling1D()(concat_hidden_q)
 x_a = keras.layers.GlobalAveragePooling1D()(concat_hidden_a)
 x = keras.layers.Concatenate()([x, input_new_features])
 x_a = keras.layers.Concatenate()([x_a,x, input_new_features])
 out = keras.layers.Dense(len([i for i in output_categories if i.startswith('question')]), activation='sigmoid', name='final_dense_output_q' )(x)
 out_a = keras.layers.Dense(len([i for i in output_categories if i.startswith('answer')]), activation='sigmoid', name='final_dense_output_a' )(x_a)
 out = keras.layers.Concatenate()([out,out_a])
 model = keras.models.Model(inputs=[input_ids_q, input_mask_q, input_segments_q,
input_new_features, input_ids_a, input_mask_a, input_segments_a]
, outputs=out)
 model.compile(loss='binary_crossentropy', optimizer = keras.optimizers.Adam(lr=3e-5))
return model<split>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body )<randomize_order>
regression_models = [
'LinearRegression',
'KNN',
'Ridge',
'Lasso',
'XGBRegressor',
'RandomForestRegressor',
'ExtraTreesRegressor'
]
best_model = None
best_error = math.inf
outputs = compute_output_arrays(df_train, output_categories)
 inputs_q = compute_input_array_questions(df_train, ['treated_question_title','treated_question_body'], tokenizer2, MAX_SEQUENCE_LENGTH)
 inputs_a = compute_input_array_answers(df_train, ['treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs_q = compute_input_array_questions(df_test, ['treated_question_title','treated_question_body'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs_a = compute_input_array_answers(df_test, ['treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH )<load_pretrained>
print('-' * 50)
 print('Training evaluation')
 print('-' * 50)
 for name in regression_models:
print(f':: {name} - Training Pipeline')
 regressor = model_factory(name)
 training_pipeline = make_pipeline(
 preprocessing_step,
regressor 
)
 kfold = KFold(n_splits=3)
 y_pred = cross_val_predict(
 training_pipeline,
X_train,
y_train,
cv = kfold,
n_jobs = -1,
verbose = 1 
)
 wmae_metric = WMAE(X_train, y_train, y_pred)
 if wmae_metric < best_error:
best_error = wmae_metric
best_model = name
plot_regression_results(
 y_train,
y_pred,
name, 
(r'$WMAE ={:.2f}$' ).format(wmae_metric)
)
 print('-' * 50 )
<define_variables>
print(f'Best model is {best_model}')
 print(f'Best score is {best_error}' )
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold>8:
keras.backend.clear_session()
model = bertModel()
train_inputs_q = [inputs_q[i][train_idx] for i in range(3)]
train_inputs_a = [inputs_a[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs_q = [inputs_q[i][valid_idx] for i in range(3)]
valid_inputs_a = [inputs_a[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
custom_callback = CustomCallback(valid_data=(valid_inputs_q +
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[valid_idx,:].values]+ valid_inputs_a,
valid_outputs),
test_data=test_inputs_q +
[df_test[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].values]+test_inputs_a,
batch_size=8, fold=fold)
 H = model.fit(train_inputs_q+
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[train_idx,:].values] + train_inputs_a,
train_outputs, batch_size=4, epochs=4, callbacks=[custom_callback])
 histories.append(H)
 model.save_weights(f'/kaggle/working/bert_base_fold_{fold}.hdf5' )<normalization>
n_estimators = [x for x in range(50, 250, 50)]
random_grid = {
'model__n_estimators': n_estimators
}
random_grid
def _get_masks_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_google_qa(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input_google_qa(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer_google_qa.tokenize(title)
 q = tokenizer_google_qa.tokenize(question)
 a = tokenizer_google_qa.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_google_qa(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_google_qa(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks_google_qa(stoken, max_sequence_length)
 input_segments = _get_segments_google_qa(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_google_qa(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance['treated_question_title'], instance['treated_question_body'], instance['treated_answer']
t, q, a = _trim_input_google_qa(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_google_qa(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays_google_qa(df, columns):
return np.asarray(df[columns])
 def bert_model_google_qa(learning_rate=4e-5,loss_function='binary_crossentropy'):
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[[input_word_ids, input_masks, input_segments]], outputs=out)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 return model 
<split>
def get_best_model_parameters(random_grid, X, y):
if best_model in ['ExtraTreesRegressor', 'RandomForestRegressor']:
model = model_factory(best_model)
 else:
return {'model__n_estimators': None}
grid_pipeline = Pipeline(
 [("preprocessing", preprocessing_step), 
("model", model)] 
)
 kfold = KFold(n_splits=3)
 model_random = GridSearchCV(
 grid_pipeline,
param_grid = random_grid,
cv = kfold,
verbose = 2,
n_jobs = -1 
)
 X_sample, _, y_sample, _ = train_test_split(X, y, test_size=0.9)
 return model_random.fit(X_sample, y_sample ).best_params_
best_params = get_best_model_parameters(random_grid, X, y)
 best_params
gkf_google_qa = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs_google_qa = compute_output_arrays_google_qa(df_train, output_categories)
 inputs_google_qa = compute_input_arays_google_qa(df_train, ['treated_question_title','treated_question_body','treated_answer'], tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
 test_inputs_google_qa = compute_input_arays_google_qa(df_test, ['treated_question_title','treated_question_body','treated_answer'],tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
<choose_model_class>
print('-' * 70)
 print(f'Best Model: {best_model}: {best_params}
')
 print('Tip: click on the Pipeline diagram below to have a better view of each step.')
 print('-' * 70)
 model = model_factory(
 best_model,
n_estimators = best_params['model__n_estimators'],
verbose = 1,
n_jobs = -1 
)
 best_model_pipeline = Pipeline(
 [ 
("preprocessing", preprocessing_step), 
("model", model)
 ] 
)
 set_config(display='diagram')
 best_model_pipeline.fit(X_train, y_train )
test_predictions_google_qa=[]
for fold,(train_idx, valid_idx)in enumerate(gkf_google_qa):
if fold<5:
keras.backend.clear_session()
model_qa = bert_model_google_qa()
print(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert_base_quest_kernel_{fold}.hdf5')
 model_qa.load_weights(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert_base_quest_kernel_{fold}.hdf5')
 preds_qa = model_qa.predict(test_inputs_google_qa , batch_size=8, verbose=1)
 test_predictions_google_qa.append(preds_qa)
 print(f'Fold:{fold} Completed' )<normalization>
df_test = input_dfs['features'].merge(input_dfs['stores'], how='left', on='Store')
 df_test = input_dfs['test'].merge(df_test, how='left', on=['Store', 'Date', 'IsHoliday'])
 create_features(df_test)
 df_test = df_test[list(X_train.columns)]
df_test.head()
test_preds_google_qa = [test_predictions_google_qa[i] for i in range(len(test_predictions_google_qa)) ]
test_preds_google_qa = [np.average(test_preds_google_qa, axis=0)for i in range(len(test_preds_google_qa)) ]
test_preds_google_qa = np.mean(test_preds_google_qa, axis=0)
 test_preds_google_qa.shape<compute_test_metric>
y_pred = best_model_pipeline.predict(df_test )
final_preds = np.average(np.array([test_preds, test_preds_google_qa]),axis=0)
 final_preds.shape<save_to_csv>
df_test['Weekly_Sales'] = y_pred
df_test['DatasetType'] = 'Predicted'
df_test['Date'] = input_dfs['test']['Date'].values
df_train['DatasetType'] = 'Actual'
df_train['Date'] = input_dfs['train']['Date'].values
df_result = df_train.append(df_test)
 graph_relation_to_weekly_sale('DatasetType', df_result, x='Date' )
df_sub.iloc[:, 1:] = final_preds
df_sub.to_csv('submission.csv', index=False )<load_from_csv>
input_dfs['sampleSubmission']['Weekly_Sales'] = y_pred
input_dfs['sampleSubmission'].to_csv('submission.csv',index=False)
 input_dfs['sampleSubmission']