Kaggle competition: Walmart Recruiting - Store Sales Forecasting
<SOS> metric: WMAE Kaggle data source: walmart-recruiting-store-sales-forecasting<predict_on_test>
sns.set_style("whitegrid")
 warnings.filterwarnings('ignore')
def predict_result_question(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 21))
model.eval() ;
tk0 = tqdm(enumerate(test_loader),position=0, leave=True)
 for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device),
usefeatures = x_batch[3].to(device)
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output
def predict_result_answer(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 9))
model.eval() ;
tk0 = tqdm(enumerate(test_loader),position=0, leave=True)
 for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device),
usefeatures = x_batch[3].to(device)
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output<load_pretrained>
train_data=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/train.csv.zip',parse_dates=True)
 sample_submission=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/sampleSubmission.csv.zip')
 features_data=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/features.csv.zip',parse_dates=True)
 stores_data=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/stores.csv')
 test_data=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/test.csv.zip' )
set_seeds(SEED)
 question_tokenizer = RobertaTokenizer.from_pretrained(ROBERTA_PRETRAINED)
 input_categories_q = list(train.columns[[1,2]]);
robert_config_question = RobertaConfig.from_pretrained(ROBERTA_PRETRAINED)
 robert_config_question.num_labels = 21
robert_config_question.output_hidden_states = True
answer_tokenizer = RobertaTokenizer.from_pretrained(ROBERTA_PRETRAINED)
 input_categories_a = list(train.columns[[1,2,5]]);
robert_config_answer = RobertaConfig.from_pretrained(ROBERTA_PRETRAINED)
 robert_config_answer.num_labels = 9
robert_config_answer.output_hidden_states = True
device = 'cuda' if torch.cuda.is_available() else 'cpu'<feature_engineering>
df = pd.DataFrame()
for i in tq.tqdm(range(1,46)) :
model=Prophet()
filled=features_data[(( features_data['Store']==i)&(features_data['Date']<'2013-05-03')) ][['Date','CPI']]
tserie = filled.rename(columns = {'Date': 'ds', 'CPI': 'y'}, inplace = False)
 tserie =tserie.sort_values(by=['ds'])
 tserie['ds'] = pd.to_datetime(tserie['ds'])
 model.fit(tserie)
 future_dates=model.make_future_dataframe(periods=13,freq = 'W',include_history =True)
 future_dates['ds'] = future_dates['ds'].apply(lambda x: x + timedelta(days=5))
prediction=model.predict(future_dates)
 df=df.append(prediction)
 df.reset_index(drop=True)
 features_data['CPI1']=np.nan
for i,j in enumerate(df['yhat']):
features_data['CPI1'].iloc[i]=j
features_data=features_data.drop(['CPI'],axis=1)
 features_data = features_data.rename(columns = {'CPI1': 'CPI'} )
test_inputs_q = compute_input_arays(df = test, columns = input_categories_q, tokenizer = question_tokenizer, max_sequence_length=512)
 lengths_test_q = np.argmax(test_inputs_q[0] == 0, axis=1)
 lengths_test_q[lengths_test_q == 0] = test_inputs_q[0].shape[1]
test_inputs_a = compute_input_arays_answers(df = test, columns = input_categories_a, tokenizer = answer_tokenizer, max_sequence_length=512)
 lengths_test_a = np.argmax(test_inputs_a[0] == 0, axis=1)
 lengths_test_a[lengths_test_a == 0] = test_inputs_a[0].shape[1]<create_dataframe>
df = pd.DataFrame()
for i in tq.tqdm(range(1,46)) :
model=Prophet()
filled=features_data[(( features_data['Store']==i)&(features_data['Date']<'2013-05-03')) ][['Date','Unemployment']]
tserie = filled.rename(columns = {'Date': 'ds', 'Unemployment': 'y'}, inplace = False)
 tserie =tserie.sort_values(by=['ds'])
 tserie['ds'] = pd.to_datetime(tserie['ds'])
 model.fit(tserie)
 future_dates=model.make_future_dataframe(periods=13,freq = 'W',include_history =True)
 future_dates['ds'] = future_dates['ds'].apply(lambda x: x + timedelta(days=5))
prediction=model.predict(future_dates)
 df=df.append(prediction)
 df.reset_index(drop=True)
 features_data['Unemployment1']=np.nan
for i,j in enumerate(df['yhat']):
features_data['Unemployment1'].iloc[i]=j
features_data=features_data.drop(['Unemployment'],axis=1)
 features_data = features_data.rename(columns = {'Unemployment1': 'Unemployment'} )
NUM_FOLDS = 5
BATCH_SIZE = 8
epochs = 5
test_predictions_q = []
test_predictions_a = []
test_predictions = []
test_set_q = QuestDataset(inputs=test_inputs_q,
usefeatures=torch.FloatTensor(X_test),
lengths=lengths_test_q, labels=None)
 test_loader_q = DataLoader(test_set_q, batch_size=32, shuffle=False)
 result_q = np.zeros(( len(test), 21))
test_set_a = QuestDataset(inputs=test_inputs_a,
usefeatures=torch.FloatTensor(X_test),
lengths=lengths_test_a, labels=None)
 test_loader_a = DataLoader(test_set_a, batch_size=32, shuffle=False)
 result_a = np.zeros(( len(test), 9))
set_seeds(SEED )<define_variables>
stores = stores_data.merge(features_data, on ='Store' , how = 'left')
 final_data_train = train_data.merge(stores, on = ['Store', 'Date', 'IsHoliday'], how = 'left' )
ROBERTA_SAVED_MODEL_ANSWERS = ['.. /input/roberta-dataset-answers/',
'.. /input/roberta-seed-42-answers/' ,
'.. /input/roberta-dataset-seed101-answers/',
'.. /input/roberta-dataset-seed1029-answers'
]
ROBERTA_SAVED_MODEL_QUESTIONS = ['.. /input/roberta-dataset-questions/',
'.. /input/roberta-seed-42-questions/',
'.. /input/roberta-dataset-seed101-questions/',
'.. /input/roberta-dataset-seed1029-questions']
for qa in range(len(ROBERTA_SAVED_MODEL_ANSWERS)) :
for f in range(NUM_FOLDS):
model_q = RobertaForSequenceClassification_v2.from_pretrained(ROBERTA_PRETRAINED, config=robert_config_question)
 model_q.zero_grad() ;
model_q.to(device);
torch.cuda.empty_cache()
dict_load = torch.load(path_join(ROBERTA_SAVED_MODEL_QUESTIONS[qa], 'Q_model_save_fold_{}.pt'.format(f)))
 model_q.load_state_dict(dict_load)
 out_q = predict_result_question(model_q, test_loader_q)
 model_a = RobertaForSequenceClassification_v2.from_pretrained(ROBERTA_PRETRAINED,config=robert_config_answer)
 model_a.zero_grad() ;
model_a.to(device);
torch.cuda.empty_cache()
dict_load = torch.load(path_join(ROBERTA_SAVED_MODEL_ANSWERS[qa], 'A_model_save_fold_{}.pt'.format(f)))
 model_a.load_state_dict(dict_load)
 out_a = predict_result_answer(model_a, test_loader_a)
 preds = np.hstack(( out_q,out_a))
if POST_PROCESS:
for l in class_labels_dict.keys() :
preds[:,class_labels_dict[l]] = pd.cut(preds[:,class_labels_dict[l]], train[l].nunique() , labels=train[l].value_counts().index.sort_values())
 test_predictions.append(preds)
<feature_engineering>
stores = stores_data.merge(features_data, on ='Store' , how = 'left')
 final_data_test = test_data.merge(stores, on = ['Store', 'Date', 'IsHoliday'], how = 'left' )
test_preds_roberta = np.mean(test_predictions,axis=0)
 idx = np.argwhere(test.category!='CULTURE' ).flatten()
arr = 0.0001 * np.clip(np.random.normal(size=len(idx)) ,0.1,0.99)
 test_preds_roberta[idx,19] = arr<load_from_csv>
def markdown_imputation(final_data):
final_data.loc[final_data.MarkDown1.isnull() ,'MarkDown1']= 0
final_data.loc[final_data.MarkDown2.isnull() ,'MarkDown2']= 0
final_data.loc[final_data.MarkDown3.isnull() ,'MarkDown3']= 0
final_data.loc[final_data.MarkDown4.isnull() ,'MarkDown4']= 0
final_data.loc[final_data.MarkDown5.isnull() ,'MarkDown5']= 0
return final_data
class_labels = ['question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation']
submission = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv')
 submission.iloc[:,1:] = test_preds_bert * 0.70 + test_preds_roberta*0.30
for l in class_labels:
submission[l] = pd.cut(submission[l], train[l].nunique() , labels=train[l].value_counts().index.sort_values())
 submission.to_csv('submission.csv', index=False)
 submission.head() 
<import_modules>
def train_temp_bins(final_data):
temp_100_110_f=final_data[(( final_data.Temperature>100)&(final_data.Temperature< 110)) ].Weekly_Sales.sum()
temp_90_100_f=final_data[(( final_data.Temperature>90)&(final_data.Temperature< 100)) ].Weekly_Sales.sum()
temp_80_90_f=final_data[(( final_data.Temperature>80)&(final_data.Temperature< 90)) ].Weekly_Sales.sum()
temp_70_80_f=final_data[(( final_data.Temperature>70)&(final_data.Temperature< 80)) ].Weekly_Sales.sum()
temp_60_70_f=final_data[(( final_data.Temperature>60)&(final_data.Temperature< 70)) ].Weekly_Sales.sum()
temp_50_60_f=final_data[(( final_data.Temperature>50)&(final_data.Temperature< 60)) ].Weekly_Sales.sum()
temp_40_50_f=final_data[(( final_data.Temperature>40)&(final_data.Temperature< 50)) ].Weekly_Sales.sum()
temp_30_40_f=final_data[(( final_data.Temperature>30)&(final_data.Temperature< 40)) ].Weekly_Sales.sum()
temp_0_30_f=final_data[(( final_data.Temperature>0)&(final_data.Temperature< 30)) ].Weekly_Sales.sum()
temp_less_than_0_f=final_data[(( final_data.Temperature>-10)&(final_data.Temperature< 0)) ].Weekly_Sales.sum()
final_data['Temp_bins'] = np.nan
final_data.loc[(( final_data.Temperature>-10)&(final_data.Temperature<0)) ,'Temp_bins']= temp_less_than_0_f
final_data.loc[(( final_data.Temperature>0)&(final_data.Temperature< 30)) ,'Temp_bins']= temp_0_30_f
final_data.loc[(( final_data.Temperature>30)&(final_data.Temperature< 40)) ,'Temp_bins']= temp_30_40_f
final_data.loc[(( final_data.Temperature>40)&(final_data.Temperature< 50)) ,'Temp_bins']= temp_40_50_f
final_data.loc[(( final_data.Temperature>50)&(final_data.Temperature< 60)) ,'Temp_bins']= temp_50_60_f
final_data.loc[(( final_data.Temperature>60)&(final_data.Temperature< 70)) ,'Temp_bins']= temp_60_70_f
final_data.loc[(( final_data.Temperature>70)&(final_data.Temperature< 80)) ,'Temp_bins']= temp_70_80_f
final_data.loc[(( final_data.Temperature>80)&(final_data.Temperature< 90)) ,'Temp_bins']= temp_80_90_f
final_data.loc[(( final_data.Temperature>90)&(final_data.Temperature< 100)) ,'Temp_bins']= temp_90_100_f
final_data.loc[(( final_data.Temperature>100)&(final_data.Temperature< 110)) ,'Temp_bins']= temp_100_110_f
final_data.loc[final_data.Temp_bins.isnull() ,'Temp_bins']= 0
list1=[temp_less_than_0_f,temp_0_30_f,temp_30_40_f,temp_40_50_f,temp_50_60_f,temp_60_70_f,temp_70_80_f,temp_80_90_f,temp_90_100_f,temp_100_110_f]
return final_data,list1
import numpy as np
from transformers import BertTokenizer, BertModel
import re
import pandas as pd<load_from_csv>
def test_temp_bins(final_data,list1):
final_data['Temp_bins'] = np.nan
final_data.loc[(( final_data.Temperature>-10)&(final_data.Temperature<0)) ,'Temp_bins']= list1[0]
final_data.loc[(( final_data.Temperature>0)&(final_data.Temperature< 30)) ,'Temp_bins']= list1[1]
final_data.loc[(( final_data.Temperature>30)&(final_data.Temperature< 40)) ,'Temp_bins']= list1[2]
final_data.loc[(( final_data.Temperature>40)&(final_data.Temperature< 50)) ,'Temp_bins']= list1[3]
final_data.loc[(( final_data.Temperature>50)&(final_data.Temperature< 60)) ,'Temp_bins']= list1[4]
final_data.loc[(( final_data.Temperature>60)&(final_data.Temperature< 70)) ,'Temp_bins']= list1[5]
final_data.loc[(( final_data.Temperature>70)&(final_data.Temperature< 80)) ,'Temp_bins']= list1[6]
final_data.loc[(( final_data.Temperature>80)&(final_data.Temperature< 90)) ,'Temp_bins']= list1[7]
final_data.loc[(( final_data.Temperature>90)&(final_data.Temperature< 100)) ,'Temp_bins']= list1[8]
final_data.loc[(( final_data.Temperature>100)&(final_data.Temperature< 110)) ,'Temp_bins']= list1[9]
final_data.loc[final_data.Temp_bins.isnull() ,'Temp_bins']= 0
return final_data
sample_submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 test = pd.read_csv(".. /input/google-quest-challenge/test.csv")
 train = pd.read_csv(".. /input/google-quest-challenge/train.csv" )<define_variables>
def split(final_data):
final_data['Date'] = pd.to_datetime(final_data['Date'])
 final_data['Year'] = final_data['Date'].dt.year
final_data['Month']= final_data['Date'].dt.month
final_data['Week'] = final_data['Date'].dt.week
final_data['Day'] = final_data['Date'].dt.day
return final_data
question_output_columns = [col for col in output_columns \
if 'question' in col]
answer_output_colmns = [col for col in output_columns
if col not in question_output_columns]<load_pretrained>
def days_from_christmas_for_train(x):
if x['Year']== 2010 :
diff=datetime.datetime(2010, 12, 31)-x['Date']
return diff.days
if(( x['Year']== 2011)and(x['Date']< datetime.datetime(2011, 12, 30))):
diff=datetime.datetime(2011, 12, 30)-x['Date']
return diff.days
else:
return 0
tokenizer = BertTokenizer.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-tokenizer' )<define_variables>
def days_from_christmas_for_test(x):
if x['Year']== 2010 :
diff=datetime.datetime(2010, 12, 31)-x['Date']
return diff.days
if(( x['Year']== 2011)and(x['Date']< datetime.datetime(2011, 12, 30))):
diff=datetime.datetime(2011, 12, 30)-x['Date']
return diff.days
if(( x['Year']== 2012)and(x['Date']< datetime.datetime(2012, 12, 28))):
diff=datetime.datetime(2012, 12, 28)-x['Date']
return diff.days
if(( x['Year']== 2013)and(x['Date']< datetime.datetime(2013, 12, 27))):
diff=datetime.datetime(2013, 12, 27)-x['Date']
return diff.days
else:
return 0
max_length_map = {'question_title': 32,
'question_body': 512,
'answer': 512
}<categorify>
def days_from_thanksgiving_for_train(x):
if(( x['Year']== 2010)and(x['Date']< datetime.datetime(2010, 11, 26))):
diff=datetime.datetime(2010, 11, 26)-x['Date']
return diff.days
if(( x['Year']== 2011)and(x['Date']< datetime.datetime(2011, 11, 25))):
diff=datetime.datetime(2011, 11, 25)-x['Date']
return diff.days
else:
return 0
def txt_re(txt):
txt = txt.strip()
txt = re.sub('https?.*$', '', txt)
 txt = re.sub('https?.*\s', '', txt)
 txt = re.sub('
+', ' ', txt)
 txt = re.sub('\r+', ' ', txt)
 txt = re.sub('\t+', ' ', txt)
 txt = re.sub('&gt;', '>', txt)
 txt = re.sub('&lt;', '<', txt)
 txt = re.sub('&amp;', '&', txt)
 txt = re.sub('&quot;', '"', txt)
 return txt<categorify>
def days_from_thanksgiving_for_test(x):
if(( x['Year']== 2010)and(x['Date']< datetime.datetime(2010, 11, 26))):
diff=datetime.datetime(2010, 11, 26)-x['Date']
return diff.days
if(( x['Year']== 2011)and(x['Date']< datetime.datetime(2011, 11, 25))):
diff=datetime.datetime(2011, 11, 25)-x['Date']
return diff.days
if(( x['Year']== 2012)and(x['Date']< datetime.datetime(2012, 11, 23))):
diff=datetime.datetime(2012, 11, 23)-x['Date']
return diff.days
if(( x['Year']== 2013)and(x['Date']< datetime.datetime(2013, 11, 29))):
diff=datetime.datetime(2013, 11, 29)-x['Date']
return diff.days
else:
return 0
def get_input(txt, pair_txt, tokenizer, max_length):
txt = txt_re(txt)
 txt = tokenizer.encode_plus(txt, pair_txt, add_special_tokens=True,max_length=max_length, \
pad_to_max_length='right')
 input_ids = txt['input_ids']
segment_masks = txt['token_type_ids']
input_masks = txt['attention_mask']
return input_ids, segment_masks, input_masks<define_variables>
def holiday_type(x):
if(x['IsHoliday']== 1)&(x['Week']==6):
return 1
elif(x['IsHoliday']== 1)&(x['Week']==36):
return 2
elif(x['IsHoliday']== 1)&(x['Week']==47):
return 3
elif(x['IsHoliday']== 1)&(x['Week']==52):
return 4
else:
return 0
question_train, answer_train = computer_input_array(train)
 question_test, answer_test = computer_input_array(test )<define_variables>
def holiday_label(final_data):
final_data.loc[(final_data.IsHoliday==True),'IsHoliday']= 1
final_data.loc[(final_data.IsHoliday==False),'IsHoliday']= 0
return final_data
labels = train[output_columns].values.tolist()<define_variables>
def type_label(final_data):
final_data.loc[(final_data.Type=='A'),'Type']= 1
final_data.loc[(final_data.Type=='B'),'Type']= 2
final_data.loc[(final_data.Type=='C'),'Type']= 3
return final_data
train_dict = {'question': question_train, 'answer': answer_train, 'label': labels}
test_dict = {'question': question_test, 'answer': answer_test}<save_model>
def holiday_in_week_train(final_data):
dates =[]
for ptr in holidays.US(years = 2010 ).items() :
dates.append(ptr[0])
 for ptr in holidays.US(years = 2011 ).items() :
dates.append(ptr[0])
 for ptr in holidays.US(years = 2012 ).items() :
dates.append(ptr[0])
 holiday_count=[]
for index, row in final_data.iterrows() :
dat = final_data['Date'][index]
dt=[]
for i in range(0,5):
dt.append(dat - datetime.timedelta(days = i))
for i in range(1,3):
dt.append(dat + datetime.timedelta(days = i))
count = 0
for date in dates:
if date in dt:
count +=1
holiday_count.append(count)
 return holiday_count
torch.save(train_dict, './data/train_data.t7')
 torch.save(test_dict, './data/test_data.t7' )<import_modules>
def holiday_in_week_test(final_data):
dates =[]
for ptr in holidays.US(years = 2010 ).items() :
dates.append(ptr[0])
 for ptr in holidays.US(years = 2011 ).items() :
dates.append(ptr[0])
 for ptr in holidays.US(years = 2012 ).items() :
dates.append(ptr[0])
 for ptr in holidays.US(years = 2013 ).items() :
dates.append(ptr[0])
 holiday_count=[]
for index, row in final_data.iterrows() :
dat = final_data['Date'][index]
dt=[]
for i in range(0,5):
dt.append(dat - datetime.timedelta(days = i))
for i in range(1,3):
dt.append(dat + datetime.timedelta(days = i))
count = 0
for date in dates:
if date in dt:
count +=1
holiday_count.append(count)
 return holiday_count
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BertTokenizer, BertModel
from scipy.stats import spearmanr<choose_model_class>
final_data_train=markdown_imputation(final_data_train)
 final_data_train=weekly_sales_imputation(final_data_train)
 final_data_train,list1=train_temp_bins(final_data_train)
 final_data_train=split(final_data_train)
 final_data_train['diff_from_christmas'] = final_data_train.apply(days_from_christmas_for_train, axis=1)
 final_data_train['days_from_thanksgiving'] = final_data_train.apply(days_from_thanksgiving_for_train, axis=1)
 final_data_train['IsHoliday_bins'] = final_data_train.apply(holiday_type, axis=1)
 final_data_train=holiday_label(final_data_train)
 final_data_train=type_label(final_data_train)
 final_data_train['Holidays'] = np.array(holiday_in_week_train(final_data_train))
class Model_v1(nn.Module):
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-pytorch-model')
 self.dropout = nn.Dropout(0.2)
 self.pool = nn.AvgPool2d(( 512, 1))
self.output = nn.Linear(768 * 2, 30)
 def forward(self, q_inputs, q_input_masks, q_segment_masks, \
a_inputs, a_input_masks, a_segment_masks):
q_outputs = self.bert(q_inputs, attention_mask=q_input_masks, token_type_ids=q_segment_masks)
 q_x = q_outputs[0]
q_x = self.dropout(q_x)
 q_x = q_x.unsqueeze(1)
 q_x = self.pool(q_x)
 q_x = q_x.squeeze(1 ).squeeze(1)
 a_outputs = self.bert(a_inputs, attention_mask=a_input_masks, token_type_ids=a_segment_masks)
 a_x = a_outputs[0]
a_x = self.dropout(a_x)
 a_x = a_x.unsqueeze(1)
 a_x = self.pool(a_x)
 a_x = a_x.squeeze(1 ).squeeze(1)
 t_q_a = torch.cat(( q_x, a_x), -1)
 output = self.output(t_q_a)
 x = torch.sigmoid(output)
 return x<choose_model_class>
final_data_test=markdown_imputation(final_data_test)
 final_data_test=test_temp_bins(final_data_test,list1)
 final_data_test=split(final_data_test)
 final_data_test['diff_from_christmas'] = final_data_test.apply(days_from_christmas_for_test, axis=1)
 final_data_test['days_from_thanksgiving'] = final_data_test.apply(days_from_thanksgiving_for_test, axis=1)
 final_data_test['IsHoliday_bins'] = final_data_test.apply(holiday_type, axis=1)
 final_data_test=holiday_label(final_data_test)
 final_data_test=type_label(final_data_test)
 final_data_test['Holidays'] = np.array(holiday_in_week_test(final_data_test))
class Model_v2(nn.Module):
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-pytorch-model')
 self.dropout = nn.Dropout(0.2)
 self.pool = nn.AvgPool2d(( 512, 1))
self.output_q = nn.Linear(768, 21)
 self.output_a = nn.Linear(768, 9)
 def forward(self, q_inputs, q_input_masks, q_segment_masks, \
a_inputs, a_input_masks, a_segment_masks):

 q_outputs = self.bert(q_inputs, attention_mask=q_input_masks, token_type_ids=q_segment_masks)
 q_x = q_outputs[0]
q_x = self.dropout(q_x)
 q_x = q_x.unsqueeze(1)
 q_x = self.pool(q_x)
 q_x = q_x.squeeze(1 ).squeeze(1)
 a_outputs = self.bert(a_inputs, attention_mask=a_input_masks, token_type_ids=a_segment_masks)
 a_x = a_outputs[0]
a_x = self.dropout(a_x)
 a_x = a_x.unsqueeze(1)
 a_x = self.pool(a_x)
 a_x = a_x.squeeze(1 ).squeeze(1)
 output_q = self.output_q(q_x)
 output_a = self.output_a(a_x)
 output = torch.cat(( output_q, output_a), -1)
 x = torch.sigmoid(output)
 return x<import_modules>
final_data_train=final_data_train.reset_index(drop=True )
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import DataLoader,Dataset, TensorDataset
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GroupKFold
from transformers import AdamW, get_linear_schedule_with_warmup<load_pretrained>
final_data_train=final_data_train[['Store','Dept','IsHoliday','Size','Week','Type','Year','Weekly_Sales','Holidays','Day']]
final_data_test=final_data_test[['Store','Dept','IsHoliday','Size','Week','Type','Year','Holidays','Day']]
final_data_train['IsHoliday']=final_data_train['IsHoliday'].astype('bool')
 final_data_test['IsHoliday']=final_data_test['IsHoliday'].astype('bool')
 final_data_train['Type']=final_data_train['Type'].astype('int')
 final_data_test['Type']=final_data_test['Type'].astype('int')
 y = final_data_train['Weekly_Sales']
X = final_data_train.drop(['Weekly_Sales'], axis=1)
 rf_Model = RandomForestRegressor(n_estimators= 140,max_depth=27,n_jobs = -1)
 rf_Model.fit(X, y)
 y_hat= rf_Model.predict(final_data_test )
data = torch.load('./data/train_data.t7')
 test_data = torch.load('./data/test_data.t7' )<create_dataframe>
sample_submission['Weekly_Sales'] = list(y_hat )
test_set = TensorDataset(torch.LongTensor(np.array(test_data['question'])) ,
torch.LongTensor(np.array(test_data['answer'])) )<load_pretrained>
sample_submission.to_csv('submission.csv',index = False )