Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<save_to_csv>
!pip install.. /input/sacremoses/sacremoses-master/
!pip install.. /input/transformers/transformers-master/
submission.to_csv('submission.csv', index=False )<save_to_csv>
DATA_DIR = '.. /input/google-quest-challenge'
submission.to_csv('submission.csv', index=False )<load_from_csv>
sub = pd.read_csv(f'{DATA_DIR}/sample_submission.csv')
 sub.head()
train_df=pd.read_csv('/kaggle/input/bike-sharing-demand/train.csv')
 train_df.head(6 )<feature_engineering>
train = pd.read_csv(f'{DATA_DIR}/train.csv')
 train.head()
new_df=train_df.copy(deep=True)
 new_df['day']=new_df['datetime'].apply(lambda dateString : calendar.day_name[datetime.strptime(dateString,"%Y-%m-%d %H:%M:%S" ).weekday() ])
 new_df['datetime'] = pd.to_datetime(new_df['datetime'], format='%Y-%m-%d %H:%M:%S')
 new_df['month']=new_df['datetime'].apply(lambda x:x.month)
 new_df['hour']=new_df['datetime'].apply(lambda x:x.hour)
 new_df['year']=new_df['datetime'].apply(lambda x:x.year)
 final_df=new_df.copy(deep=True)
 final_df=new_df.drop(['datetime','temp','casual','registered'], axis=1)
 final_df.head()<categorify>
test = pd.read_csv(f'{DATA_DIR}/test.csv')
 test.head()
weather_df=pd.get_dummies(final_df['weather'],prefix='w',drop_first=True)
 yr_df=pd.get_dummies(final_df['year'],prefix='y',drop_first=True)
 month_df=pd.get_dummies(final_df['month'],prefix='m',drop_first=True)
 hour_df=pd.get_dummies(final_df['hour'],prefix='h',drop_first=True)
 season_df=pd.get_dummies(final_df['season'],prefix='s',drop_first=True)
 day_df=pd.get_dummies(final_df['day'],prefix='d',drop_first=True)
 final_df=final_df.drop(['weather','year','month','hour','season','day'], axis=1)
 final_df=final_df.join(weather_df)
 final_df=final_df.join(yr_df)
 final_df=final_df.join(month_df)
 final_df=final_df.join(hour_df)
 final_df=final_df.join(season_df)
 final_df=final_df.join(day_df )<groupby>
MAX_LEN = 512
SEP_TOKEN_ID = 102
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, df, model_type="bert-base-cased", max_len=512, content="Question_Answer", train_mode=True, labeled=True):
self.df = df
self.train_mode = train_mode
self.labeled = labeled
self.max_len = max_len
self.content = content
bert_tokenizer_path = '.. /input/pretrained-bert-models-for-pytorch/' + model_type + '-vocab.txt'
xlnet_tokenizer_path = '.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-spiece.model'
roberta_tokenizer_path = '.. /input/roberta-transformers-pytorch/roberta-base/vocab.json'
roberta_tokenizer_merges_file = '.. /input/roberta-transformers-pytorch/roberta-base/merges.txt'
if model_type == "bert-base-uncased":
self.tokenizer = BertTokenizer.from_pretrained(bert_tokenizer_path)
 elif model_type == "bert-base-cased":
self.tokenizer = BertTokenizer.from_pretrained(bert_tokenizer_path)
 elif model_type == "xlnet-base-cased":
self.tokenizer = XLNetTokenizer.from_pretrained(xlnet_tokenizer_path)
 elif model_type == "roberta-base":
self.tokenizer = RobertaTokenizer(vocab_file=roberta_tokenizer_path, merges_file=roberta_tokenizer_merges_file)
 def __getitem__(self, index):
row = self.df.iloc[index]
token_ids, seg_ids = self.get_token_ids(row)
 if self.labeled:
labels = self.get_label(row)
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def __len__(self):
return len(self.df)
 def select_tokens(self, tokens, max_num):
if len(tokens)<= max_num:
return tokens
if self.train_mode:
num_remove = len(tokens)- max_num
remove_start = random.randint(0, len(tokens)-num_remove-1)
 return tokens[:remove_start] + tokens[remove_start + num_remove:]
else:
return tokens[:max_num//2] + tokens[-(max_num - max_num//2):]
def trim_input_single_content(self, title, content, max_sequence_length=512,
t_max_len=30, c_max_len=512-30-4, num_token=3):
t = self.tokenizer.tokenize(title)
 c = self.tokenizer.tokenize(content)
 t_len = len(t)
 c_len = len(c)
 if(t_len+c_len+num_token)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
c_max_len = c_max_len + floor(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if c_max_len > c_len:
c_new_len = c_len
else:
c_new_len = c_max_len
if t_new_len+c_new_len+num_token > max_sequence_length:
raise ValueError("New sequence length should be less or equal than %d, but is %d"
%(max_sequence_length,(t_new_len+c_new_len+num_token)))
 if len(t)- t_new_len > 0:
t = t[:t_new_len//4] + t[len(t)-t_new_len+t_new_len//4:]
else:
t = t[:t_new_len]
if len(c)- c_new_len > 0:
c = c[:c_new_len//4] + c[len(c)-c_new_len+c_new_len//4:]
else:
c = c[:c_new_len]
if(len(t)+ len(c)+ num_token > max_sequence_length):
more_token = len(t)+ len(c)+ num_token - max_sequence_length
c = c[:(len(c)-more_token)]
return t, c
def trim_input(self, title, question, answer, max_sequence_length=MAX_LEN,
t_max_len=30, q_max_len=239, a_max_len=239, num_token=4):
t = self.tokenizer.tokenize(title)
 q = self.tokenizer.tokenize(question)
 a = self.tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+num_token)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+num_token > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+num_token)))
 if len(t)- t_new_len > 0:
t = t[:t_new_len//4] + t[len(t)-t_new_len+t_new_len//4:]
else:
t = t[:t_new_len]
if len(q)- q_new_len > 0:
q = q[:q_new_len//4] + q[len(q)-q_new_len+q_new_len//4:]
else:
q = q[:q_new_len]
if len(a)- a_new_len > 0:
a = a[:a_new_len//4] + a[len(a)-a_new_len+a_new_len//4:]
else:
a = a[:a_new_len]
return t, q, a
def get_token_ids(self, row):
num_token = 4
if self.content == "Question":
num_token -= 1
elif self.content == "Answer":
num_token -= 1
if self.content == "Question_Answer":
t_max_len=30
q_max_len=int(( self.max_len-t_max_len-num_token)/2)
 a_max_len=(self.max_len-t_max_len - num_token - int(( self.max_len-t_max_len-num_token)/2))
elif self.content == "Question":
t_max_len=30
q_max_len=self.max_len-t_max_len-num_token
a_max_len=0
elif self.content == "Answer":
t_max_len=30
q_max_len=0
a_max_len=self.max_len-t_max_len-num_token
else:
raise NotImplementedError
if self.content == "Question_Answer":
t_tokens, q_tokens, a_tokens = self.trim_input(row.question_title, row.question_body, row.answer, max_sequence_length=self.max_len, \
t_max_len=t_max_len, q_max_len=q_max_len, a_max_len=a_max_len, num_token=num_token)
 elif self.content == "Question":
t_tokens, c_tokens = self.trim_input_single_content(row.question_title, row.question_body, max_sequence_length=self.max_len, \
t_max_len=t_max_len, c_max_len=q_max_len, num_token=num_token)
 elif self.content == "Answer":
t_tokens, c_tokens = self.trim_input_single_content(row.question_title, row.answer, max_sequence_length=self.max_len, \
t_max_len=t_max_len, c_max_len=a_max_len, num_token=num_token)
 else:
raise NotImplementedError
if self.content == "Question_Answer":
tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + q_tokens + ['[SEP]'] + a_tokens + ['[SEP]']
elif(( self.content == "Question")or(self.content == "Answer")) :
tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + c_tokens + ['[SEP]']
else:
raise NotImplementedError
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
 if len(token_ids)< self.max_len:
token_ids += [0] *(self.max_len - len(token_ids))
ids = torch.tensor(token_ids)
 seg_ids = self.get_seg_ids(ids)
 return ids, seg_ids
def get_seg_ids(self, ids):
seg_ids = torch.zeros_like(ids)
 seg_idx = 0
first_sep = True
for i, e in enumerate(ids):
seg_ids[i] = seg_idx
if e == self.tokenizer.sep_token_id:
if first_sep:
first_sep = False
else:
seg_idx = 1
pad_idx = torch.nonzero(ids == 0)
 seg_ids[pad_idx] = 0
return seg_ids
def get_label(self, row):
return torch.tensor(row[TARGET_COLUMNS].values.astype(np.float32))
def collate_fn(self, batch):
token_ids = torch.stack([x[0] for x in batch])
 seg_ids = torch.stack([x[1] for x in batch])
 if self.labeled:
labels = torch.stack([x[2] for x in batch])
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def get_test_loader(model_type="bert-base-cased", max_len=512, content="Question_Answer", batch_size=4):
df = pd.read_csv(f'{DATA_DIR}/test.csv')
 ds_test = QuestDataset(df, model_type, max_len=max_len, content=content, train_mode=False, labeled=False)
 loader = torch.utils.data.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=2, collate_fn=ds_test.collate_fn, drop_last=False)
 loader.num = len(df)
 return loader, ds_test.tokenizer
def get_train_val_loaders(model_type="bert-base-cased", max_len=512, content="Question_Answer", batch_size=4, val_batch_size=4, ifold=0):
df = pd.read_csv(f'{DATA_DIR}/train.csv')
 df = shuffle(df, random_state=42)
 gkf = GroupKFold(n_splits=5 ).split(X=df.question_body, groups=df.question_body)
 for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold == ifold:
df_train = df.iloc[train_idx]
df_val = df.iloc[valid_idx]
break
print(df_train.shape)
 print(df_val.shape)
 ds_train = QuestDataset(df_train, model_type, max_len=max_len, content=content)
 train_loader = torch.utils.data.DataLoader(ds_train, batch_size=batch_size, shuffle=True, num_workers=2, collate_fn=ds_train.collate_fn, drop_last=True)
 train_loader.num = len(df_train)
 ds_val = QuestDataset(df_val, model_type, max_len=max_len, content=content, train_mode=False)
 val_loader = torch.utils.data.DataLoader(ds_val, batch_size=val_batch_size, shuffle=False, num_workers=2, collate_fn=ds_val.collate_fn, drop_last=False)
 val_loader.num = len(df_val)
 val_loader.df = df_val
return train_loader, val_loader, ds_train.tokenizer
def test_train_loader() :
loader, _, _ = get_train_val_loaders("xlnet-base-cased", 512, "Question", 4, 4, 1)
 for ids, seg_ids, labels in loader:
print(ids)
 print(seg_ids.numpy())
 print(labels)
 break
def test_test_loader() :
loader, _ = get_test_loader("roberta-base", 512, "Question", 4)
 for ids, seg_ids in loader:
print(ids)
 print(seg_ids)
 break
print(final_df.columns.to_series().groupby(final_df.dtypes ).groups)
 final_df.head(5 )<prepare_x_and_y>
test_test_loader()
X=final_df.iloc[:,final_df.columns!='count'].values
Y=final_df.iloc[:,5].values<train_on_grid>
test_train_loader()
model = Ridge()
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
 grid = dict()
grid['alpha'] = arange(0, 1, 0.01)
 search = GridSearchCV(model, grid, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 results = search.fit(X, Y)
 print('MSE(NEGATIVE): %.3f' % results.best_score_)
 print('Config: %s' % results.best_params_ )<train_on_grid>
class QuestModel(nn.Module):
def __init__(self, model_type="xlnet-base-cased", tokenizer=None, n_classes=30, hidden_layers=[-1, -3, -5, -7, -9]):
super(QuestModel, self ).__init__()
self.model_name = 'QuestModel'
self.model_type = model_type
self.hidden_layers = hidden_layers
if model_type == "bert-base-uncased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.output_hidden_states = True
if n_classes == 30:
bert_config.hidden_dropout_prob = 0.1
else:
bert_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/pretrained-bert-models-for-pytorch/' + model_type)
 self.bert_model = BertModel.from_pretrained(model_path, config=bert_config)
 elif model_type == "bert-base-cased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-cased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.output_hidden_states = True
model_path = os.path.join('.. /input/pretrained-bert-models-for-pytorch/' + model_type)
 self.bert_model = BertModel.from_pretrained(model_path, config=bert_config)
 elif model_type == "xlnet-base-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-base-cased-config.json'
xlnet_config = XLNetConfig.from_json_file(xlnet_model_config)
 xlnet_config.output_hidden_states = True
xlnet_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.xlnet_model = XLNetModel.from_pretrained(model_path, config=xlnet_config)
 elif model_type == "xlnet-large-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-large-cased-config.json'
xlnet_config = XLNetConfig.from_json_file(xlnet_model_config)
 xlnet_config.output_hidden_states = True
xlnet_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.xlnet_model = XLNetModel.from_pretrained(model_path, config=xlnet_config)
 elif model_type == "roberta-base":
roberta_model_config = '.. /input/roberta-transformers-pytorch/roberta-base/config.json'
roberta_config = RobertaConfig.from_json_file(roberta_model_config)
 roberta_config.output_hidden_states = True
roberta_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/roberta-transformers-pytorch/roberta-base/pytorch_model.bin')
 self.roberta_model = RobertaModel.from_pretrained(model_path, config=roberta_config)
 self.roberta_model.resize_token_embeddings(len(tokenizer))
if model_type == "bert-base-uncased":
self.hidden_size = 768
elif model_type == "bert-large-uncased":
self.hidden_size = 1024
elif model_type == "bert-base-cased":
self.hidden_size = 768
elif model_type == "xlnet-base-cased":
self.hidden_size = 768
elif model_type == "xlnet-large-cased":
self.hidden_size = 1024
elif model_type == "roberta-base":
self.hidden_size = 768
else:
raise NotImplementedError
self.fc_1 = nn.Linear(self.hidden_size * len(hidden_layers), self.hidden_size)
 self.fc = nn.Linear(self.hidden_size, n_classes)
 self.selu = nn.SELU()
self.relu = nn.ReLU()
self.tanh = nn.Tanh()
self.dropouts = nn.ModuleList([
nn.Dropout(0.5)for _ in range(5)
 ])
 def forward(self, ids, seg_ids):
attention_mask =(ids > 0)
 if(( self.model_type == "bert-base-uncased")\
or(self.model_type == "bert-base-cased")\
or(self.model_type == "bert-large-uncased")\
or(self.model_type == "bert-large-cased")) :
outputs = self.bert_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[2]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
elif(( self.model_type == "xlnet-base-cased")\
or(self.model_type == "xlnet-large-cased")) :
attention_mask = attention_mask.float()
outputs = self.xlnet_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[1]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
elif(self.model_type == "roberta-base"):
attention_mask = attention_mask.float()
outputs = self.roberta_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[2]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
for j, dropout in enumerate(self.dropouts):
if j == 0:
logit = self.fc(dropout(h))
else:
logit += self.fc(dropout(h))
return logit / len(self.dropouts)
 def test_model(model_type="bert-base-cased", hidden_layers=[-1, -3, -5, -7, -9]):
x = torch.tensor([[1,2,3,4,5, 0, 0], [1,2,3,4,5, 0, 0]])
 seg_ids = torch.tensor([[0,0,0,0,0, 0, 0], [0,0,0,0,0, 0, 0]])
 model = QuestModel(model_type=model_type, hidden_layers=hidden_layers)
 y = model(x, seg_ids)
 print(y )
model = Lasso()
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
 grid = dict()
grid['alpha'] = arange(0, 1, 0.01)
 search = GridSearchCV(model, grid, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 results = search.fit(X, Y)
 print('MSE(NEGATIVE): %.3f' % results.best_score_)
 print('Config: %s' % results.best_params_ )<train_on_grid>
test_model(model_type="bert-base-cased", hidden_layers=[-3, -4, -5, -6, -7] )
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
 dtm = DecisionTreeRegressor(random_state=42)
 param_grid = {"criterion": ["mse", "mae"],
}
search = GridSearchCV(dtm,param_grid, scoring='neg_mean_squared_error', cv=cv)
 results = search.fit(X, Y)
 print('MSE(NEGATIVE): %.3f' % results.best_score_)
 print('Config: %s' % results.best_params_ )<train_on_grid>
def create_bert_base_uncased_models() :
models = []
for i in range(10):
model = QuestModel(model_type="bert-base-uncased", hidden_layers=[-1, -3, -5, -7, -9])
 model.load_state_dict(torch.load(f'.. /input/qabertuncasedaugdiffv2swa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_models() :
models = []
for i in range(10):
model = QuestModel(model_type="bert-base-cased", hidden_layers=[-1, -3, -5, -7, -9])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffv2swa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugquestionanswerswa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_uncased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-uncased", n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qabertbaseuncasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_uncased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-uncased", n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qabertbaseuncasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-cased", n_classes=21, hidden_layers=[-2, -4, -6, -8, -10])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-cased", n_classes=9, hidden_layers=[-2, -4, -6, -8, -10])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=30, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabaseaugdiffswa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_question_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_answer_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
 dtm = DecisionTreeRegressor(random_state=42)
 param_grid = {"criterion": ["mse", "mae"],
"max_depth": [2, 6, 8],
}
search = GridSearchCV(dtm,param_grid, scoring='neg_mean_squared_error', cv=cv)
 results = search.fit(X, Y)
 print('MSE(NEGATIVE): %.3f' % results.best_score_)
 print('Config: %s' % results.best_params_ )<train_on_grid>
def predict(models, test_loader):
all_scores = []
with torch.no_grad() :
for ids, seg_ids in tqdm(test_loader, total=test_loader.num // test_loader.batch_size):
ids, seg_ids = ids.cuda() , seg_ids.cuda()
scores = []
for model in models:
model = model.cuda()
outputs = torch.sigmoid(model(ids, seg_ids)).cpu()
scores.append(outputs)
 all_scores.append(torch.mean(torch.stack(scores), 0))
all_scores = torch.cat(all_scores, 0 ).numpy()
return all_scores
grid = dict()
grid['n_estimators'] = [1000]
grid['max_depth'] = [125,150,175]
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
 model = RandomForestRegressor()
search = GridSearchCV(model,param_grid=grid, scoring='neg_mean_squared_error', cv=cv)
 results = search.fit(X, Y)
 print('MSE(NEGATIVE): %.3f' % results.best_score_)
 print('Config: %s' % results.best_params_ )<train_on_grid>
test_loader, tokenizer = get_test_loader(model_type="roberta-base", content="Question_Answer", batch_size=32 )
def grid_search() :
print('lets go')
 model = GradientBoostingRegressor()
grid = dict()
grid['n_estimators'] = [4000]
grid['learning_rate'] = [ 0.001, 0.01, 0.1]
grid['max_depth'] = [4]
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
 grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='neg_mean_squared_error')
 grid_search=grid_search.fit(X,Y)
 best_accuracy=grid_search.best_score_
best_parameters=grid_search.best_params_
print(best_accuracy)
 print(best_parameters)
 grid_search()<train_model>
roberta_base_models = create_roberta_base_models(tokenizer)
 roberta_base_preds = predict(roberta_base_models, test_loader )
rgr=GradientBoostingRegressor(learning_rate=0.1,n_estimators=4000, max_depth=4)
 rgr.fit(X,Y )<feature_engineering>
del roberta_base_models, test_loader
torch.cuda.empty_cache()
gc.collect()
test_df=pd.read_csv('/kaggle/input/bike-sharing-demand/test.csv')
 test_df['day']=test_df['datetime'].apply(lambda dateString : calendar.day_name[datetime.strptime(dateString,"%Y-%m-%d %H:%M:%S" ).weekday() ])
 test_df['datetime']=pd.to_datetime(test_df['datetime'], format='%Y-%m-%d %H:%M:%S')
 test_df['month']=test_df['datetime'].apply(lambda x:x.month)
 test_df['hour']=test_df['datetime'].apply(lambda x:x.hour)
 test_df['year']=test_df['datetime'].apply(lambda x:x.year)
 test_df=test_df.drop(['datetime','temp'], axis=1)
 weather_df=pd.get_dummies(test_df['weather'],prefix='w',drop_first=True)
 yr_df=pd.get_dummies(test_df['year'],prefix='y',drop_first=True)
 month_df=pd.get_dummies(test_df['month'],prefix='m',drop_first=True)
 hour_df=pd.get_dummies(test_df['hour'],prefix='h',drop_first=True)
 season_df=pd.get_dummies(test_df['season'],prefix='s',drop_first=True)
 day_df=pd.get_dummies(test_df['day'],prefix='d',drop_first=True)
 test_df=test_df.drop(['weather','year','month','hour','season','day'], axis=1)
 test_df=test_df.join(weather_df)
 test_df=test_df.join(yr_df)
 test_df=test_df.join(month_df)
 test_df=test_df.join(hour_df)
 test_df=test_df.join(season_df)
 test_df=test_df.join(day_df )<save_to_csv>
test_loader, _ = get_test_loader(model_type="xlnet-base-cased", batch_size=32 )
temp=pd.read_csv('.. /input/bike-sharing-demand/sampleSubmission.csv')
 X_test=test_df.iloc[:,:].values
y_output=rgr.predict(X_test)
 y_output
op=pd.DataFrame({'count':np.exp(y_output)})
 op['datetime']=temp['datetime']
op.to_csv('finalSubmission.csv', index=False )<feature_engineering>
xlnet_base_cased_models = create_xlnet_base_cased_models()
xlnet_base_cased_preds = predict(xlnet_base_cased_models, test_loader )
train = pd.read_csv(".. /input/train.csv", parse_dates = ["datetime"])
 test = pd.read_csv(".. /input/test.csv", parse_dates = ["datetime"])
 train["year"] = train["datetime"].dt.year
train["hour"] = train["datetime"].dt.hour
train["dayofweek"] = train["datetime"].dt.dayofweek
test["year"] = test["datetime"].dt.year
test["hour"] = test["datetime"].dt.hour
test["dayofweek"] = test["datetime"].dt.dayofweek
y_casual = np.log1p(train.casual)
 y_registered = np.log1p(train.registered)
 train.drop(["datetime", "windspeed", "casual", "registered", "count"], 1, inplace=True)
 test.drop(["datetime", "windspeed", ], 1, inplace=True)
 hyperparameters = { 'colsample_bytree': 0.725, 'learning_rate': 0.013,
'num_leaves': 56, 'reg_alpha': 0.754, 'reg_lambda': 0.071,
'subsample': 0.523, 'n_estimators': 1093}
model = lgb.LGBMRegressor(**hyperparameters)
 model.fit(train, y_casual)
 preds1 = model.predict(test)
 hyperparameters = { 'colsample_bytree': 0.639, 'learning_rate': 0.011,
'num_leaves': 30, 'reg_alpha': 0.351, 'reg_lambda': 0.587,
'subsample': 0.916, 'n_estimators': 2166}
model = lgb.LGBMRegressor(**hyperparameters,)
 model.fit(train, y_registered)
 preds2 = model.predict(test)
 submission=pd.read_csv(".. /input/sampleSubmission.csv")
 submission["count"] = np.expm1(preds1)+ np.expm1(preds2)
<feature_engineering>
del xlnet_base_cased_models, test_loader
torch.cuda.empty_cache()
gc.collect()
pd.options.display.max_rows = 200
submission["holiday"] = test["holiday"]
submission.loc[(submission["holiday"]==1)] 
<normalization>
test_loader, _ = get_test_loader(model_type="xlnet-base-cased", content="Question", batch_size=32 )
submission.iloc[1258:1269, 1]= submission.iloc[1258:1269, 1]*0.5
submission.iloc[4492:4515, 1]= submission.iloc[4492:4515, 1]*0.5
submission.iloc[6308:6330, 1]= submission.iloc[6308:6330, 1]*0.5
submission.iloc[3041:3063, 1]= submission.iloc[3041:3063, 1]*0.5
submission.iloc[6332:6354, 1]= submission.iloc[6332:6354, 1]*0.5
submission.iloc[3065:3087, 1]= submission.iloc[3065:3087, 1]*0.5
submission.iloc[5992:6015, 1]= submission.iloc[5992:6015, 1]*0.5
submission.iloc[2771:2794, 1]= submission.iloc[2771:2794, 1]*0.5 
<save_to_csv>
xlnet_base_cased_question_models = create_xlnet_base_cased_question_models()
xlnet_base_cased_question_preds = predict(xlnet_base_cased_question_models, test_loader )
submission.drop("holiday",1,inplace=True)
 submission.to_csv("allrf2.csv", index=False )<load_from_csv>
del xlnet_base_cased_question_models, test_loader
torch.cuda.empty_cache()
gc.collect()
train = pd.read_csv("/kaggle/input/bike-sharing-demand/train.csv")
 train.head(10)
<load_from_csv>
test_loader, _ = get_test_loader(model_type="xlnet-base-cased", content="Answer", batch_size=32 )
test = pd.read_csv("/kaggle/input/bike-sharing-demand/test.csv")
 test.head(10 )<prepare_x_and_y>
xlnet_base_cased_answer_models = create_xlnet_base_cased_answer_models()
xlnet_base_cased_answer_preds = predict(xlnet_base_cased_answer_models, test_loader )
y = train['count']
y.sort_values()<prepare_x_and_y>
del xlnet_base_cased_answer_models, test_loader
torch.cuda.empty_cache()
gc.collect()
y = np.log(train['count'])
 y<load_from_csv>
xlnet_base_cased_question_answer_preds = np.concatenate([xlnet_base_cased_question_preds, xlnet_base_cased_answer_preds], axis=1 )
train['datetime'] = train['datetime'].astype('datetime64')
 train.dtypes 
<load_from_csv>
test_loader, tokenizer = get_test_loader(model_type="roberta-base", content="Question", batch_size=32 )
test['datetime'] = test['datetime'].astype('datetime64')
 test.dtypes<feature_engineering>
roberta_base_question_models = create_roberta_base_question_models(tokenizer)
 roberta_base_question_preds = predict(roberta_base_question_models, test_loader )
train['year'] = train['datetime'].dt.year
train['weekday'] = train['datetime'].dt.weekday
train['hour'] = train['datetime'].dt.hour
train.head()<feature_engineering>
del roberta_base_question_models, test_loader, tokenizer
torch.cuda.empty_cache()
gc.collect()
test['year'] = test['datetime'].dt.year
test['weekday'] = test['datetime'].dt.weekday
test['hour'] = test['datetime'].dt.hour
test.head()<data_type_conversions>
test_loader, tokenizer = get_test_loader(model_type="roberta-base", content="Answer", batch_size=32 )
<train_model>
roberta_base_answer_models = create_roberta_base_answer_models(tokenizer)
 roberta_base_answer_preds = predict(roberta_base_answer_models, test_loader )
<drop_column>
del roberta_base_answer_models, test_loader, tokenizer
torch.cuda.empty_cache()
gc.collect()
train = train.drop(['datetime', 'casual', 'registered', 'count'], 1)
 test = test.drop('datetime', 1)
<import_modules>
roberta_base_question_answer_preds = np.concatenate([roberta_base_question_preds, roberta_base_answer_preds], axis=1 )
<train_on_grid>
test_loader, _ = get_test_loader(model_type="bert-base-cased", content="Question", batch_size=32 )
<train_model>
bert_base_cased_question_models = create_bert_base_cased_question_models()
bert_base_cased_question_preds = predict(bert_base_cased_question_models, test_loader )
xgb = XGBRegressor()
xgb.fit(train, y)
 preds = predict(test )<load_from_csv>
del bert_base_cased_question_models, test_loader
torch.cuda.empty_cache()
gc.collect()
sample = pd.read_csv("/kaggle/input/bike-sharing-demand/sampleSubmission.csv")
 sample.head()<prepare_output>
test_loader, _ = get_test_loader(model_type="bert-base-cased", content="Answer", batch_size=32 )
sample['count'] = np.exp(preds)
 sample.head()<save_to_csv>
bert_base_cased_answer_models = create_bert_base_cased_answer_models()
bert_base_cased_answer_preds = predict(bert_base_cased_answer_models, test_loader )
sample.to_csv("sample.csv", index = False)
<feature_engineering>
del bert_base_cased_answer_models, test_loader
torch.cuda.empty_cache()
gc.collect()
train2 = pd.read_csv("/kaggle/input/bike-sharing-demand/train.csv", parse_dates=['datetime'])
 train2['year'] = train2['datetime'].dt.year
train2['month'] = train2['datetime'].dt.month
train2['day'] = train2['datetime'].dt.day
train2['weekday'] = train2['datetime'].dt.weekday
train2['hour'] = train2['datetime'].dt.hour
test2 = pd.read_csv('/kaggle/input/bike-sharing-demand/test.csv', parse_dates=['datetime'])
 test2['day'] = test2['datetime'].dt.day
train2.groupby('hour')['count'].median() 
<count_values>
bert_base_cased_question_answer_preds = np.concatenate([bert_base_cased_question_preds, bert_base_cased_answer_preds], axis=1 )
train2['datetime'].dt.month.value_counts()<load_from_csv>
test_loader, _ = get_test_loader(model_type="bert-base-uncased", content="Question", batch_size=32 )
train = pd.read_csv(".. /input/train.csv" )<load_from_csv>
bert_base_uncased_question_models = create_bert_base_uncased_question_models()
bert_base_uncased_question_preds = predict(bert_base_uncased_question_models, test_loader )
train = pd.read_csv(".. /input/train.csv", parse_dates = ["datetime"])
 test = pd.read_csv(".. /input/test.csv", parse_dates = ["datetime"] )<data_type_conversions>
del bert_base_uncased_question_models, test_loader
torch.cuda.empty_cache()
gc.collect()
train["datetime"] = train["datetime"].astype("datetime64" )<data_type_conversions>
test_loader, _ = get_test_loader(model_type="bert-base-uncased", content="Answer", batch_size=32 )
train["datetime"] = pd.to_datetime(train["datetime"] )<feature_engineering>
bert_base_uncased_answer_models = create_bert_base_uncased_answer_models()
bert_base_uncased_answer_preds = predict(bert_base_uncased_answer_models, test_loader )
train["datetime"].dt.hour<feature_engineering>
del bert_base_uncased_answer_models, test_loader
torch.cuda.empty_cache()
gc.collect()
train["year"] = train["datetime"].dt.year
train["hour"] = train["datetime"].dt.hour
train["dayofweek"] = train["datetime"].dt.dayofweek
test["year"] = test["datetime"].dt.year
test["hour"] = test["datetime"].dt.hour
test["dayofweek"] = test["datetime"].dt.dayofweek<count_values>
bert_base_uncased_question_answer_preds = np.concatenate([bert_base_uncased_question_preds, bert_base_uncased_answer_preds], axis=1 )
train["datetime"].dt.day.value_counts()<train_model>
test_loader, _ = get_test_loader(model_type="bert-base-cased", batch_size=32 )
<prepare_x_and_y>
bert_base_cased_models = create_bert_base_cased_models()
bert_base_cased_preds = predict(bert_base_cased_models, test_loader )
y_train = train["count"]<drop_column>
del bert_base_cased_models, test_loader
torch.cuda.empty_cache()
gc.collect()
train = train.drop(["datetime", "windspeed", "casual", "registered", "count"], 1)
 test.drop(["datetime", "windspeed"], 1, inplace=True )<prepare_output>
test_loader, _ = get_test_loader(model_type="bert-base-uncased", batch_size=32 )
y_train = np.log1p(y_train )<import_modules>
bert_base_uncased_models = create_bert_base_uncased_models()
bert_base_uncased_preds = predict(bert_base_uncased_models, test_loader )
<import_modules>
del bert_base_uncased_models, test_loader
torch.cuda.empty_cache()
gc.collect()
from sklearn.ensemble import RandomForestRegressor<import_modules>
preds =(( bert_base_uncased_preds + bert_base_uncased_question_answer_preds)/2.0 \
+(xlnet_base_cased_preds + xlnet_base_cased_question_answer_preds)/2.0 \
+(bert_base_cased_preds + bert_base_cased_question_answer_preds)/2.0 \
+(roberta_base_preds + roberta_base_question_answer_preds)/2.0)/4.0
from sklearn.ensemble import RandomForestRegressor<choose_model_class>
sub[TARGET_COLUMNS] = bert_base_uncased_question_answer_preds
sub.to_csv('submission_bert_base_uncased.csv', index=False)
 sub[TARGET_COLUMNS] =roberta_base_preds
sub.to_csv('submission_roberta_base.csv', index=False)
rf = RandomForestRegressor(n_estimators=100 )<train_model>
sub[TARGET_COLUMNS] = preds
rf.fit(train, y_train )<predict_on_test>
test = pd.read_csv(f'{DATA_DIR}/test.csv' )
preds = rf.predict(test )<load_from_csv>
test = test.set_index('qa_id' ).join(sub.set_index('qa_id'))
submission=pd.read_csv(".. /input/sampleSubmission.csv" )<prepare_output>
def postprocessing(oof_df):
scaler = MinMaxScaler()
type_one_column_list = [
'question_conversational', \
'question_has_commonly_accepted_answer', \
'question_not_really_a_question', \
'question_type_choice', \
'question_type_compare', \
'question_type_consequence', \
'question_type_definition', \
'question_type_entity', \
'question_type_instructions',
]
oof_df[type_one_column_list] = scaler.fit_transform(oof_df[type_one_column_list])
 tmp = oof_df.copy(deep=True)
 for column in type_one_column_list:
oof_df.loc[tmp[column] <= 0.16667, column] = 0
oof_df.loc[(tmp[column] > 0.16667)&(tmp[column] <= 0.41667), column] = 0.333333
oof_df.loc[(tmp[column] > 0.41667)&(tmp[column] <= 0.58333), column] = 0.500000
oof_df.loc[(tmp[column] > 0.58333)&(tmp[column] <= 0.73333), column] = 0.666667
oof_df.loc[(tmp[column] > 0.73333), column] = 1
type_three_column_list = [
'question_interestingness_self',
]
scaler = MinMaxScaler(feature_range=(0, 1))
oof_df[type_three_column_list] = scaler.fit_transform(oof_df[type_three_column_list])
 tmp[type_three_column_list] = scaler.fit_transform(tmp[type_three_column_list])
 for column in type_three_column_list:
oof_df.loc[tmp[column] <= 0.385, column] = 0.333333
oof_df.loc[(tmp[column] > 0.385)&(tmp[column] <= 0.47), column] = 0.444444
oof_df.loc[(tmp[column] > 0.47)&(tmp[column] <= 0.525), column] = 0.5
oof_df.loc[(tmp[column] > 0.525)&(tmp[column] <= 0.605), column] = 0.555556
oof_df.loc[(tmp[column] > 0.605)&(tmp[column] <= 0.715), column] = 0.666667
oof_df.loc[(tmp[column] > 0.715)&(tmp[column] <= 0.8), column] = 0.833333
oof_df.loc[(tmp[column] > 0.8)&(tmp[column] <= 0.94), column] = 0.888889
oof_df.loc[(tmp[column] > 0.94), column] = 1
type_four_column_list = [
'answer_satisfaction'
]
scaler = MinMaxScaler(feature_range=(0.2, 1))
oof_df[type_four_column_list] = scaler.fit_transform(oof_df[type_four_column_list])
 tmp[type_four_column_list] = scaler.fit_transform(tmp[type_four_column_list])
 for column in type_four_column_list:
oof_df.loc[tmp[column] <= 0.233, column] = 0.200000
oof_df.loc[(tmp[column] > 0.233)&(tmp[column] <= 0.283), column] = 0.266667
oof_df.loc[(tmp[column] > 0.283)&(tmp[column] <= 0.315), column] = 0.300000
oof_df.loc[(tmp[column] > 0.315)&(tmp[column] <= 0.365), column] = 0.333333
oof_df.loc[(tmp[column] > 0.365)&(tmp[column] <= 0.433), column] = 0.400000
oof_df.loc[(tmp[column] > 0.433)&(tmp[column] <= 0.483), column] = 0.466667
oof_df.loc[(tmp[column] > 0.483)&(tmp[column] <= 0.517), column] = 0.500000
oof_df.loc[(tmp[column] > 0.517)&(tmp[column] <= 0.567), column] = 0.533333
oof_df.loc[(tmp[column] > 0.567)&(tmp[column] <= 0.633), column] = 0.600000
oof_df.loc[(tmp[column] > 0.633)&(tmp[column] <= 0.683), column] = 0.666667
oof_df.loc[(tmp[column] > 0.683)&(tmp[column] <= 0.715), column] = 0.700000
oof_df.loc[(tmp[column] > 0.715)&(tmp[column] <= 0.767), column] = 0.733333
oof_df.loc[(tmp[column] > 0.767)&(tmp[column] <= 0.833), column] = 0.800000
oof_df.loc[(tmp[column] > 0.883)&(tmp[column] <= 0.915), column] = 0.900000
oof_df.loc[(tmp[column] > 0.915)&(tmp[column] <= 0.967), column] = 0.933333
oof_df.loc[(tmp[column] > 0.967), column] = 1
oof_values = oof_df[TARGET_COLUMNS].values
DEGREE = len(oof_df)//45*9
oof_values = np.around(oof_values * DEGREE)/ DEGREE
oof_df[TARGET_COLUMNS] = oof_values
return oof_df
submission["count"] = preds<prepare_output>
test = postprocessing(test )
submission["count"] = np.expm1(preds )<save_to_csv>
for column in TARGET_COLUMNS:
print(test[column].value_counts() )
submission.to_csv("allrf.csv", index=False )<define_variables>
sub = test[TARGET_COLUMNS].reset_index()
sample = 'This movie sucks'
fixed_size = 1024
print(sample.split())
 features = [(hash(f)%fixed_size, 1)for f in sample.split() ]
print(features )<load_from_disk>
sub[ sub[TARGET_COLUMNS] > 1.0] = 1.0
!unzip /kaggle/input/word2vec-nlp-tutorial/labeledTrainData.tsv.zip
!unzip /kaggle/input/word2vec-nlp-tutorial/unlabeledTrainData.tsv.zip
!unzip /kaggle/input/word2vec-nlp-tutorial/testData.tsv.zip<define_variables>
test = pd.read_csv(f'{DATA_DIR}/test.csv' )
PATH = '/kaggle/working/'<import_modules>
n=test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)
import re
import random
from math import exp, log
from datetime import datetime
from operator import itemgetter<string_transform>
sub['question_type_spelling'] = spelling
def clean(s):
return " ".join(re.findall(r'\w+', s, flags=re.UNICODE)).lower()<load_from_csv>
sub['question_type_spelling'].value_counts()
def get_data_tsv(loc_dataset, opts):
for e, line in enumerate(open(loc_dataset, 'rb')) :
if e > 0:
r = line.decode('utf-8' ).strip().split('\t')
 id = r[0]
if opts['clean']:
try:
r[2] = clean(r[2])
 except:
r[1] = clean(r[1])
 if len(r)== 3:
features = [(hash(f)%opts['D'], 1)for f in r[2].split() ]
label = int(r[1])
 else:
features = [(hash(f)%opts['D'], 1)for f in r[1].split() ]
label = 1
if opts['2grams']:
for i in range(len(features)-1):
features.append(
(hash(str(features[i][0])+str(features[i+1][0])) %opts['D'], 1))
yield label, id, features<concatenate>
sub.to_csv('submission.csv', index=False )
<train_model><EOS> 
sub.to_csv('submission.csv', index=False )