Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<feature_engineering>
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
test_preds_bert = np.mean(test_predictions,axis=0)
 idx = np.argwhere(test.category!='CULTURE' ).flatten()
arr = 0.0001 * np.clip(np.random.normal(size=len(idx)) ,0.1,0.99)
 test_preds_bert[idx,19] = arr 
<define_variables>
train_X_0 = pd.read_csv(".. /input/feature1/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/feature2/train_X_features_865_1.csv")
 y_0 = pd.read_csv(".. /input/feature1/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/feature2/train_y1.csv", index_col=False, header=None)
 train_X = pd.concat([train_X_0, train_X_1], axis=0)
 train_X = train_X.reset_index(drop=True)
 train_X.head()
ROBERTA_PRETRAINED = '.. /input/roberta-transformers-pytorch/roberta-base'
sys.path.insert(0, data_dir)
 RobertaForSequenceClassification, RobertaConfig, AdamW,
get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP)
 print(transformers.__version__)
 def set_seeds(SEED=42):
os.environ["PYTHONHASHSEED"] = str(SEED)
 np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
SEED=2019
set_seeds(SEED)
 HEAD_TAIL = True<choose_model_class>
y = pd.concat([y_0, y_1], axis=0)
 y = y.reset_index(drop=True)
 y[0].shape
LEN_USE_FEATURES = 1646
class RobertaClassificationHead(nn.Module):

 def __init__(self, config):
super().__init__()
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
 def forward(self, features, **kwargs):
x = features[:, 0, :]
x = self.dropout(x)
 x = self.dense(x)
 x = torch.tanh(x)
 x = self.dropout(x)
 x = self.out_proj(x)
 return x
class RobertaForSequenceClassification_v2(BertPreTrainedModel):
config_class = RobertaConfig
pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
base_model_prefix = "roberta"
def __init__(self, config):
super().__init__(config)
 self.num_labels = config.num_labels
self.roberta = RobertaModel(config)
 self.classifier = RobertaClassificationHead(config)
 def forward(
 self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
labels=None,
usefeatures = None 
):
r
outputs = self.roberta(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 sequence_output = outputs[0]
pooled_output = outputs[1]
hidden_states = outputs[2]
logits = self.classifier(sequence_output)
 outputs =(logits,)+ outputs[2:]
if labels is not None:
if self.num_labels == 1:
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), labels.view(-1))
else:
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs =(loss,)+ outputs
return outputs<load_from_csv>
test_X = pd.read_csv(".. /input/feature2/test_X_features_10.csv")
 test_X.shape
train = pd.read_csv(TRAIN_CSV_PATH)
 test = pd.read_csv(TEST_CSV_PATH)
 target_cols_questions = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written']
target_cols_answers = ['answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<string_transform>
scaler = StandardScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)> max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "
":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<string_transform>
train_columns = train_X.columns
n_fold = 5
def _trim_input_answers(title, question, answer, tokenizer, max_sequence_length=512, t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"%(max_sequence_length,(t_new_len + a_new_len + q_new_len + 4)))
 q_len_head = round(q_new_len/2)
 q_len_tail = -1*(q_new_len -q_len_head)
 a_len_head = round(a_new_len/2)
 a_len_tail = -1*(a_new_len -a_len_head)
 t = t[:t_new_len]
if HEAD_TAIL :
q = q[:q_len_head]+q[q_len_tail:]
a = a[:a_len_head]+a[a_len_tail:]
else:
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_answers(title, question, answer, tokenizer, max_sequence_length):

 stoken = ['
 '] + title + ['
 
 '] + question + ['
 
 '] + answer + ['
']
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows() ,position=0, leave=True):
t, q, a = instance.question_title,instance.question_body, instance.answer
t, q, a = _trim_input_answers(t, q, a, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(t,q, a,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]<string_transform>
%%time
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 oof = np.zeros(len(train_X))
train_score = []
fold_idxs = []
predictions = np.zeros(len(test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 fold_idxs.append(val_idx)
 X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = CatBoostRegressor(n_estimators=25000, verbose=-1, objective="MAE", loss_function="MAE", boosting_type="Ordered", task_type="GPU")
 model.fit(X_tr,
y_tr,
eval_set=[(X_val, y_val)],
verbose=2500,
early_stopping_rounds=500)
 oof[val_idx] = model.predict(X_val)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += model.predict(test_X[train_columns])/ folds.n_splits
train_score.append(model.best_score_['learn']["MAE"])
 cv_score = mean_absolute_error(train_y, oof)
 print(cv_score )
<prepare_output><EOS> 
today = str(datetime.date.today())
 submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 cat_saved=pd.DataFrame(oof,columns=['oof'])
 cat_saved.to_csv('cat_oof.csv',index=False)
 submission["time_to_failure"] = predictions
submission.to_csv(f'CatBoost_submission_{cv_score:.3f}.csv', index=False)
 submission.head()