Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<categorify>
sub1 = pd.read_csv('.. /input/love-is-the-answer-new-blendwinner/blend_new1.csv')
 sub2 = pd.read_csv('.. /input/love-is-the-answer-new-blendwinner/blend_new1.csv')
 sub3 = pd.read_csv('.. /input/love-is-the-answer-new-blendwinner/blend_new1.csv')
 train = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv')
 print(train.shape, test.shape )
def preprocess(text):
text=text.lower()
text = re.sub(r'https?:\/\/.*[\r
]*', '', text)
 text = re.sub(r'http?:\/\/.*[\r
]*', '', text)
 text=text.replace(r'&amp;?',r'and')
 text=text.replace(r'&lt;',r'<')
 text=text.replace(r'&gt;',r'>')
 text = re.sub(r"(?:\@)\w+", '', text)
 text=text.encode("ascii",errors="ignore" ).decode()
text=re.sub(r'[:"
text=re.sub(r'[!]+','!',text)
 text=re.sub(r'[?]+','?',text)
 text=re.sub(r'[.]+','.',text)
 text=re.sub(r"'","",text)
 text=re.sub(r"\(","",text)
 text=re.sub(r"\)","",text)
 text=" ".join(text.split())
 return text
df_train['text'] = df_train['text'].apply(preprocess)
 df_test['text'] = df_test['text'].apply(preprocess)
 df_train=df_train[df_train["text"]!='']<count_values>
leak_col = []
for c in col:
leak1 = np.sum(( train[c]==train['target'] ).astype(int))
leak2 = np.sum(((( train[c] - train['target'])/ train['target'])< 0.05 ).astype(int))
if leak1 > 30 and leak2 > 3500:
leak_col.append(c)
 print(len(leak_col))
df_train["target"].value_counts()<define_variables>
col = list(leak_col)
 train = train[col + ['ID', 'target']]
test = test[col + ['ID']]
texts = df_train.text.values
labels = df_train.target.values<load_pretrained>
train["nz_mean"] = train[col].apply(lambda x: x[x!=0].mean() , axis=1)
 train["nz_max"] = train[col].apply(lambda x: x[x!=0].max() , axis=1)
 train["nz_min"] = train[col].apply(lambda x: x[x!=0].min() , axis=1)
 train["ez"] = train[col].apply(lambda x: len(x[x==0]), axis=1)
 train["mean"] = train[col].apply(lambda x: x.mean() , axis=1)
 train["max"] = train[col].apply(lambda x: x.max() , axis=1)
 train["min"] = train[col].apply(lambda x: x.min() , axis=1)
 test["nz_mean"] = test[col].apply(lambda x: x[x!=0].mean() , axis=1)
 test["nz_max"] = test[col].apply(lambda x: x[x!=0].max() , axis=1)
 test["nz_min"] = test[col].apply(lambda x: x[x!=0].min() , axis=1)
 test["ez"] = test[col].apply(lambda x: len(x[x==0]), axis=1)
 test["mean"] = test[col].apply(lambda x: x.mean() , axis=1)
 test["max"] = test[col].apply(lambda x: x.max() , axis=1)
 test["min"] = test[col].apply(lambda x: x.min() , axis=1)
 col += ['nz_mean', 'nz_max', 'nz_min', 'ez', 'mean', 'max', 'min']
tokenizer = ElectraTokenizer.from_pretrained('google/electra-base-discriminator')
 model = ElectraForSequenceClassification.from_pretrained('google/electra-base-discriminator',num_labels=2)
 model.cuda()<categorify>
for i in range(2, 100):
train['index'+str(i)] =(( train.index + 2)% i == 0 ).astype(int)
 test['index'+str(i)] =(( test.index + 2)% i == 0 ).astype(int)
 col.append('index'+str(i))
indices=tokenizer.batch_encode_plus(texts,max_length=64,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids=indices["input_ids"]
attention_masks=indices["attention_mask"]<split>
test = pd.merge(test, sub3, how='left', on='ID', )
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=42, test_size=0.2)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=42, test_size=0.2 )<data_type_conversions>
train = train.replace(0, np.nan)
 test = test.replace(0, np.nan)
 train = pd.concat(( train, test), axis=0, ignore_index=True )
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels, dtype=torch.long)
 validation_labels = torch.tensor(validation_labels, dtype=torch.long)
 train_masks = torch.tensor(train_masks, dtype=torch.long)
 validation_masks = torch.tensor(validation_masks, dtype=torch.long )<load_pretrained>
test['target'] = 0.0
folds = 5
for fold in range(folds):
x1, x2, y1, y2 = model_selection.train_test_split(train[col], np.log1p(train.target.values), test_size=0.20, random_state=fold)
 params = {'learning_rate': 0.02, 'max_depth': 7, 'boosting': 'gbdt', 'objective': 'regression', 'metric': 'rmse', 'is_training_metric': True, 'feature_fraction': 0.9, 'bagging_fraction': 0.8, 'bagging_freq': 5, 'seed':fold}
model = lgb.train(params, lgb.Dataset(x1, label=y1), 3000, lgb.Dataset(x2, label=y2), verbose_eval=200, early_stopping_rounds=100)
 test['target'] += np.expm1(model.predict(test[col], num_iteration=model.best_iteration))
test['target'] /= folds
test[['ID', 'target']].to_csv('submission.csv', index=False )
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )<compute_test_metric>
b1 = sub1.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('submission.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.8)+(b1['dp2'] * 0.2)
 b1[['ID','target']].to_csv('blend01.csv', index=False )
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<categorify>
b1 = sub2.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend01.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.8)+(b1['dp2'] * 0.2)
 b1[['ID','target']].to_csv('blend02.csv', index=False )
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<define_variables>
b1 = sub2.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend02.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.5)+(b1['dp2'] * 0.5)
 b1[['ID','target']].to_csv('blend03.csv', index=False )
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )<train_model>
b1 = sub3.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend03.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.6)+(b1['dp2'] * 0.4)
 b1[['ID','target']].to_csv('blend04.csv', index=False )
<define_variables><EOS> 
!rm submission.csv
!rm blend01.csv
!rm blend02.csv
!rm blend03.csv