Kaggle competition: Santander Value Prediction Challenge
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
optimizer = AdamW(model.parameters() , lr = 2e-5)
 total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = 0,num_training_steps = total_steps )<train_model>
from sklearn.decomposition import PCA
from sklearn.preprocessing import normalize
for epoch in range(epochs):
model.train()
total_loss, total_val_loss = 0, 0
for step, batch in enumerate(train_dataloader):
model.zero_grad()
loss, logits = model(batch[0].to(device),
token_type_ids = None,
attention_mask =(batch[0] > 0 ).to(device),
labels = batch[1].to(device))
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
model.eval()
for i, batch in enumerate(val_dataloader):
with torch.no_grad() :
loss, logits = model(batch[0].to(device),
token_type_ids = None,
attention_mask =(batch[0] > 0 ).to(device),
labels = batch[1].to(device))
total_val_loss += loss.item()
avg_train_loss = total_loss / len(train_dataloader)
 avg_val_loss = total_val_loss / len(val_dataloader)
 print('Train Loss : ', avg_train_loss)
 print('Validation Loss: ', avg_val_loss)
 print('
' )<create_dataframe>
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import GridSearchCV
import lightgbm as lgb
import xgboost as xgb
text_values = test.final_text.values
all_input_ids = encode_fn(text_values)
 pred_data = TensorDataset(all_input_ids)
 pred_dataloader = DataLoader(pred_data, batch_size = batch_size, shuffle = False )<predict_on_test>
train = pd.read_csv(".. /input/santander-value-prediction-challenge/train.csv")
 test = pd.read_csv(".. /input/santander-value-prediction-challenge/test.csv" )
model.eval()
preds = []
for i,(batch,)in enumerate(pred_dataloader):
with torch.no_grad() :
outputs = model(batch.to(device),
token_type_ids = None,
attention_mask =(batch > 0 ).to(device))
logits = outputs[0]
logits = logits.detach().cpu().numpy()
preds.append(logits)
 final_preds = np.concatenate(preds, axis=0)
 final_preds = np.argmax(final_preds, axis=1 )<save_to_csv>
all_zero_columns=[i for i in train.columns if train[i].nunique() ==1]
train=train[[i for i in train.columns if i not in all_zero_columns]]
test=test[[i for i in test.columns if i not in all_zero_columns]]
duplicte_columns=['d60ddde1b', 'acc5b709d', '912836770', 'f8d75792f', 'f333a5f60']
train=train[[i for i in train.columns if i not in duplicte_columns]]
test=test[[i for i in test.columns if i not in duplicte_columns]]
X = np.log1p(train.drop(["ID", "target"], axis=1))
y = np.log1p(train["target"].values)
 test = np.log1p(test.drop(["ID"], axis=1))
submission = pd.DataFrame()
submission['id'] = test['id']
submission['target'] = final_preds
submission.to_csv('submission.csv', index=False )<set_options>
lgbbest=lgb.LGBMRegressor(bagging_fraction=0.5, boosting_type='gbdt', class_weight=None,
colsample_bytree=1.0, feature_fraction=0.5,
importance_type='split', learning_rate=0.01, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=500, n_jobs=-1, num_leaves=130,
objective='regression', random_state=42, reg_alpha=0.0,
reg_lambda=1, silent=True, subsample=1.0,
subsample_for_bin=200000, subsample_freq=0 )
<load_from_csv><EOS> 
lgbbest.fit(X, y)
 y_pred = lgbbest.predict(test)
 sub = pd.read_csv('.. /input/santander-value-prediction-challenge/sample_submission.csv')
 sub["target"] = np.expm1(y_pred)
 sub.to_csv('submit_lgbbestpara_allfeatures.csv', index=False )